1 files changed, 18 insertions, 2 deletions
diff --git a/python/citco_submission.py b/python/citco_submission.py
index b17272fe..6e46b84c 100644
--- a/python/citco_submission.py
+++ b/python/citco_submission.py
@@ -1,10 +1,24 @@
 from stat import S_ISREG
 import time
+from contextlib import contextmanager
 from report_ops.utils import CitcoSubmission
 from paramiko.ssh_exception import SSHException
 import logging
 
 
+@contextmanager
+def retry_on_exception_sftp():
+    retries = 5
+    for i in range(retries):
+        try:
+            yield
+        except (SSHException, OSError) as e:
+            if i == retries - 1:
+                raise e
+            else:
+                time.sleep(60)
+
+
 def run():
     CitcoSubmission.init_sftp()
     while True:
@@ -18,8 +32,10 @@ def run():
                         continue
             CitcoSubmission.commit()
         except (SSHException, OSError):
-            CitcoSubmission._sftp.client.close()
-            CitcoSubmission.init_sftp()
+            with retry_on_exception_sftp():
+                CitcoSubmission._sftp.client.close()
+                CitcoSubmission.init_sftp()
+
         time.sleep(60)
         CitcoSubmission.check_cache()