From 5af54c41e162d10ea258c5bc3f308b54b0a72bdd Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Feb 28 2016 21:49:39 +0000
Subject: [PATCH 1/3] If failing to process a repo, wait 30 seconds and try again


---

diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md
index 397f289..c4af3ce 100644
--- a/mdapi-get_repo_md
+++ b/mdapi-get_repo_md
@@ -41,6 +41,7 @@ import multiprocessing
 import os
 import shutil
 import tempfile
+import time
 import hashlib
 import xml.etree.ElementTree as ET
 
@@ -505,10 +506,15 @@ def main():
 
     # In serial
     for t in itertools.product(
-        [CONFIG.get('DB_FOLDER', '/var/tmp')],
-        repositories):
-
-        process_repo(t)
+            [CONFIG.get('DB_FOLDER', '/var/tmp')],
+            repositories):
+
+        try:
+            process_repo(t)
+        except OSError:
+            # Most often due to an invalid stream, so let's try a second time
+            time.sleep(30)
+            process_repo(t)
 
     return 0
 

From 1d5465822ee6ee76bfa72ffdba5b440468f31309 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Feb 29 2016 15:32:37 +0000
Subject: [PATCH 2/3] Make the sleeping time configurable


---

diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md
index c4af3ce..dd37510 100644
--- a/mdapi-get_repo_md
+++ b/mdapi-get_repo_md
@@ -505,6 +505,7 @@ def main():
     #)
 
     # In serial
+    sleep_for = CONFIG.get('CRON_SLEEP', 30)
     for t in itertools.product(
             [CONFIG.get('DB_FOLDER', '/var/tmp')],
             repositories):
@@ -513,7 +514,7 @@ def main():
             process_repo(t)
         except OSError:
             # Most often due to an invalid stream, so let's try a second time
-            time.sleep(30)
+            time.sleep(sleep_for)
             process_repo(t)
 
     return 0

From 65ed2daf1769f51c60b920fda3c4a12255cab9a1 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Feb 29 2016 16:50:18 +0000
Subject: [PATCH 3/3] Let's loop a few time to retrieve the repo before bailing out


---

diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md
index dd37510..5acc443 100644
--- a/mdapi-get_repo_md
+++ b/mdapi-get_repo_md
@@ -510,12 +510,19 @@ def main():
             [CONFIG.get('DB_FOLDER', '/var/tmp')],
             repositories):
 
-        try:
-            process_repo(t)
-        except OSError:
-            # Most often due to an invalid stream, so let's try a second time
-            time.sleep(sleep_for)
-            process_repo(t)
+        loop = True
+        cnt = 0
+        while loop:
+            cnt += 1
+            try:
+                process_repo(t)
+                loop = False
+            except OSError:
+                if cnt == 4:
+                    raise
+                # Most often due to an invalid stream, so let's try a second time
+                time.sleep(sleep_for)
+                process_repo(t)
 
     return 0