From ba583ae00754083924e6c0ca759a51bc1a8242fb Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Mar 29 2021 20:03:52 +0000
Subject: [PATCH 1/2] rewrite of msgcat_recursive


---

diff --git a/build_tm.py b/build_tm.py
index 9a04e0c..238c2f0 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -5,6 +5,7 @@ import argparse
 import gzip
 import json
 import os
+import re
 import subprocess
 import shutil
 import tempfile
@@ -171,13 +172,12 @@ def process_compendium(langfiles, dest, debug_folder):
         if len(all_files) == 1:
             shutil.copyfile(os.path.join(tmp, all_files[0]), dest)
         else:
-            msgcat_recursive(dest, tmp, debug_folder, all_files, list(), list())
+            msgcat_loop(dest, tmp, debug_folder, all_files)
 
 
-def msgcat(files, destination, path, doubt=False):
+def msgcat(files, destination, path):
     """ Call the msgcat command on a list of po files
-    Only print output if a bug is suspected """
-    log = logging.getLogger("buildTm.msgcat")
+    Return stderr, if any """
     command = [
                   "msgcat",
                   "--force-po",
@@ -186,14 +186,14 @@ def msgcat(files, destination, path, doubt=False):
                   destination,
               ] + files
 
+    stderr = None
     try:
-        subprocess.run(command, check=True, cwd=path, capture_output=True)
+        p = subprocess.run(command, check=True, cwd=path, capture_output=True)
+        stderr = p.stderr.decode('utf8')
     except subprocess.CalledProcessError as e:
         # msgcat often raise exception but continues its processing
-        if doubt is not False:
-            log.error("Error with file {d}: {e}".format(d=doubt, e=e.stderr.decode('utf8')))
-        pass
-
+        stderr = e.stderr.decode('utf8')
+    return stderr
 
 def store_debug_file(path, name, file, debug_folder):
     """ Move the temporary move file in debug folder """
@@ -203,52 +203,30 @@ def store_debug_file(path, name, file, debug_folder):
     shutil.move(os.path.join(path, file), target)
 
 
-def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
-    """ Try to call msgcat, retry with half of the files if it fails """
-    log = logging.getLogger("buildTm.msgcat_recursive")
-    doubt = False
-    log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok)))
-    if len(ongoing) == 0:
-        ongoing = backlog.copy()
-        backlog = []
-
-    # we can't use msgcat with one single file
-    if len(ongoing) == 1:
-        doubt = ongoing.copy().pop()
-        ongoing.append(ok[0])
-
-    msgcat(ongoing, destination, path, doubt)
-
-    if os.path.isfile(destination) is True:
-        processed = len(ongoing)
-        ok += ongoing
-        # if we added one item from 'ok', we want to make sure it's not duplicated
-        ok = list(set(ok))
-        ongoing = []
-        if len(ok) == processed and len(backlog) == 0:
-            log.debug("First generation worked")
+def msgcat_loop(destination, path, debug_folder, files):
+    """ call msgcat, and exclude any problematic files """
+    log = logging.getLogger("buildTm.msgcat_loop")
+    log.debug("Starting msgcat loop")
+
+    while ret := msgcat(files, destination, path):
+        ids = re.findall(r"msgcat: Conversion of file (\d+) from", ret)
+        ids += re.findall(r"\d+:\d+: (\d+): input is not valid in", ret)
+        if ids:
+            file = ids[0]
+            log.debug("This file raised a msgcat bug: {f}".format(f=file))
+            store_debug_file(path, "tm-msgcat-" + destination.split("/")[-1], file, debug_folder)
+            files.remove(file)
         else:
-            os.remove(destination)
-    else:
-        if doubt is not False:
-            log.debug("This file raised a msgcat bug: {f}".format(f=doubt))
-            store_debug_file(path, "tm-msgcat-" + destination.split("/")[-1], doubt, debug_folder)
-            ongoing = []
-
-        half = int(len(ongoing) / 2)
-        backlog += ongoing[half:]
-        ongoing = ongoing[:half]
-
-    if len(backlog) + len(ongoing) > 0:
-        msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok)
-    else:
-        if os.path.isfile(destination) is False:
-            log.debug("Generating remaining files")
-            msgcat(ok, destination, path)
-
-        if os.path.isfile(destination) is False:
-            log.error("weird, some files raising bugs were missed?")
-            msgcat_recursive(destination, path, debug_folder, ok, list(), list())
+            # nothing found in stderr
+            if os.path.isfile(destination) is False:
+                # and destination not here : unhandled exception
+                # TODO: maybe actually throw an exception here?
+                log.error("Error with msgcat: {e}".format(e=ret))
+                return False
+            # no stderr and final file is here : all good
+            break
+        log.debug("next try")
+    log.debug("msgcat loop over")
 
 
 def process_tmx(lang, source, dest):

From 1811a896b4c6ffa4e09e706618391a202d787a4f Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Apr 05 2021 20:36:00 +0000
Subject: [PATCH 2/2] force LOCALE to C for build_tm


---

diff --git a/runall.sh b/runall.sh
index ecbe585..5951544 100755
--- a/runall.sh
+++ b/runall.sh
@@ -25,7 +25,7 @@ podman run -it --rm -v ./:/src:z -v ./results:/src/results:z -v ./srpms:/srpms:z
 
 # Creates useful translator files for every languages
 # ~ 3 h 00
-./build_tm.py --results "$results" --compress
+LANG=C ./build_tm.py --results "$results" --compress
 
 # Computes stats for each srpm detected
 # ~ 2 h 15