From e319bdca4a9e61ddea9aaecc3453d7e741fc73b5 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Mar 07 2020 23:21:13 +0000
Subject: add po consolidation for TM


---

diff --git a/build.py b/build.py
index 2aea144..8b6989d 100755
--- a/build.py
+++ b/build.py
@@ -11,7 +11,7 @@ import subprocess
 import tempfile
 import yaml
 
-from shutil import copyfile
+from shutil import copyfile, copy2
 from translation_finder import discover
 
 def main():
@@ -30,6 +30,7 @@ def main():
 
     srpm_folder="./src.rpms/f{v}/".format(v=args.release)
     result_folder="./results/f{v}/".format(v=args.release)
+    tm_folder="./tm/f{v}/".format(v=args.release)
 
     pkgs = []
     for (dirpath, dirnames, filenames) in os.walk(srpm_folder):
@@ -43,7 +44,7 @@ def main():
             package = [x for x in pkgs if x == args.srpm][0]
             srpm_file = "{srpm}/{a}".format(srpm=srpm_folder, a=package)
             extract_srpm(tmp, srpm_file, result_folder)
-            discover_translations(tmp, package, result_folder)
+            discover_translations(tmp, package, result_folder, tm_folder)
     else:
         with tempfile.TemporaryDirectory() as tmp:
             if args.offset:
@@ -60,7 +61,7 @@ def main():
 
                 srpm_file = "{srpm}/{a}".format(srpm=srpm_folder, a=package)
                 extract_srpm(tmp, srpm_file, result_folder)
-                discover_translations(tmp, package, result_folder)
+                discover_translations(tmp, package, result_folder, tm_folder)
 
     subprocess.run(['./concat_csv.sh', result_folder],
                    check=True)
@@ -78,7 +79,7 @@ def extract_srpm(tmp, name, result_folder):
     out.close()
     error.close()
 
-def discover_translations(tmp, name, result_folder):
+def discover_translations(tmp, name, result_folder, tm_folder):
     """find po file"""
     print("discover_translations: "+tmp)
     translation_files = []
@@ -102,7 +103,7 @@ def discover_translations(tmp, name, result_folder):
         for translation in translation_files:
             # TODO: multiple translation files for same package gnome-clocks-3.32.0-1.fc30.src.rpm
             if translation["file_format"] == "po":
-                get_po_translation_level(tmp, translation, name, result_folder)
+                get_po_translation_level(tmp, translation, name, result_folder, tm_folder)
             elif translation["file_format"] == "ts":
                 get_ts_translation_level(tmp, translation, name, result_folder)
             elif translation["file_format"] == "json":
@@ -113,7 +114,7 @@ def discover_translations(tmp, name, result_folder):
             else:
                 unknown_format(tmp, translation, name, translation["file_format"], result_folder)
 
-def get_po_translation_level(path, mask, name, result_folder):
+def get_po_translation_level(path, mask, name, result_folder, tm_folder):
     filemask = mask["filemask"]
     print("get_po_translation_level: " + filemask)
 
@@ -121,12 +122,16 @@ def get_po_translation_level(path, mask, name, result_folder):
     error = open(result_folder + '/{p}.errors.txt'.format(p=name), 'a')
     subprocess.run(["pocount", filemask.split("*")[0], "--csv"],
                    stdout=stats, stderr=error, check=True, cwd=path)
-    print(filemask.split("*")[0])
-    print(os.listdir(path))
 
     stats.close()
     error.close()
 
+    # Copy translation files in translation memory
+    for po in glob.glob(path +"/"+ filemask):
+        dest = tm_folder +"/"+ name +"/"+ filemask.split("*")[0]
+        os.makedirs(dest, exist_ok=True)
+        copy2(po, dest)
+
     subprocess.run(["sed", "-i", "-e", "s|{p}|.|g".format(p=path),
                     result_folder + '/{p}.errors.txt'.format(p=name)], check=True)
 
diff --git a/build_tm.py b/build_tm.py
new file mode 100755
index 0000000..582cc54
--- /dev/null
+++ b/build_tm.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Consolidate each po files into compendium"""
+
+import argparse
+import glob
+import os
+import subprocess
+import tempfile
+
+def main():
+    """Handle params"""
+
+    parser = argparse.ArgumentParser(
+    description="Creates compendium for every languages")
+
+    parser.add_argument("--release", required=True, type=int, default=31,
+                        choices=[30, 31],
+                        help="Provide the Fedora release to analyze")
+
+    parser.add_argument("--lang", required=False, type=str,
+                        help="Filter a language to analyze")
+
+    args = parser.parse_args()
+
+    tm_folder="./tm/f{v}/".format(v=args.release)
+
+    pofiles = glob.glob(tm_folder+'*/*/*.po')
+    po_langs = list(set([ os.path.basename(po) for po in pofiles ]))
+    po_langs.sort()
+
+    if len(args.lang) > 0:
+        compute_lang(args.lang+".po", tm_folder)
+    else:
+        for langfile in po_langs:
+            compute_lang(langfile, tm_folder)
+
+def compute_lang(langfile, tm_folder):
+    """ Generate compendium and convert it to tmx"""
+    lang = os.path.splitext(langfile)[0]
+    print("Creating lang: " + lang)
+
+    pofiles = glob.glob(tm_folder+'*/*/'+langfile)
+
+    print(" - po consolidation")
+    compendium_file = tm_folder + lang + ".po"
+    command = ["pocompendium", compendium_file] + pofiles
+    subprocess.run(command, check=True)
+
+    print("  - po to tmx convertion")
+    tmx_file = tm_folder + lang + ".tmx"
+    command = ["po2tmx", "--language="+lang, "--progress=none",
+               compendium_file, "--output="+tmx_file]
+    subprocess.run(command, check=True)
+
+    print("  - language terminology")
+    terminology_file = tm_folder + lang + ".terminology.po"
+    command = ["poterminology", "--ignore-case", "--fold-titlecase",
+                "--inputs-needed", "1",
+                "--progress=verbose", compendium_file, "--output="+terminology_file]
+    subprocess.run(command, check=True)
+
+if __name__ == '__main__':
+    main()
+