From e52117e74f106a1f6569bb111f595979953c02fc Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Nov 30 2020 22:49:13 +0000 Subject: [PATCH 1/4] remplace subprocess by direct calls to translate toolbox --- diff --git a/build_tm.py b/build_tm.py index 4dda1f1..d9385ac 100755 --- a/build_tm.py +++ b/build_tm.py @@ -6,14 +6,21 @@ import gzip import json import os import subprocess +import shutil import tempfile +from io import BytesIO +from translate.storage import tmx +from translate.convert import po2tmx +from translate.storage import factory, po +from translate.tools import poterminology + def main(): """Handle params""" parser = argparse.ArgumentParser( - description="Creates compendium for every languages") + description="Creates useful translator files for every languages") parser.add_argument("--results", required=True, help="Set the results folder to use") @@ -24,9 +31,6 @@ def main(): parser.add_argument("--compress", action="store_true", help="Compress output files") - parser.add_argument("--lang", required=False, type=str, - help="Filter a language to analyze") - args = parser.parse_args() results_folder = "./results/{v}/".format(v=args.results) @@ -34,84 +38,123 @@ def main(): tm_folder = os.path.join(results_folder, "languages-tm/") os.makedirs(tm_folder, exist_ok=True) + # clean destination folders + if args.refresh and os.path.isdir(tm_folder): + shutil.rmtree(tm_folder) + + if not os.path.exists(tm_folder): + os.makedirs(tm_folder) + print("Building the translation memory for every languages") - if args.lang: - with open(os.path.join(lang_path, args.lang + ".json"), "r") as read_file: + langs = [f for f in os.listdir(lang_path) if os.path.isfile(os.path.join(lang_path, f))] + + for lang in sorted(langs): + lang_code = lang[:-len('.json')] + with open(os.path.join(lang_path, lang), "r") as read_file: files = json.load(read_file)["po"] - compute_lang(args.lang, files, tm_folder) - else: - langs = [f for f in os.listdir(lang_path) if os.path.isfile(os.path.join(lang_path, f))] + compendium_file = os.path.join(tm_folder, lang_code + ".po") + compendium_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), compendium_file) + if not os.path.isfile(compendium_file): + process_compendium(lang_code, files, compendium_file) + + tmx_file = os.path.join(tm_folder, lang_code + ".tmx") + if not os.path.isfile(tmx_file): + process_tmx(lang_code, compendium_file, tmx_file) - for lang in sorted(langs): - with open(os.path.join(lang_path, lang), "r") as read_file: - files = json.load(read_file)["po"] + terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po") + if not os.path.isfile(terminology_file): + process_terminology(lang_code, compendium_file, terminology_file) - compute_lang(lang[:-len('.json')], files, tm_folder, args.refresh) - print("Detecting missing files") - for lang in sorted(langs): - check_lang(lang[:-len('.json')], tm_folder) + print("Detecting missing files") + for lang in sorted(langs): + check_lang(lang[:-len('.json')], tm_folder) if args.compress: print("Compressing files") compress(tm_folder) -def compute_lang(lang, langfiles, tm_folder, refresh): - """ Generate compendium and convert it to tmx """ - """ """ - print(" Computing: " + lang) +def process_compendium(lang, langfiles, dest): + """ Generate a compendium (a concatenation of all po files) """ - # po consolidation - compendium_file = os.path.join(tm_folder, lang + ".po") - compendium_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), compendium_file) + pofiles = [os.path.join(os.path.dirname(os.path.abspath(__file__)), f) for f in langfiles] + count = 0 - if not os.path.isfile(compendium_file) or refresh is True: - pofiles = [os.path.join(os.path.dirname(os.path.abspath(__file__)), f) for f in langfiles] - count = 0 - - with tempfile.TemporaryDirectory(prefix="l10n-tm") as tmp: - for i in pofiles: + with tempfile.TemporaryDirectory(prefix="l10n-tm") as tmp: + for i in pofiles: + try: + command = ["msguniq", i, "--output-file", count.__str__(), "--no-location"] + subprocess.run(command, check=True, cwd=tmp, capture_output=True) + except subprocess.CalledProcessError: try: - command = ["msguniq", i, "--output-file", count.__str__(), "--no-location"] + command = ["msguniq", i, "--output-file", count.__str__(), "--to-code", "utf-8", "--no-location"] subprocess.run(command, check=True, cwd=tmp, capture_output=True) - except subprocess.CalledProcessError: - try: - command = ["msguniq", i, "--output-file", count.__str__(), "--to-code", "utf-8", "--no-location"] - subprocess.run(command, check=True, cwd=tmp, capture_output=True) - except subprocess.CalledProcessError as e: - print("Error with msguniq {i}, error: {e}".format(i=i, e=e)) + except subprocess.CalledProcessError as e: + print("Error with msguniq {i}, error: {e}".format(i=i, e=e)) - count += 1 + count += 1 - onlyfiles = [f for f in os.listdir(tmp) if os.path.isfile(os.path.join(tmp, f))] - command = ["msgcat", "--force-po", "--no-location", "--output-file", compendium_file] + onlyfiles + onlyfiles = [f for f in os.listdir(tmp) if os.path.isfile(os.path.join(tmp, f))] + command = ["msgcat", "--force-po", "--no-location", "--output-file", dest] + onlyfiles - try: - subprocess.run(command, check=True, cwd=tmp, capture_output=True) - except subprocess.CalledProcessError: - print(" msgcat exception...") + try: + subprocess.run(command, check=True, cwd=tmp, capture_output=True) + except subprocess.CalledProcessError: + print(" msgcat exception...") - # po to tmx convertion - tmx_file = os.path.join(tm_folder, lang + ".tmx") - command = ["po2tmx", "--language="+lang, "--progress=none", - compendium_file, "--output="+tmx_file] - if not os.path.isfile(tmx_file) or refresh is True: - subprocess.run(command, check=True, capture_output=True) - # language terminology - terminology_file = os.path.join(tm_folder, lang + ".terminology.po") - command = ["poterminology", "--ignore-case", "--fold-titlecase", - "--inputs-needed", "1", - "--progress=none", compendium_file, "--output=" + terminology_file] - if not os.path.isfile(terminology_file) or refresh is True: - subprocess.run(command, check=True, capture_output=True) +def process_tmx(lang, source, dest): + """ Generate a translation memory from a po file """ + + outputfile = po2tmx.tmxmultifile(dest) + tmx_file_result = po2tmx.convertpo( + inputfile=BytesIO(open(source, "r").read().encode("utf-8")), + outputfile=outputfile, + templatefile=None, + sourcelanguage="en", + targetlanguage=lang, + comment="source" + ) + + outputfile.tmxfile.savefile(dest) + + +def process_terminology(lang, source, dest): + """ Generate a termonology from a po file """ + + extractor = poterminology.TerminologyExtractor() + options = { + "inputmin": "1", + "fullmsgmin": "1", + "substrmin": "2", + "locmin": "2", + "nonstopmin": 1, + "sortorders": ["frequency", "dictionary", "length"], + "output": dest, + } + + with open(source, "rb") as fh: + inputfile = factory.getobject(fh) + + extractor.processunits(inputfile.units, source) + terms = extractor.extract_terms() + + termfile = po.pofile() + termitems = extractor.filter_terms( + terms, nonstopmin=options["nonstopmin"], sortorders=options["sortorders"] + ) + for count, unit in termitems: + termfile.units.append(unit) + + with open(options["output"], "wb") as fh: + termfile.serialize(fh) def check_lang(lang, tm_folder): - """ make sure the files were generated """ + """ Check if expected files were generated """ compendium_file = os.path.join(tm_folder, lang + ".po") tmx_file = os.path.join(tm_folder, lang + ".tmx") From 3e3ec76fd8a259c65252e86080aeb1aa77b9fa9d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Nov 30 2020 23:13:36 +0000 Subject: [PATCH 2/4] add "error management" --- diff --git a/build_tm.py b/build_tm.py index d9385ac..12e1696 100755 --- a/build_tm.py +++ b/build_tm.py @@ -51,21 +51,33 @@ def main(): for lang in sorted(langs): lang_code = lang[:-len('.json')] + + print(" {l}".format(l=lang_code)) + with open(os.path.join(lang_path, lang), "r") as read_file: files = json.load(read_file)["po"] compendium_file = os.path.join(tm_folder, lang_code + ".po") compendium_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), compendium_file) if not os.path.isfile(compendium_file): - process_compendium(lang_code, files, compendium_file) + try: + process_compendium(lang_code, files, compendium_file) + except Exception as e: + print(" Compendium generation triggered an {t} exception: {e}".format(t=type(e).__name__, e=e)) tmx_file = os.path.join(tm_folder, lang_code + ".tmx") if not os.path.isfile(tmx_file): - process_tmx(lang_code, compendium_file, tmx_file) + try: + process_tmx(lang_code, compendium_file, tmx_file) + except Exception as e: + print(" TMX generation triggered an {t} exception: {e}".format(t=type(e).__name__, e=e)) terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po") if not os.path.isfile(terminology_file): - process_terminology(lang_code, compendium_file, terminology_file) + try: + process_terminology(lang_code, compendium_file, terminology_file) + except Exception as e: + print(" Terminology generation triggered an {t} exception: {e}".format(t=type(e).__name__, e=e)) print("Detecting missing files") @@ -111,7 +123,7 @@ def process_tmx(lang, source, dest): outputfile = po2tmx.tmxmultifile(dest) tmx_file_result = po2tmx.convertpo( - inputfile=BytesIO(open(source, "r").read().encode("utf-8")), + inputfile=BytesIO(open(source, "r").read().encode()), outputfile=outputfile, templatefile=None, sourcelanguage="en", From 1b7276efdea24eba2368da9c389ba3eaa5b80165 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Dec 21 2020 17:10:14 +0000 Subject: [PATCH 3/4] remove useless --- diff --git a/build_tm.py b/build_tm.py index 12e1696..e79c29e 100755 --- a/build_tm.py +++ b/build_tm.py @@ -10,7 +10,6 @@ import shutil import tempfile from io import BytesIO -from translate.storage import tmx from translate.convert import po2tmx from translate.storage import factory, po from translate.tools import poterminology @@ -61,7 +60,7 @@ def main(): compendium_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), compendium_file) if not os.path.isfile(compendium_file): try: - process_compendium(lang_code, files, compendium_file) + process_compendium(files, compendium_file) except Exception as e: print(" Compendium generation triggered an {t} exception: {e}".format(t=type(e).__name__, e=e)) @@ -75,11 +74,10 @@ def main(): terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po") if not os.path.isfile(terminology_file): try: - process_terminology(lang_code, compendium_file, terminology_file) + process_terminology(compendium_file, terminology_file) except Exception as e: print(" Terminology generation triggered an {t} exception: {e}".format(t=type(e).__name__, e=e)) - print("Detecting missing files") for lang in sorted(langs): check_lang(lang[:-len('.json')], tm_folder) @@ -89,7 +87,7 @@ def main(): compress(tm_folder) -def process_compendium(lang, langfiles, dest): +def process_compendium(langfiles, dest): """ Generate a compendium (a concatenation of all po files) """ pofiles = [os.path.join(os.path.dirname(os.path.abspath(__file__)), f) for f in langfiles] @@ -122,7 +120,7 @@ def process_tmx(lang, source, dest): """ Generate a translation memory from a po file """ outputfile = po2tmx.tmxmultifile(dest) - tmx_file_result = po2tmx.convertpo( + po2tmx.convertpo( inputfile=BytesIO(open(source, "r").read().encode()), outputfile=outputfile, templatefile=None, @@ -134,7 +132,7 @@ def process_tmx(lang, source, dest): outputfile.tmxfile.savefile(dest) -def process_terminology(lang, source, dest): +def process_terminology(source, dest): """ Generate a termonology from a po file """ extractor = poterminology.TerminologyExtractor() From 0e77fb12f6a54eb14134b4dbdd3e59618ec16053 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Dec 22 2020 06:16:37 +0000 Subject: [PATCH 4/4] add translate-toolkit requirement --- diff --git a/requirements.txt b/requirements.txt index 7783663..6f584b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pandas polib weblate-language-data langtable +translate-toolkit \ No newline at end of file