From aa3c84262ce0dd59d7e10c97e479201729bdbae2 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 1/10] update Dockerfile for openshift use --- diff --git a/docker/Dockerfile.33 b/docker/Dockerfile.33 index b2c8d10..5736959 100644 --- a/docker/Dockerfile.33 +++ b/docker/Dockerfile.33 @@ -1,13 +1,18 @@ -FROM registry.fedoraproject.org/fedora:33 - -RUN dnf install -y lbzip2 unzip xz git cpio translate-toolkit dnf-plugins-core python3-pip rsync vim +FROM registry.fedoraproject.org/fedora:33 as builder +RUN dnf install -y lbzip2 unzip xz cpio dnf-plugins-core rsync python3-pip hugo gettext git COPY requirements.txt /src/requirements.txt + RUN pip install --no-cache -r /src/requirements.txt RUN pip install --upgrade https://github.com/WeblateOrg/language-data/archive/master.zip RUN pip install charamel RUN pip install git+https://github.com/WeblateOrg/translation-finder.git - -VOLUME /src -VOLUME /srpms +RUN mkdir -p /src/results /srpms; chmod g+rwX /srpms; chmod -R 1777 /tmp WORKDIR /src + +COPY *.py *.sh /src/ +COPY website /src/website/ +RUN chmod -R g+rwX /src/website +COPY templates /src/templates/ +VOLUME /src/results +ENV VERS f33 diff --git a/docker/Dockerfile.latest b/docker/Dockerfile.latest new file mode 100644 index 0000000..9f73dec --- /dev/null +++ b/docker/Dockerfile.latest @@ -0,0 +1,18 @@ +FROM registry.fedoraproject.org/fedora:34 as builder + +RUN dnf install -y lbzip2 unzip xz cpio dnf-plugins-core rsync python3-pip hugo gettext git +COPY requirements.txt /src/requirements.txt + +RUN pip install --no-cache -r /src/requirements.txt +RUN pip install --upgrade https://github.com/WeblateOrg/language-data/archive/master.zip +RUN pip install charamel +RUN pip install git+https://github.com/WeblateOrg/translation-finder.git +RUN mkdir -p /src/results /srpms; chmod g+rwX /srpms; chmod -R 1777 /tmp +WORKDIR /src + +COPY *.py *.sh /src/ +COPY website /src/website/ +RUN chmod -R g+rwX /src/website +COPY templates /src/templates/ +VOLUME /src/results +ENV VERS f34 From aeb576ad0fa7c0a362f79e731f99741d45bc979b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 2/10] recursive search of buggy files it saves quite a lot of computation time --- diff --git a/build_tm.py b/build_tm.py index 4b4eaaa..c672c7f 100755 --- a/build_tm.py +++ b/build_tm.py @@ -69,7 +69,7 @@ def main(): for lang in sorted(langs): lang_code = lang[: -len(".json")] - log.info(" {l}".format(l=lang_code)) + log.info("Processing {l}".format(l=lang_code)) with open(os.path.join(lang_path, lang), "r") as read_file: files = json.load(read_file)["po"] @@ -79,14 +79,7 @@ def main(): os.path.dirname(os.path.abspath(__file__)), compendium_file ) if not os.path.isfile(compendium_file): - try: - process_compendium(files, compendium_file, debug_folder) - except Exception as e: - log.error( - " Compendium generation triggered an {t} exception: {e}".format( - t=type(e).__name__, e=e - ) - ) + process_compendium(files, compendium_file, debug_folder) tmx_file = os.path.join(tm_folder, lang_code + ".tmx") if not os.path.isfile(tmx_file): @@ -110,6 +103,8 @@ def main(): ) ) + log.info("All languages are processed".format(l=lang_code)) + log.info("Detecting missing files") for lang in sorted(langs): check_lang(lang[: -len(".json")], tm_folder) @@ -158,30 +153,15 @@ def process_compendium(langfiles, dest, debug_folder): count += 1 - # search every file that were successful - search_guilty_file(tmp, dest, debug_folder) + all_files = [f for f in os.listdir(tmp) if os.path.isfile(os.path.join(tmp, f))] + if len(all_files) == 1: + shutil.copyfile(os.path.join(tmp, all_files[0]), dest) + else: + msgcat_recursive(dest, tmp, debug_folder, all_files, list(), list()) -def search_guilty_file(path, dest, debug_folder): - log = logging.getLogger("buildTm.process_compendium.guilty") - all_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] - - try_msgcat(all_files, dest, path) - - guilty_file = None - while os.path.isfile(dest) is False: - guilty_file = all_files.pop() - try_msgcat(all_files, dest, path) - - if guilty_file is not None: - debug_filename = "tm-msgcat-{lang}-{name}".format(lang=dest.split("/")[-1], name=guilty_file) - log.error("the file {f} raised error with msgcat, a copy of this file is into {d} as {n}".format(f=guilty_file, d=debug_folder, n=debug_filename)) - shutil.move(os.path.join(path, guilty_file), os.path.join(debug_folder, debug_filename)) - os.remove(dest) - search_guilty_file(path, dest, debug_folder) - - -def try_msgcat(files, destination, cwd): +def msgcat(files, destination, path, doubt=False): + log = logging.getLogger("buildTm.msgcat") command = [ "msgcat", "--force-po", @@ -191,12 +171,66 @@ def try_msgcat(files, destination, cwd): ] + files try: - subprocess.run(command, check=True, cwd=cwd, capture_output=True) - except subprocess.CalledProcessError: + subprocess.run(command, check=True, cwd=path, capture_output=True) + except subprocess.CalledProcessError as e: # msgcat often raise exception but continues its processing + if doubt is not False: + log.error("Error with file {d}: {e}".format(d=doubt, e=e.stderr.decode('utf8'))) pass +def store_debug_file(path, name, file, debug_folder): + log = logging.getLogger("buildTm.store_debug_file") + debug_filename = "{n}-{f}".format(n=name, f=file) + log.error("A copy of the file {f} is into {d} as {n}".format(f=file, + d=debug_folder, + n=debug_filename)) + shutil.move(os.path.join(path, file), os.path.join(debug_folder, debug_filename)) + + +def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok=[]): + log = logging.getLogger("buildTm.search_msgcat_buggy_files") + doubt = False + log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok))) + if len(ongoing) == 0: + ongoing = backlog.copy() + backlog = [] + + # we can't use msgcat with one single file + if len(ongoing) == 1: + doubt = ongoing.copy().pop() + ongoing.append(ok[0]) + + msgcat(ongoing, destination, path, doubt) + + if os.path.isfile(destination) is True: + processed = len(ongoing) + ok += ongoing + # if we added one item from 'ok', we want to make sure it's not duplicated + ok = list(set(ok)) + ongoing = [] + if len(ok) == processed and len(backlog) == 0: + log.debug("First generation worked") + else: + os.remove(destination) + else: + if doubt is not False: + log.debug("This file raised a msgcat bug: {f}".format(f=doubt)) + store_debug_file(path, "tm-msgcat-"+destination.split("/")[-1], doubt, debug_folder) + ongoing = [] + + half = int(len(ongoing) / 2) + backlog += ongoing[half:] + ongoing = ongoing[:half] + + if len(backlog) + len(ongoing) > 0: + msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok) + else: + if os.path.isfile(destination) is False: + log.debug("Generating remaining files") + msgcat(ok, destination, path) + + def process_tmx(lang, source, dest): """ Generate a translation memory from a po file """ @@ -253,13 +287,13 @@ def check_lang(lang, tm_folder): terminology_file = os.path.join(tm_folder, lang + ".terminology.po") if not os.path.isfile(compendium_file): - log.warning(" {l}-compendium is missing".format(l=lang)) + log.warning("{l}-compendium is missing".format(l=lang)) if not os.path.isfile(tmx_file): - log.warning(" {l}-tmx is missing".format(l=lang)) + log.warning("{l}-tmx is missing".format(l=lang)) if not os.path.isfile(terminology_file): - log.warning(" {l}-terminology is missing".format(l=lang)) + log.warning("{l}-terminology is missing".format(l=lang)) def compress(folder): From 131411a84f35b99b3081db6474d4206633913559 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 3/10] add last check, risk of infinite loop here... --- diff --git a/build_tm.py b/build_tm.py index c672c7f..06ea08a 100755 --- a/build_tm.py +++ b/build_tm.py @@ -41,7 +41,7 @@ def main(): loglevel = logging.INFO if args.verbose: - loglevel = logging.DEBUG + loglevel = logging.DEBUG logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel) log = logging.getLogger("buildTm") @@ -103,7 +103,7 @@ def main(): ) ) - log.info("All languages are processed".format(l=lang_code)) + log.info("All languages are processed") log.info("Detecting missing files") for lang in sorted(langs): @@ -161,6 +161,8 @@ def process_compendium(langfiles, dest, debug_folder): def msgcat(files, destination, path, doubt=False): + """ Call the msgcat command on a list of po files + Only print output if a bug is suspected """ log = logging.getLogger("buildTm.msgcat") command = [ "msgcat", @@ -180,15 +182,15 @@ def msgcat(files, destination, path, doubt=False): def store_debug_file(path, name, file, debug_folder): + """ Move the temporary move file in debug folder """ log = logging.getLogger("buildTm.store_debug_file") - debug_filename = "{n}-{f}".format(n=name, f=file) - log.error("A copy of the file {f} is into {d} as {n}".format(f=file, - d=debug_folder, - n=debug_filename)) - shutil.move(os.path.join(path, file), os.path.join(debug_folder, debug_filename)) + target = os.path.join(debug_folder, "{n}-{f}".format(n=name, f=file)) + log.error("The file {f} were moved into {t}".format(f=file, t=target)) + shutil.move(os.path.join(path, file), target) -def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok=[]): +def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok): + """ Try to call msgcat, retry with half of the files if it fails """ log = logging.getLogger("buildTm.search_msgcat_buggy_files") doubt = False log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok))) @@ -230,6 +232,10 @@ def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok log.debug("Generating remaining files") msgcat(ok, destination, path) + if os.path.isfile(destination) is False: + log.error("weird, some files raising bugs were missed?") + msgcat_recursive(destination, path, debug_folder, ok, list(), list()) + def process_tmx(lang, source, dest): """ Generate a translation memory from a po file """ From 52528678ee1ca864be76fd7c6d496aa32a041a32 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 4/10] fix logger name --- diff --git a/build_tm.py b/build_tm.py index 06ea08a..3b42928 100755 --- a/build_tm.py +++ b/build_tm.py @@ -191,7 +191,7 @@ def store_debug_file(path, name, file, debug_folder): def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok): """ Try to call msgcat, retry with half of the files if it fails """ - log = logging.getLogger("buildTm.search_msgcat_buggy_files") + log = logging.getLogger("buildTm.msgcat_recursive") doubt = False log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok))) if len(ongoing) == 0: From 2e1678a539f6aaa79d9d70e418d1d06cb6fb0ff5 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 5/10] fix tm and terminology generation --- diff --git a/build_tm.py b/build_tm.py index 3b42928..3c7dfe9 100755 --- a/build_tm.py +++ b/build_tm.py @@ -124,6 +124,7 @@ def process_compendium(langfiles, dest, debug_folder): count = 0 with tempfile.TemporaryDirectory(prefix="l10n-tm") as tmp: + for i in pofiles: try: command = [ @@ -240,6 +241,7 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok): def process_tmx(lang, source, dest): """ Generate a translation memory from a po file """ + """ outputfile = po2tmx.tmxmultifile(dest) po2tmx.convertpo( inputfile=BytesIO(open(source, "r").read().encode()), @@ -251,11 +253,16 @@ def process_tmx(lang, source, dest): ) outputfile.tmxfile.savefile(dest) + """ + command = ["po2tmx", "--language=" + lang, "--progress=none", source, "--output=" + dest] + subprocess.run(command, check=True, capture_output=True) def process_terminology(source, dest): """ Generate a termonology from a po file """ + + """ extractor = poterminology.TerminologyExtractor() options = { "inputmin": "1", @@ -282,6 +289,12 @@ def process_terminology(source, dest): with open(options["output"], "wb") as fh: termfile.serialize(fh) + + """ + command = ["poterminology", "--ignore-case", "--fold-titlecase", + "--inputs-needed", "1", + "--progress=none", source, "--output=" + dest] + subprocess.run(command, check=True, capture_output=True) def check_lang(lang, tm_folder): From d8f8e79dbd5475d8a80049073225a72a0b43a129 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 6/10] website: simple display of language list --- diff --git a/build_website.py b/build_website.py index 464928a..e2d0af3 100755 --- a/build_website.py +++ b/build_website.py @@ -166,6 +166,13 @@ def main(): generate_static_pages_packages(args.results, code, content, dest_file) + log.info("Generating indexes") + dest_file = os.path.join(static_langs_folder, "_index.md") + generate_language_index(args.results, dest_file) + + dest_file = os.path.join(static_pkgs_folder, "_index.md") + generate_package_index(args.results, dest_file) + log.info("Copy translation memories") languages = [ f for f in os.listdir(tm_folder) if os.path.isfile(os.path.join(tm_folder, f)) @@ -383,6 +390,38 @@ def generate_static_pages_packages(results, code, content, dest_file): write_out.write(outputText) +def generate_language_index(distribution, dest_file): + log = logging.getLogger("buildWebsite.generate_language_index") + data = dict() + data["distribution"] = distribution + data["now"] = datetime.datetime.utcnow() + + templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") + templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) + TEMPLATE_FILE = "_index.language.md" + template = templateEnv.get_template(TEMPLATE_FILE) + outputText = template.render(data) + + with open(dest_file, "w") as write_out: + write_out.write(outputText) + + +def generate_package_index(distribution, dest_file): + log = logging.getLogger("buildWebsite.generate_package_index") + data = dict() + data["distribution"] = distribution + data["now"] = datetime.datetime.utcnow() + + templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") + templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) + TEMPLATE_FILE = "_index.package.md" + template = templateEnv.get_template(TEMPLATE_FILE) + outputText = template.render(data) + + with open(dest_file, "w") as write_out: + write_out.write(outputText) + + def store_json_file(content, dest_file): with open(dest_file, "w") as f: f.write(json.dumps(content, indent=2)) diff --git a/templates/_index.language.md b/templates/_index.language.md new file mode 100644 index 0000000..bb7141a --- /dev/null +++ b/templates/_index.language.md @@ -0,0 +1,5 @@ +--- +title: "Languages for {{ distribution }}" +date: {{ now }} +layout: "list_languages" +--- \ No newline at end of file diff --git a/templates/_index.package.md b/templates/_index.package.md new file mode 100644 index 0000000..a973b26 --- /dev/null +++ b/templates/_index.package.md @@ -0,0 +1,4 @@ +--- +title: "Packages for {{ distribution }}" +date: {{ now }} +--- \ No newline at end of file diff --git a/templates/language.md b/templates/language.md index 9ac40af..622d9d8 100644 --- a/templates/language.md +++ b/templates/language.md @@ -1,6 +1,10 @@ --- -title: "{{ lang_name_en }} ({{ lang_name_local }})" +title: "{{ lang_code }}-{{ lang_name_en }} ({{ lang_name_local }})" date: {{ now }} +code: {{ lang_code }} +name_english: {{ lang_name_en }} +name_local: {{ lang_name_local }} +progress_d: {{ progress_d }} --- Language progress for {{ lang_name_en }} ({{ lang_code }}) in Fedora {{ results }} is: diff --git a/website/config.toml b/website/config.toml index c777f70..9beb5cf 100644 --- a/website/config.toml +++ b/website/config.toml @@ -1,5 +1,8 @@ baseURL = "https://jibecfed.fedorapeople.org/partage/fedora-localization-statistics/" languageCode = "en-us" -title = "Temporary demo" +title = "Fedora localization statistics" theme = "beautifulhugo" staticDir = "static" + +[markup.goldmark.renderer] +unsafe= true \ No newline at end of file diff --git a/website/themes/beautifulhugo/layouts/_default/f33.html b/website/themes/beautifulhugo/layouts/_default/f33.html new file mode 100644 index 0000000..1a00bdc --- /dev/null +++ b/website/themes/beautifulhugo/layouts/_default/f33.html @@ -0,0 +1,12 @@ +{{ define "main" }} +
+
+ {{ .Content }} + +
+
+{{ end }} diff --git a/website/themes/beautifulhugo/layouts/_default/list.html b/website/themes/beautifulhugo/layouts/_default/list.html index e08da36..5f7bf09 100644 --- a/website/themes/beautifulhugo/layouts/_default/list.html +++ b/website/themes/beautifulhugo/layouts/_default/list.html @@ -7,11 +7,6 @@ {{.}} {{ end }} -
- {{ range .Paginator.Pages }} - {{ partial "post_preview.html" .}} - {{ end }} -
{{ if or (.Paginator.HasPrev) (.Paginator.HasNext) }}
    {{ if .Paginator.HasPrev }} diff --git a/website/themes/beautifulhugo/layouts/_default/list_languages.html b/website/themes/beautifulhugo/layouts/_default/list_languages.html new file mode 100644 index 0000000..42b9620 --- /dev/null +++ b/website/themes/beautifulhugo/layouts/_default/list_languages.html @@ -0,0 +1,24 @@ +{{ define "main" }} +
    +
    + {{ .Content }} + + + + + + + + + {{ range sort .Pages "Title" "asc" }} + + + + + + + {{ end }} +
    codeEnglish nameLocal nameProgress
    {{ .Params.code }}{{ .Params.name_english }}{{ .Params.name_local }}{{ .Params.progress_d }}
    +
    +
    +{{ end }} \ No newline at end of file diff --git a/website/themes/beautifulhugo/layouts/index.html b/website/themes/beautifulhugo/layouts/index.html index cc15000..bfd14e7 100644 --- a/website/themes/beautifulhugo/layouts/index.html +++ b/website/themes/beautifulhugo/layouts/index.html @@ -7,28 +7,11 @@ {{.}} {{ end }} - -
    - {{ $pag := .Paginate (where site.RegularPages "Type" "in" site.Params.mainSections) }} - {{ range $pag.Pages }} - {{ partial "post_preview" . }} +
    - - {{ if or (.Paginator.HasPrev) (.Paginator.HasNext) }} - - {{ end }} +
diff --git a/website/themes/beautifulhugo/layouts/partials/breadcrumb.html b/website/themes/beautifulhugo/layouts/partials/breadcrumb.html new file mode 100644 index 0000000..62f1b66 --- /dev/null +++ b/website/themes/beautifulhugo/layouts/partials/breadcrumb.html @@ -0,0 +1,13 @@ +
    + {{ template "breadcrumbnav" (dict "p1" . "p2" .) }} +
+{{ define "breadcrumbnav" }} +{{ if .p1.Parent }} +{{ template "breadcrumbnav" (dict "p1" .p1.Parent "p2" .p2 ) }} +{{ else if not .p1.IsHome }} +{{ template "breadcrumbnav" (dict "p1" .p1.Site.Home "p2" .p2 ) }} +{{ end }} + + {{ .p1.Title }} + +{{ end }} \ No newline at end of file From f46f7bb879c9efd9c113eadc95351a5d075c53d9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 7/10] website: allow to sort language table --- diff --git a/website/themes/beautifulhugo/layouts/_default/list_languages.html b/website/themes/beautifulhugo/layouts/_default/list_languages.html index 42b9620..d108de7 100644 --- a/website/themes/beautifulhugo/layouts/_default/list_languages.html +++ b/website/themes/beautifulhugo/layouts/_default/list_languages.html @@ -3,12 +3,13 @@
{{ .Content }} - +
+ - - - - + + + + {{ range sort .Pages "Title" "asc" }} @@ -21,4 +22,60 @@
Click on columns headers to sort values
codeEnglish nameLocal nameProgresscodeEnglish nameLocal nameProgress
+ {{ end }} \ No newline at end of file From bee3225736f151417a8cbafa9fbe7ec8653dec68 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 8/10] prevent error by using sed, and catch new polib errors --- diff --git a/.gitignore b/.gitignore index ab40337..bba57c8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ website/content/* website/public/ website/static/* srpms_*.lst +log.* diff --git a/build_language_list.py b/build_language_list.py index b8402bf..6eb4499 100755 --- a/build_language_list.py +++ b/build_language_list.py @@ -120,12 +120,14 @@ def analyze_lang(lang_folder, analized_lang): metadata = dict() try: metadata = polib.pofile(file).metadata - except UnicodeDecodeError: - # encoding error, to investigate before using it in TM - metadata["Language"] = "error-unicode" except OSError: # maybe a polib bug? to investigate before using it in TM metadata["Language"] = "error-os" + except TypeError: + metadata["Language"] = "error-type" + except UnicodeDecodeError: + # encoding error, to investigate before using it in TM + metadata["Language"] = "error-unicode" if "Language" not in metadata.keys(): metadata["Language"] = "zzz_null" @@ -154,9 +156,7 @@ def analyze_lang(lang_folder, analized_lang): results[metadata.get("Language")] = language - results = dict(sorted(results.items(), key=lambda item: item[0])) - - return results + return dict(sorted(results.items(), key=lambda item: item[0])) def describe(lang_folder): @@ -189,8 +189,11 @@ def detect_languages(package_folder, results_folder): log_file = os.path.join(results_folder, "build_language_list.log") file_object = open(log_file, "w") - + count = 0 + total = len(packages) for package in packages: + count += 1 + log.debug("{c}/{t}".format(c=count, t=total)) discovery_file = os.path.join(package_folder, package, "discover.json") with open(discovery_file, "r") as read_file: @@ -212,13 +215,15 @@ def detect_languages(package_folder, results_folder): except UnicodeDecodeError: # encoding error, to investigate before using it in TM error = "error-unicode" + except TypeError: + error = "error-type" except OSError: # maybe a polib bug? to investigate before using it in TM error = "error-os" lang, decision = choose_lang(lang_code, metadata, error) - log = ",".join( + debug = ",".join( [ po, lang_code, @@ -228,7 +233,7 @@ def detect_languages(package_folder, results_folder): str(decision), ] ) - file_object.write(log + "\n") + file_object.write(debug + "\n") lang_result = langs.get(lang, dict()) po_results = lang_result.get("po", list()) diff --git a/build_stats.py b/build_stats.py index fe0970f..02dba08 100755 --- a/build_stats.py +++ b/build_stats.py @@ -6,6 +6,8 @@ import glob import json import os import shutil +import subprocess + import polib import logging @@ -71,30 +73,31 @@ def main(): for package in sorted(packages): count += 1 log.info(" {c}/{t} - {p}".format(c=count, t=len(packages), p=package)) - with open(os.path.join(packages_folder, package, "discover.json"), "r") as f: - discoveries = json.load(f) src_folder = os.path.join(packages_folder, package) stats_file = os.path.join(packages_stats_folder, package + ".json") - if os.path.isfile(stats_file): - continue + if os.path.isfile(stats_file) is False: + with open(os.path.join(packages_folder, package, "discover.json"), "r") as f: + discoveries = json.load(f) - results = dict() - for discover in discoveries: - files = glob.glob(os.path.join(src_folder, discover["filemask"])) + results = dict() + for discover in discoveries: + files = glob.glob(os.path.join(src_folder, discover["filemask"])) - if discover["file_format"] == "po": - results[discover["filemask"]] = get_po_translation_level( - files, stats_file - ) + if discover["file_format"] == "po": + results[discover["filemask"]] = get_po_translation_level( + files, stats_file + ) - if len(results) > 0: - distribution_stats = extract_release_stats(distribution_stats, results) + if len(results) > 0: + with open(stats_file, "w") as f: + json.dump(results, f, indent=2) + else: + with open(stats_file, "r") as f: + results = json.load(f) - if len(results) > 0: - with open(stats_file, "w") as f: - json.dump(results, f, indent=2) + distribution_stats = extract_release_stats(distribution_stats, results) log.info("Storing distribution stats") if not os.path.exists(distribution_stats_folder): @@ -137,6 +140,11 @@ def get_po_translation_level(files, stats_file): stats = dict() for file in files: + # remove non standard comments + # taken from: https://github.com/translate/translate/blob/master/tools/pocommentclean + command = ["sed", "-i", "/^#$/d;/^#[^\:\~,\.]/d", file] + subprocess.run(command, check=True, capture_output=True) + try: stat = calcstats(file) except Exception as e: @@ -168,12 +176,15 @@ def get_language_team(file): metadata = dict() try: metadata = polib.pofile(file).metadata - except UnicodeDecodeError: - # encoding error, to investigate before using it in TM - metadata["Language"] = "error-unicode" except OSError: # maybe a polib bug? to investigate before using it in TM metadata["Language"] = "error-os" + except UnicodeDecodeError: + # encoding error, to investigate before using it in TM + metadata["Language"] = "error-unicode" + except TypeError: + # TypeError: '>' not supported between instances of 'str' and 'int' + metadata["Language"] = "error-valuerror" team = "Unknown..." try: diff --git a/build_tm.py b/build_tm.py index 3c7dfe9..4e7b280 100755 --- a/build_tm.py +++ b/build_tm.py @@ -10,12 +10,6 @@ import shutil import tempfile import logging -from io import BytesIO -from translate.convert import po2tmx -from translate.storage import factory, po -from translate.tools import poterminology - - def main(): """Handle params""" @@ -79,10 +73,12 @@ def main(): os.path.dirname(os.path.abspath(__file__)), compendium_file ) if not os.path.isfile(compendium_file): + log.info("Compendium generation") process_compendium(files, compendium_file, debug_folder) tmx_file = os.path.join(tm_folder, lang_code + ".tmx") if not os.path.isfile(tmx_file): + log.info("TMX generation") try: process_tmx(lang_code, compendium_file, tmx_file) except Exception as e: @@ -94,6 +90,7 @@ def main(): terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po") if not os.path.isfile(terminology_file): + log.info("Terminology generation") try: process_terminology(compendium_file, terminology_file) except Exception as e: From 580d7b0fc43361a61ec497efecd0e882065feec3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:27:40 +0000 Subject: [PATCH 9/10] remove comments from compendium files --- diff --git a/build_tm.py b/build_tm.py index 4e7b280..666665f 100755 --- a/build_tm.py +++ b/build_tm.py @@ -75,6 +75,10 @@ def main(): if not os.path.isfile(compendium_file): log.info("Compendium generation") process_compendium(files, compendium_file, debug_folder) + # remove non standard comments + # taken from: https://github.com/translate/translate/blob/master/tools/pocommentclean + command = ["sed", "-i", "/^#$/d;/^#[^\:\~,\.]/d", compendium_file] + subprocess.run(command, check=True, capture_output=True) tmx_file = os.path.join(tm_folder, lang_code + ".tmx") if not os.path.isfile(tmx_file): From b114f2f5d10b96dd83bca9f50039e8b22e30b2af Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 25 2021 20:55:24 +0000 Subject: [PATCH 10/10] reduce space consumption --- diff --git a/build_tm.py b/build_tm.py index 666665f..d265757 100755 --- a/build_tm.py +++ b/build_tm.py @@ -10,6 +10,7 @@ import shutil import tempfile import logging + def main(): """Handle params""" @@ -72,7 +73,8 @@ def main(): compendium_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), compendium_file ) - if not os.path.isfile(compendium_file): + compendium_archive = compendium_file + ".gz" + if os.path.isfile(compendium_file) is False and os.path.isfile(compendium_archive) is False: log.info("Compendium generation") process_compendium(files, compendium_file, debug_folder) # remove non standard comments @@ -81,7 +83,8 @@ def main(): subprocess.run(command, check=True, capture_output=True) tmx_file = os.path.join(tm_folder, lang_code + ".tmx") - if not os.path.isfile(tmx_file): + tmx_archive = tmx_file + ".gz" + if os.path.isfile(tmx_file) is False and os.path.isfile(tmx_archive) is False: log.info("TMX generation") try: process_tmx(lang_code, compendium_file, tmx_file) @@ -93,7 +96,8 @@ def main(): ) terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po") - if not os.path.isfile(terminology_file): + terminology_archive = terminology_file + ".gz" + if os.path.isfile(terminology_file) is False and os.path.isfile(terminology_archive) is False: log.info("Terminology generation") try: process_terminology(compendium_file, terminology_file) @@ -104,15 +108,21 @@ def main(): ) ) + if args.compress: + if os.path.isfile(compendium_file): + compress(compendium_file, compendium_archive) + + if os.path.isfile(tmx_file): + compress(tmx_file, tmx_archive) + + if os.path.isfile(terminology_file): + compress(terminology_file, terminology_archive) + log.info("All languages are processed") log.info("Detecting missing files") for lang in sorted(langs): - check_lang(lang[: -len(".json")], tm_folder) - - if args.compress: - log.info("Compressing files") - compress(tm_folder) + check_lang(lang[: -len(".json")], tm_folder, args.compress) def process_compendium(langfiles, dest, debug_folder): @@ -150,7 +160,9 @@ def process_compendium(langfiles, dest, debug_folder): subprocess.run(command, check=True, cwd=tmp, capture_output=True) except subprocess.CalledProcessError as e: debug_filename = "tm-msguniq-{lang}-{name}".format(lang=dest.split("/")[-1], name=count.__str__()) - log.error(" msguniq error with {i} a copy of this file is into {d} as {n}".format(i=i, e=e.output, d=debug_folder, n=debug_filename)) + log.error(" msguniq error with {i} a copy of this file is into {d} as {n}".format(i=i, e=e.output, + d=debug_folder, + n=debug_filename)) shutil.copyfile(i, os.path.join(debug_folder, debug_filename)) count += 1 @@ -220,7 +232,7 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok): else: if doubt is not False: log.debug("This file raised a msgcat bug: {f}".format(f=doubt)) - store_debug_file(path, "tm-msgcat-"+destination.split("/")[-1], doubt, debug_folder) + store_debug_file(path, "tm-msgcat-" + destination.split("/")[-1], doubt, debug_folder) ongoing = [] half = int(len(ongoing) / 2) @@ -242,63 +254,20 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok): def process_tmx(lang, source, dest): """ Generate a translation memory from a po file """ - """ - outputfile = po2tmx.tmxmultifile(dest) - po2tmx.convertpo( - inputfile=BytesIO(open(source, "r").read().encode()), - outputfile=outputfile, - templatefile=None, - sourcelanguage="en", - targetlanguage=lang, - comment="source", - ) - - outputfile.tmxfile.savefile(dest) - """ - command = ["po2tmx", "--language=" + lang, "--progress=none", source, "--output=" + dest] subprocess.run(command, check=True, capture_output=True) + def process_terminology(source, dest): """ Generate a termonology from a po file """ - - """ - extractor = poterminology.TerminologyExtractor() - options = { - "inputmin": "1", - "fullmsgmin": "1", - "substrmin": "2", - "locmin": "2", - "nonstopmin": 1, - "sortorders": ["frequency", "dictionary", "length"], - "output": dest, - } - - with open(source, "rb") as fh: - inputfile = factory.getobject(fh) - - extractor.processunits(inputfile.units, source) - terms = extractor.extract_terms() - - termfile = po.pofile() - termitems = extractor.filter_terms( - terms, nonstopmin=options["nonstopmin"], sortorders=options["sortorders"] - ) - for count, unit in termitems: - termfile.units.append(unit) - - with open(options["output"], "wb") as fh: - termfile.serialize(fh) - - """ command = ["poterminology", "--ignore-case", "--fold-titlecase", - "--inputs-needed", "1", - "--progress=none", source, "--output=" + dest] + "--inputs-needed", "1", + "--progress=none", source, "--output=" + dest] subprocess.run(command, check=True, capture_output=True) -def check_lang(lang, tm_folder): +def check_lang(lang, tm_folder, compress): """ Check if expected files were generated """ log = logging.getLogger("buildTm.check_lang") @@ -306,33 +275,31 @@ def check_lang(lang, tm_folder): tmx_file = os.path.join(tm_folder, lang + ".tmx") terminology_file = os.path.join(tm_folder, lang + ".terminology.po") - if not os.path.isfile(compendium_file): + if compress is True: + compendium_file += ".gz" + tmx_file += ".gz" + terminology_file += ".gz" + + if os.path.isfile(compendium_file) is False: log.warning("{l}-compendium is missing".format(l=lang)) - if not os.path.isfile(tmx_file): + if os.path.isfile(tmx_file) is False: log.warning("{l}-tmx is missing".format(l=lang)) - if not os.path.isfile(terminology_file): + if os.path.isfile(terminology_file) is False: log.warning("{l}-terminology is missing".format(l=lang)) -def compress(folder): +def compress(source, archive): """ Compress files uzing gzip """ log = logging.getLogger("buildTm.compress") - files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))] - - for file in sorted(files): - if file.endswith(".gz"): - continue - - dest = file + ".gz" - if os.path.isfile(os.path.join(folder, dest)): - continue + log.info("Compressing") + with open(source, "rb") as file_in: + with gzip.open(archive, "wb") as file_out: + file_out.writelines(file_in) - with open(os.path.join(folder, file), "rb") as file_in: - with gzip.open(os.path.join(folder, dest), "wb") as file_out: - file_out.writelines(file_in) + os.remove(source) if __name__ == "__main__":