From cb0b155c8f0c5005d7a8c8844af1335dcf2f2223 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Aug 25 2023 22:34:30 +0000 Subject: store language results as csv and display file size fix: #35 --- diff --git a/build_website.py b/build_website.py index 14c5fc5..450a706 100755 --- a/build_website.py +++ b/build_website.py @@ -3,6 +3,7 @@ import argparse import datetime +from collections import defaultdict import jinja2 import json @@ -92,17 +93,17 @@ def main(): for package in sorted(packages): count += 1 log.debug("Preparing package {c}/{t} - {p}".format(c=count, t=total, p=package)) - name = package[: -len(".json")] - dest_file = os.path.join(data_pkgs_folder, name + ".json") - file_stats = os.path.join(packages_stats, name + ".json") + package_name = package[: -len(".json")] + package_statistics_file = os.path.join(data_pkgs_folder, package_name + ".json") + file_stats = os.path.join(packages_stats, package_name + ".json") - results = consolidate_package_stats(file_stats, log_files, os.path.join(results_folder, "package", name)) - store_json_file(results, dest_file) + results = consolidate_package_stats(file_stats, log_files, os.path.join(results_folder, "package", package_name)) + store_json_file(results, package_statistics_file) for lang in results.get("stats", []): val = packages_langs_results.get(lang["lang_code"], []) val.append( { - "name": name, + "name": package_name, "progress": lang["progress"], "translated": lang["translated"], "team": lang["team"], @@ -119,14 +120,14 @@ def main(): for lang in sorted(languages): if lang.endswith(".json"): code = lang[: -len(".json")] - dest_file = os.path.join(data_langs_folder, code + ".json") + package_statistics_file = os.path.join(data_langs_folder, code + ".json") - if os.path.isfile(dest_file): + if os.path.isfile(package_statistics_file): continue results = consolidate_language_stats(os.path.join(langs_stats, lang), distribution_stats) results["packages"] = packages_langs_results.get(code, dict()) - store_json_file(results, dest_file) + store_json_file(results, package_statistics_file) log.info("Load CLDR data") with open("CLDR-raw/languageData.json", "r") as read_file: @@ -149,14 +150,16 @@ def main(): ] for lang in sorted(languages): code = lang[: -len(".json")] - dest_file = os.path.join(static_langs_folder, code + ".adoc") + package_statistics_file = os.path.join(static_langs_folder, code + ".adoc") - if os.path.isfile(dest_file): + if os.path.isfile(package_statistics_file): continue with open(os.path.join(data_langs_folder, lang), "r") as read_file: content = json.load(read_file) + pd.DataFrame(content["packages"]).to_csv(os.path.join(static_tm_folder, f"{code}.csv"), index=False) + cldr_code = code.split("_", 1)[0] # ro_MD or zh_Hant_HK cldr_code = cldr_code.split("@", 1)[0] # ca@valencia @@ -170,7 +173,7 @@ def main(): if len(territories) == 0: log.warning("The language {l} does not exist in territories data from CLDR".format(l=code)) - generate_static_pages_langs(args.results, code, content, dest_file, territories) + generate_static_pages_langs(args.results, code, content, package_statistics_file, territories, tm_folder, static_tm_folder) log.info("Generate static content for packages") packages = [ @@ -180,31 +183,31 @@ def main(): ] for package in sorted(packages): code = package[: -len(".json")] - dest_file = os.path.join(static_pkgs_folder, code + ".adoc") + package_statistics_file = os.path.join(static_pkgs_folder, code + ".adoc") - if os.path.isfile(dest_file): + if os.path.isfile(package_statistics_file): continue with open(os.path.join(data_pkgs_folder, package), "r") as read_file: content = json.load(read_file) - generate_static_pages_packages(args.results, code, content, dest_file) + generate_static_pages_packages(args.results, code, content, package_statistics_file) log.info("Generating indexes") - dest_file = os.path.join(static_folder, "_index.adoc") - generate_release_index(args.results, dest_file, distribution_stats) + package_statistics_file = os.path.join(static_folder, "_index.adoc") + generate_release_index(args.results, package_statistics_file, distribution_stats) - dest_file = os.path.join(static_langs_folder, "_index.adoc") - generate_language_index(args.results, dest_file) + package_statistics_file = os.path.join(static_langs_folder, "_index.adoc") + generate_language_index(args.results, package_statistics_file) - dest_file = os.path.join(static_pkgs_folder, "_index.adoc") - generate_package_index(args.results, dest_file) + package_statistics_file = os.path.join(static_pkgs_folder, "_index.adoc") + generate_package_index(args.results, package_statistics_file) for code in cldr_territories.keys(): # prevent containers and alternative names to be included if code in cldr_territories_info.keys(): - dest_file = os.path.join(static_territories_folder, code, "_index.adoc") - generate_territory_index(dest_file, cldr_territories[code], code, cldr_territories_info.get(code, {})) + package_statistics_file = os.path.join(static_territories_folder, code, "_index.adoc") + generate_territory_index(package_statistics_file, cldr_territories[code], code, cldr_territories_info.get(code, {})) log.info("Copy translation memories") languages = [ @@ -384,7 +387,7 @@ def consolidate_package_stats(stats_file, log_files, package_folder): return results -def generate_static_pages_langs(results, code, content, dest_file, territories): +def generate_static_pages_langs(results: str, code: str, content: dict, destination_file: str, territories: list[str], tm_folder: str, static_tm_folder: str) -> None: log = logging.getLogger("buildWebsite.generate_static_pages_langs") data = content data["lang_name_en"] = langtable.language_name( @@ -395,103 +398,81 @@ def generate_static_pages_langs(results, code, content, dest_file, territories): data["results"] = results data["lang_code"] = code data["now"] = datetime.datetime.utcnow() + data["files"] = defaultdict(dict) + data["files"]["compendium"]["url"] = f"/{results}/{code}.po.gz" + data["files"]["compendium"]["size"] = os.path.getsize(os.path.join(tm_folder, f"{code}.po.gz")) + data["files"]["terminology"]["url"] = f"/{results}/{code}.terminology.po.gz" + data["files"]["terminology"]["size"] = os.path.getsize(os.path.join(tm_folder, f"{code}.terminology.po.gz")) + data["files"]["tmx"]["url"] = f"/{results}/{code}.tmx.gz" + data["files"]["tmx"]["size"] = os.path.getsize(os.path.join(tm_folder, f"{code}.tmx.gz")) + data["files"]["csv"]["url"] = f"/{results}/{code}.csv" + data["files"]["csv"]["size"] = os.path.getsize(os.path.join(static_tm_folder, f"{code}.csv")) if len(territories) > 0: data["territories"] = territories - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "language.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) - - with open(dest_file, "w") as write_out: - write_out.write(outputText) + apply_jinja_template(data, destination_file, "language.adoc") -def generate_static_pages_packages(results, code, content, dest_file): +def generate_static_pages_packages(results, code, content, destination_file): log = logging.getLogger("buildWebsite.generate_static_pages_packages") data = content data["results"] = results data["package"] = code data["now"] = datetime.datetime.utcnow() - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "package.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) - - with open(dest_file, "w") as write_out: - write_out.write(outputText) + apply_jinja_template(data, destination_file, "package.adoc") -def generate_release_index(release, dest_file, data): +def generate_release_index(release, destination_file, data): log = logging.getLogger("buildWebsite.generate_release_index") data["release"] = release data["now"] = datetime.datetime.utcnow() - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "_index.release.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) + apply_jinja_template(data, destination_file, "_index.release.adoc") - with open(dest_file, "w") as write_out: - write_out.write(outputText) - -def generate_language_index(release, dest_file): +def generate_language_index(release, destination_file): log = logging.getLogger("buildWebsite.generate_language_index") data = dict() data["release"] = release data["now"] = datetime.datetime.utcnow() - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "_index.language.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) - - with open(dest_file, "w") as write_out: - write_out.write(outputText) + apply_jinja_template(data, destination_file, "_index.language.adoc") -def generate_package_index(distribution, dest_file): +def generate_package_index(distribution, destination_file): log = logging.getLogger("buildWebsite.generate_package_index") data = dict() data["distribution"] = distribution data["now"] = datetime.datetime.utcnow() - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "_index.package.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) - - with open(dest_file, "w") as write_out: - write_out.write(outputText) + apply_jinja_template(data, destination_file, "_index.package.adoc") -def generate_territory_index(dest_file: str, name: str, code: str, data: list): +def generate_territory_index(destination_file: str, name: list[str], code: str, data: dict): log = logging.getLogger("buildWebsite.generate_package_index") data["name"] = name data["code"] = code - templateLoader = jinja2.FileSystemLoader(searchpath="./templates/") - templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined) - TEMPLATE_FILE = "_index.territory.adoc" - template = templateEnv.get_template(TEMPLATE_FILE) - outputText = template.render(data) + apply_jinja_template(data, destination_file, "_index.territory.adoc") - os.makedirs(os.path.dirname(os.path.abspath(dest_file)), exist_ok=True) - with open(dest_file, "w") as write_out: - write_out.write(outputText) - -def store_json_file(content, dest_file): - with open(dest_file, "w") as f: +def store_json_file(content, destination_file): + with open(destination_file, "w") as f: f.write(json.dumps(content, indent=2)) +def apply_jinja_template(data: dict, destination_file: str, template_file: str): + os.makedirs(os.path.dirname(os.path.abspath(destination_file)), exist_ok=True) + + template_loader = jinja2.FileSystemLoader(searchpath="./templates/") + template_env = jinja2.Environment(loader=template_loader, undefined=jinja2.Undefined) + template = template_env.get_template(template_file) + output_text = template.render(data) + + with open(destination_file, "w") as write_out: + write_out.write(output_text) + + if __name__ == "__main__": main() diff --git a/templates/language.adoc b/templates/language.adoc index 787d6f5..849481d 100644 --- a/templates/language.adoc +++ b/templates/language.adoc @@ -27,9 +27,10 @@ Possible scripts are: {% for script in scripts -%}{{ script }} {%- endfor %} Download: -* link:{{ "{{% resource url=" }}"/{{ results }}/{{ lang_code }}.po.gz" {{ "%}}" }}[{{ lang_code }} compendium] (aggregation of all strings found in po files) -* link:{{ "{{% resource url=" }}"/{{ results }}/{{ lang_code }}.terminology.po.gz" {{ "%}}" }}[{{ lang_code }} terminology] see https://docs.translatehouse.org/projects/translate-toolkit/en/latest/commands/poterminology.html[poterminology] -* link:{{ "{{% resource url=" }}"/{{ results }}/{{ lang_code }}.tmx.gz" {{ "%}}" }}[{{ lang_code }} translation memory] see https://en.wikipedia.org/wiki/Translation_Memory_eXchange[tmx] +* link:{{ "{{% resource url=" }}"{{ files["compendium"]["url"] }}" {{ "%}}" }}[{{ lang_code }} compendium ({{ files["compendium"]["size"]|filesizeformat() }})] (aggregation of all strings found in po files) +* link:{{ "{{% resource url=" }}"{{ files["terminology"]["url"] }}" {{ "%}}" }}[{{ lang_code }} terminology ({{ files["terminology"]["size"]|filesizeformat() }})] see https://docs.translatehouse.org/projects/translate-toolkit/en/latest/commands/poterminology.html[poterminology] +* link:{{ "{{% resource url=" }}"{{ files["tmx"]["url"] }}" {{ "%}}" }}[{{ lang_code }} translation memory ({{ files["tmx"]["size"]|filesizeformat() }})] see https://en.wikipedia.org/wiki/Translation_Memory_eXchange[tmx] +* link:{{ "{{% resource url=" }}"{{ files["csv"]["url"] }}" {{ "%}}" }}[{{ lang_code }} generated stats ({{ files["csv"]["size"]|filesizeformat() }})] Packages: