From 90da771cac9cd76c8f658740b5f243d6277e5990 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 17 2019 05:41:10 +0000 Subject: [PATCH 1/3] add translated-sources commit and moved commit_l10n --- diff --git a/README.md b/README.md index 83ab21a..85bbf7f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ It takes the following arguments : Should we clone/update sources or use local content? --commit_l10n {true,false} Should we commit changes on l10n repos? + --commit_tsources {true,false} + Should we commit translated sources? --component COMPONENT Antora component to convert, all if unset diff --git a/build.py b/build.py index 6da3b99..1150a05 100755 --- a/build.py +++ b/build.py @@ -10,12 +10,11 @@ from urllib.parse import urlparse import yaml -COMMIT_CHANGES = 'false' SELECTED_COMPONENT = None def main(): """Handle params""" - global COMMIT_CHANGES, SELECTED_COMPONENT + global SELECTED_COMPONENT parser = argparse.ArgumentParser( description="Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos") @@ -23,11 +22,12 @@ def main(): help="Should we clone/update sources or use local content?") parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True, help="Should we commit changes on l10n repos?") + parser.add_argument("--commit_tsources", choices=['true', 'false'], required=True, + help="Should we commit translated sources?") parser.add_argument("--component", required=False, help="Antora component to convert, all if unset") args = parser.parse_args() - COMMIT_CHANGES = args.commit_l10n SELECTED_COMPONENT = args.component source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" @@ -44,6 +44,12 @@ def main(): clone_translated_source() make_component_lists() + if args.commit_l10n == "true": + commit_l10n_repos() + + if args.commit_tsources == "true": + commit_translated_sources() + def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): """Call src-to-ot.sh to convert english source (adoc) content to pot files""" print("call_src_to_pot: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) @@ -73,10 +79,11 @@ def clone_source(doc_url, branch="master"): else: subprocess.run(["git", "clone", "-b", branch, doc_url, repo_dir], check=True) -def clone_l10n(url): +def clone_l10n(repo_name): """Clone localization repos in l10n folder (update content if existing)""" - print("* clone_l10n: " + url) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + print("* clone_l10n: " + repo_name) + + url = "https://pagure.io/fedora-l10n/" + repo_name repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name if os.path.exists(repo_dir): @@ -98,16 +105,35 @@ def clone_translated_source(): else: subprocess.run(["git", "clone", url, repo_dir], check=True) -def commit_l10n(url): - """Commit and push changes of a localization repo""" - global COMMIT_CHANGES +def commit_translated_source(): + """Commit translated sources""" + print("* commit_translated_source") + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" - if COMMIT_CHANGES != "true": - return + subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) + + subprocess.run(["git", "commit", "-m", "automatic update of translated content"], + check=True, cwd=repo_dir) + + subprocess.run(['git', 'push'], check=True, cwd=repo_dir) + + +def commit_l10n_repos(): + """Commit and push changes of localization repositories""" + + print("* commit_l10n_repos") + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + + # commit each folders of the l10n path + [commit_l10n(r) for r in next(os.walk(repo_dir))[1]] + + +def commit_l10n(repo_name): + """Commit and push changes of a localization repo""" - print("* commit_l10n: " + url) + print("* commit_l10n: " + repo_name) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + url = "https://pagure.io/fedora-l10n/" + repo_name repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) @@ -177,9 +203,8 @@ def get_component(comp_dir, basedir='/', branch='master'): target_repo_name = component else: target_repo_name = component + "-" + module - clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + clone_l10n(target_repo_name) call_src_to_pot(url=comp_dir, basedir=basedir, branch=branch, module=module) - commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_po_to_src(url=comp_dir, basedir=basedir, branch=branch, module=module) def make_component_lists(): From b5ab38641e071311353d089ca6d6cd1db33a493f Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 17 2019 06:03:37 +0000 Subject: [PATCH 2/3] make po-to-src language agnostic --- diff --git a/build.py b/build.py index 1150a05..bbbb6aa 100755 --- a/build.py +++ b/build.py @@ -62,10 +62,10 @@ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): """Call po-to-src.sh to convert po files to translated sources (adoc)""" print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" - for lang in ['fr', 'cs', 'ja']: - subprocess.run(['./po-to-src.sh', "--src-branch", branch, - "--src-basedir", basedir, "--module", module, lang, url, - output], check=True) + + subprocess.run(['./po-to-src.sh', "--src-branch", branch, + "--src-basedir", basedir, "--module", module, url, + output], check=True) def clone_source(doc_url, branch="master"): """Clone english sources in sources folder (update content if existing)""" diff --git a/po-to-src.sh b/po-to-src.sh index 3145468..113ea8e 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -9,7 +9,6 @@ usage () { echo "Arguments:" echo " REPO_URL Source repository URL." echo " OUTPUT_DIR Path to an output directory." - echo " LOCALE What language to work with. Such as 'fr-FR'." echo "" echo "Parametres:" echo " Generic:" @@ -51,16 +50,6 @@ while true ; do esac done -# Need LOCALE -if [ "$1" != "" ]; then - language=$1 - shift -else - usage - echo "Please specify LOCALE, REPO_URL and TARGET_DIR" - exit 1 -fi - # Need REPO_URL if [ "$1" != "" ]; then src_repo=$1 @@ -98,6 +87,10 @@ if [ -z "$src_basedir" ]; then src_basedir="" fi +echo "*****************" +echo "* po-to-src" +echo "*****************" + tmp_english_src=$(mktemp -d) delete_tmp() { @@ -204,76 +197,76 @@ tmp_po_pot="/home/jean-baptiste/Projects/fedora-l10n/translations-scripts/l10n/$ # Missing po for .adoc: the .adoc just get s copied # # TODO: stretch idea: if the translations don't exist, I could just copy -# the english sources over... this could simplify automation +# the english sources over... this could simplify automation + +for lang_folder in "$tmp_po_pot/po/"* ; do + [[ -d "$lang_folder" ]] || break # if no result + + language=$(basename $lang_folder) + echo "*****************" + echo "* LANG: $language" + echo "*****************" + #po_pot_module_dir="$output_dir/$po_pot_repo_name/pot/$version" + po_pot_module_dir="$tmp_po_pot/po/$language/$version" + output_module_dir="$output_dir/$language/$component/$version/modules/$module" + + # Confert all .adoc files to .pot + pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit + + for source_adoc_file in $(find . -name '*.adoc'); do + po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" + target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" + if [ -f "$po_file" ] ; then + # TODO: convert file + if ! po4a-translate \ + --format asciidoc \ + --master "$source_adoc_file" \ + --master-charset "UTF-8" \ + --po "$po_file" \ + --localized "$target_adoc_file" \ + --localized-charset "UTF-8" \ + --keep 0 + then + echo "Error converting a PO file: $po_file" + delete_tmp + exit 1 + fi + else + # "Warning: The following file is in the source, but not in the translations repo:" + mkdir -p "$(dirname "$target_adoc_file")" + cp "$source_adoc_file" "$target_adoc_file" + fi + done -#po_pot_module_dir="$output_dir/$po_pot_repo_name/pot/$version" -po_pot_module_dir="$tmp_po_pot/po/$language/$version" -output_module_dir="$output_dir/$language/$component/$version/modules/$module" + popd > /dev/null || exit -# Make sure the language exists in the PO/POT repo -# if [ ! -d "$tmp_po_pot/po/$language" ] ; then -# echo "Error. Locale $language doesn't exist in the PO/POT repo." -# delete_tmp -# exit 1 -# fi -# Confert all .adoc files to .pot -pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit - -for source_adoc_file in $(find . -name '*.adoc'); do - po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" - target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" - if [ -f "$po_file" ] ; then - # TODO: convert file - if ! po4a-translate \ - --format asciidoc \ - --master "$source_adoc_file" \ - --master-charset "UTF-8" \ - --po "$po_file" \ - --localized "$target_adoc_file" \ - --localized-charset "UTF-8" \ - --keep 0 - then - echo "Error converting a PO file: $po_file" - delete_tmp - exit 1 + # Convert the antora.yml to antora.yml.pot + # if this is the main module + if [ "$module" = "ROOT" ]; then + pushd "$tmp_english_src/$src_basedir" > /dev/null || exit + + source_antora_yml="$tmp_english_src/$src_basedir/antora.yml" + po_antora_yml="$tmp_po_pot/po/$language/$version/antora.yml.po" + target_antora_yml="$output_dir/$language/$component/$version/antora.yml" + + if [ -f "$po_antora_yml" ] ; then + po4a-translate \ + --format yaml \ + --option keys="title" \ + --master "$source_antora_yml" \ + --master-charset "UTF-8" \ + --po "$po_antora_yml" \ + --localized "$target_antora_yml" \ + --localized-charset "UTF-8" \ + --keep 0 + else + # "Warning: There should be an antora.yml.po, but there isn't." + mkdir -p "$(dirname "$target_antora_yml")" + cp "$source_antora_yml" "$target_antora_yml" fi - else - # "Warning: The following file is in the source, but not in the translations repo:" - mkdir -p "$(dirname "$target_adoc_file")" - cp "$source_adoc_file" "$target_adoc_file" + popd > /dev/null || exit fi done -popd > /dev/null || exit - - -# Convert the antora.yml to antora.yml.pot -# if this is the main module -if [ "$module" = "ROOT" ]; then - pushd "$tmp_english_src/$src_basedir" > /dev/null || exit - - source_antora_yml="$tmp_english_src/$src_basedir/antora.yml" - po_antora_yml="$tmp_po_pot/po/$language/$version/antora.yml.po" - target_antora_yml="$output_dir/$language/$component/$version/antora.yml" - - if [ -f "$po_antora_yml" ] ; then - po4a-translate \ - --format yaml \ - --option keys="title" \ - --master "$source_antora_yml" \ - --master-charset "UTF-8" \ - --po "$po_antora_yml" \ - --localized "$target_antora_yml" \ - --localized-charset "UTF-8" \ - --keep 0 - else - # "Warning: There should be an antora.yml.po, but there isn't." - mkdir -p "$(dirname "$target_antora_yml")" - cp "$source_antora_yml" "$target_antora_yml" - fi - popd > /dev/null || exit -fi - - delete_tmp From c29d684883720f34d89af7fd47c6af62d6b80908 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 18 2019 20:03:56 +0000 Subject: [PATCH 3/3] fix navigation issue --- diff --git a/po-to-src.sh b/po-to-src.sh index 113ea8e..56e0fc5 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -216,10 +216,17 @@ for lang_folder in "$tmp_po_pot/po/"* ; do for source_adoc_file in $(find . -name '*.adoc'); do po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" + + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$source_adoc_file")" = "nav" ] ; then + format="text" + fi + if [ -f "$po_file" ] ; then # TODO: convert file if ! po4a-translate \ - --format asciidoc \ + --format "$format" \ --master "$source_adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" \ diff --git a/src-to-pot.sh b/src-to-pot.sh index 68b3569..669f29b 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -214,10 +214,15 @@ while IFS= read -r -d '' adoc_file do pot_file="$module_dir/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").pot" + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$adoc_file")" = "nav" ] ; then + format="text" + fi # generate new POT files if ! po4a-gettextize \ - --format asciidoc \ + --format "$format" \ --master "$adoc_file" \ --master-charset "UTF-8" \ --localized-charset "UTF-8" \ @@ -262,8 +267,14 @@ do # po4a-updatepo would be angry otherwise sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$adoc_file")" = "nav" ] ; then + format="text" + fi + if ! po4a-updatepo \ - --format asciidoc \ + --format "$format" \ --master "$adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" ; then