From ac75733472a4bac2de74a2a30bf86939d3ff726a Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 12:31:32 +0000 Subject: [PATCH 1/5] clone source repositories locally --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index aabcaab..3472844 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -17,8 +17,10 @@ def main(): args = parser.parse_args() output_dir = args.output_dir + source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" os.makedirs(output_dir, exist_ok=True) + os.makedirs(source_dir, exist_ok=True) output_dir = os.path.abspath(output_dir) parse_yml(output_dir) @@ -27,10 +29,23 @@ def call_podman_and_src_to_po(output_dir, doc): command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) command += "-v {c}/src-to-pot.sh:/scripts/src-to-pot.sh:z ".format(c=os.getcwd()) command += "asamalik/fedora-docs-translations /scripts/src-to-pot.sh {d}".format(d=doc) - print(command) - args = shlex.split(command) - subprocess.run(args, check=True) - + # print(command) + # args = shlex.split(command) + # subprocess.run(args, check=True) + +def clone_source(url, branch = "master"): + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + + if os.path.exists(dir): + subprocess.run(['git', 'fetch'], check=True, cwd = dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) + subprocess.run(['git', 'pull'], check=True, cwd = dir) + else: + command = "git clone -b {b} {u} {d}".format(b = branch, u=url, d=dir) + print(command) + args = shlex.split(command) + subprocess.run(args, check=True) def parse_yml(output_dir): """List repositories to convert to pot from antora yaml file""" @@ -52,14 +67,17 @@ def parse_yml(output_dir): src_basedir = source['start_path'] doc = "--clone-target-repo --src-basedir \"{bd}\" {url} /output".format( bd=src_basedir, url=url) + clone_source(url) elif 'branches' in source: for branch in source['branches']: src_branch = branch doc = "--clone-target-repo --src-branch \"{br}\" {url} /output".format( br=src_branch, url=url) + clone_source(url, src_branch) else: doc = "--clone-target-repo {url} /output".format(url=url) + clone_source(url) call_podman_and_src_to_po(output_dir, doc) From 18d4d9177246fc6f3be13f9ab4e1b8c7e64a9e86 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 12:35:08 +0000 Subject: [PATCH 2/5] move source cloning in a dedicated function --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index 3472844..a25f634 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -24,6 +24,7 @@ def main(): output_dir = os.path.abspath(output_dir) parse_yml(output_dir) + clone_sources() def call_podman_and_src_to_po(output_dir, doc): command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) @@ -47,6 +48,35 @@ def clone_source(url, branch = "master"): args = shlex.split(command) subprocess.run(args, check=True) +def clone_sources(): + """List repositories to convert to pot from antora yaml file""" + + # download site.yml + urllib.request.urlretrieve( + "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" + ) + + # Read site.yml + with open("site.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + # Parse site.yml + for source in data_loaded['content']['sources']: + url = source['url'] + + if 'start_path' in source: + clone_source(url) + + elif 'branches' in source: + for branch in source['branches']: + clone_source(url, branch) + else: + clone_source(url) + + + # Remove site.yml + os.remove("site.yml") + def parse_yml(output_dir): """List repositories to convert to pot from antora yaml file""" From c7baff7e144076bf356db9d040a74de6332e61bb Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 12:37:59 +0000 Subject: [PATCH 3/5] hop --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index a25f634..d4fe88c 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -97,17 +97,14 @@ def parse_yml(output_dir): src_basedir = source['start_path'] doc = "--clone-target-repo --src-basedir \"{bd}\" {url} /output".format( bd=src_basedir, url=url) - clone_source(url) elif 'branches' in source: for branch in source['branches']: src_branch = branch doc = "--clone-target-repo --src-branch \"{br}\" {url} /output".format( br=src_branch, url=url) - clone_source(url, src_branch) else: doc = "--clone-target-repo {url} /output".format(url=url) - clone_source(url) call_podman_and_src_to_po(output_dir, doc) From f7fb5daa05f25be64b70fe5223596836f50a1545 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 13:06:04 +0000 Subject: [PATCH 4/5] build component list and pull l10n repositories --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index d4fe88c..be4b34a 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -24,7 +24,7 @@ def main(): output_dir = os.path.abspath(output_dir) parse_yml(output_dir) - clone_sources() + make_component_lists() def call_podman_and_src_to_po(output_dir, doc): command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) @@ -48,6 +48,18 @@ def clone_source(url, branch = "master"): args = shlex.split(command) subprocess.run(args, check=True) +def clone_l10n(url): + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + + if os.path.exists(dir): + subprocess.run(['git', 'pull'], check=True, cwd = dir) + else: + command = "git clone {u} {d}".format(u=url, d=dir) + print(command) + args = shlex.split(command) + subprocess.run(args, check=True) + def clone_sources(): """List repositories to convert to pot from antora yaml file""" @@ -77,6 +89,58 @@ def clone_sources(): # Remove site.yml os.remove("site.yml") +def get_component(dir, branch = 'master'): + subprocess.run(['git', 'fetch'], check=True, cwd = dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) + + with open(dir + "/antora.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + component = data_loaded['name'] + version = data_loaded['version'] + + f = [] + for (dirpath, dirnames, filenames) in os.walk(dir + "/modules"): + f.extend(dirnames) + break + + for module in f: + if module == "ROOT": + target_repo_name = component + else: + target_repo_name = component + "-" + module + clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + +def make_component_lists(): + """List repositories to convert to pot from antora yaml file""" + + # download site.yml + urllib.request.urlretrieve( + "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" + ) + + # Read site.yml + with open("site.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + # Parse site.yml + for source in data_loaded['content']['sources']: + url = source['url'] + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + + if 'start_path' in source: + get_component(dir+"/"+source['start_path']) + + elif 'branches' in source: + for branch in source['branches']: + get_component(dir, branch) + else: + get_component(dir) + + # Remove site.yml + os.remove("site.yml") + def parse_yml(output_dir): """List repositories to convert to pot from antora yaml file""" From 75556532bd232c05bec831537c9a55faa00abeb1 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 13:06:50 +0000 Subject: [PATCH 5/5] add gitignore --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8d4e1dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +l10n/ +sources/ \ No newline at end of file