From 519935b087b9680eecef1d22b027331551dfb30b Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Sep 01 2025 09:59:48 +0000 Subject: Cache the task_id between runs --- diff --git a/roles/fedpkg-build-ng/defaults/main.yaml b/roles/fedpkg-build-ng/defaults/main.yaml index 348b2bb..568df46 100644 --- a/roles/fedpkg-build-ng/defaults/main.yaml +++ b/roles/fedpkg-build-ng/defaults/main.yaml @@ -4,3 +4,6 @@ release: master arches: x86_64 fetch_artifacts: true koji_task_url_base: "https://koji.fedoraproject.org/koji/taskinfo?taskID=" +valkey_environ: + # TODO: Get an actual server to use + VALKEY_HOST: localhost diff --git a/roles/fedpkg-build-ng/files/koji_cache_task.py b/roles/fedpkg-build-ng/files/koji_cache_task.py new file mode 100644 index 0000000..778ef8a --- /dev/null +++ b/roles/fedpkg-build-ng/files/koji_cache_task.py @@ -0,0 +1,72 @@ +# /// script +# dependencies = [ +# "valkey[libvalkey]", +# ] +# /// + +import argparse +import datetime +import os + + +import valkey + + +# Task-id caching is independent of any timeouts, might as well set it quite high +TASK_ID_CACHE_TIME = datetime.timedelta(days=14.0) + + +def find_koji_task(valkey_client: valkey.Valkey, args: argparse.Namespace) -> None: + # Find the lask koji task-id matching the package and commit + cache_path = f"zuul/scratch-build/{args.repo_full_name}@{args.pr_commit}/task-id" + cached_task_id = valkey_client.get(cache_path) + if cached_task_id: + print(f"Found cached koji task_id: {cached_task_id}") + else: + print("No cached koji task_id found") + + +def save_koji_task(valkey_client: valkey.Valkey, args: argparse.Namespace) -> None: + # Save the koji task-id + cache_path = f"zuul/scratch-build/{args.repo_full_name}@{args.pr_commit}/task-id" + valkey_client.set( + cache_path, + args.task_id, + ex=TASK_ID_CACHE_TIME, + ) + print(f"Cached koji task_id: {args.task_id}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", default=os.environ.get("VALKEY_HOST", "localhost")) + parser.add_argument("--port", default=6379, type=int) + parser.add_argument("--db", default=0, type=int) + + actions = parser.add_subparsers(required=True, dest="action") + + find_parser = actions.add_parser("find") + find_parser.add_argument("repo_full_name") + find_parser.add_argument("pr_id") + + save_parser = actions.add_parser("save") + save_parser.add_argument("repo_full_name") + save_parser.add_argument("pr_id") + save_parser.add_argument("task_id") + + args = parser.parse_args() + + # We store the current (cache) state of the jobs in a valkey server + valkey_client = valkey.Valkey( + host=args.host, + port=args.port, + db=args.db, + ) + + match args.action: + case "find": + find_koji_task(valkey_client, args) + case "save": + save_koji_task(valkey_client, args) + case _: + raise NotImplementedError diff --git a/roles/fedpkg-build-ng/tasks/main.yaml b/roles/fedpkg-build-ng/tasks/main.yaml index 3d41bd5..6202002 100644 --- a/roles/fedpkg-build-ng/tasks/main.yaml +++ b/roles/fedpkg-build-ng/tasks/main.yaml @@ -10,66 +10,90 @@ - name: Install system dependencies yum: - name: "fedpkg,koji,createrepo" + name: "fedpkg,koji,createrepo,python3-valkey" state: latest become: true -# In post-merge pipelines Zuul set explicitly the origin to /dev/null -- name: Set origin and tracking branch for non-scratch builds - command: "{{ item }}" - args: - chdir: "{{ zuul.project.src_dir }}" - loop: - - git remote set-url origin https://src.fedoraproject.org/{{ zuul.project.name }} - - git branch -u origin/{{ release }} - when: not scratch_build - -- name: Get PR information - uri: - url: https://src.fedoraproject.org/api/0/{{ zuul.project.name }}/pull-request/{{ zuul.change }} - return_content: true - status_code: 200 - register: pr_info - when: scratch_build - -- name: Set origin and tracking branch for scratch builds - command: "{{ item }}" - args: - chdir: "{{ zuul.project.src_dir }}" - loop: - - git remote set-url origin https://src.fedoraproject.org/{{ pr_info.json.repo_from.fullname }} - - git fetch origin - - git branch -u origin/{{ pr_info.json.branch_from }} - when: scratch_build - -# TODO: Skip building if we already have a task from a previous timed-out run -- name: Build package in Fedora build system {% if scratch_build %}(scratch build){% endif %} - command: | - fedpkg --release {{ release }} build - {% if scratch_build %} --scratch --arches {{ arches }} --skip-nvr-check --background {% endif %} - --target {{ target }} - --nowait - args: - chdir: "{{ zuul.project.src_dir }}" - register: fedpkg_build - failed_when: - - fedpkg_build is failure or - fedpkg_build.stdout is search("Server Error") or - fedpkg_build.stdout is search("ServerOffline") - retries: 3 - delay: 60 - until: - - fedpkg_build.stdout is not search("Server Error") - - fedpkg_build.stdout is not search("ServerOffline") - ignore_errors: yes - -- fail: - msg: "Build failed to submit:\n{{ fedpkg_build.stdout }}" - when: fedpkg_build is failure - -- name: Get task ID - set_fact: - task_id: "{{ fedpkg_build.stdout | regex_replace('\n', ' ') | regex_replace('.*Created task: (\\d+).*$', '\\1') }}" +- name: Try to get cached task_id + block: + - name: Search the cache server + script: koji_task.py find {{ zuul.project.name }} {{ zuul.commit_id }} + executable: python3 + environment: "{{ valkey_environ }}" + register: koji_find_task + ignore_errors: yes + + - set_fact: + task_id: "{{ koji_find_task.stdout | regex_search('Found cached koji task_id: (\\d+)') }}" + when: > + koji_find_task is not failure and + koji_find_task.stdout is match("Found cached koji task_id") + +- name: Get a fresh task_id if we didn't have one cached + block: + + # In post-merge pipelines Zuul set explicitly the origin to /dev/null + - name: Set origin and tracking branch for non-scratch builds + command: "{{ item }}" + args: + chdir: "{{ zuul.project.src_dir }}" + loop: + - git remote set-url origin https://src.fedoraproject.org/{{ zuul.project.name }} + - git branch -u origin/{{ release }} + when: not scratch_build + + - name: Get PR information + uri: + url: https://src.fedoraproject.org/api/0/{{ zuul.project.name }}/pull-request/{{ zuul.change }} + return_content: true + status_code: 200 + register: pr_info + when: scratch_build + + - name: Set origin and tracking branch for scratch builds + command: "{{ item }}" + args: + chdir: "{{ zuul.project.src_dir }}" + loop: + - git remote set-url origin https://src.fedoraproject.org/{{ pr_info.json.repo_from.fullname }} + - git fetch origin + - git branch -u origin/{{ pr_info.json.branch_from }} + when: scratch_build + + - name: Build package in Fedora build system {% if scratch_build %}(scratch build){% endif %} + command: | + fedpkg --release {{ release }} build + {% if scratch_build %} --scratch --arches {{ arches }} --skip-nvr-check --background {% endif %} + --target {{ target }} + --nowait + args: + chdir: "{{ zuul.project.src_dir }}" + register: fedpkg_build + failed_when: + - fedpkg_build is failure or + fedpkg_build.stdout is search("Server Error") or + fedpkg_build.stdout is search("ServerOffline") + retries: 3 + delay: 60 + until: + - fedpkg_build.stdout is not search("Server Error") + - fedpkg_build.stdout is not search("ServerOffline") + ignore_errors: yes + + - fail: + msg: "Build failed to submit:\n{{ fedpkg_build.stdout }}" + when: fedpkg_build is failure + + - name: Get task ID + set_fact: + task_id: "{{ fedpkg_build.stdout | regex_replace('\n', ' ') | regex_replace('.*Created task: (\\d+).*$', '\\1') }}" + + - name: Save task ID to cache server + script: koji_task.py save {{ zuul.project.name }} {{ zuul.commit_id }} {{ task_id }} + executable: python3 + environment: "{{ valkey_environ }}" + ignore_errors: yes + when: task_id is not defined - name: Return koji task url zuul_return: