From 760544f46630dd185cfa0ac9ca691cc1e8845f42 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Mar 01 2024 18:08:36 +0000 Subject: [PATCH 1/5] justfile: Run compose-dry-run for a single variant We'll run it for all official variants in the Ansible playbook used for the Zuul CI instead. --- diff --git a/justfile b/justfile index 210f336..96d3d51 100644 --- a/justfile +++ b/justfile @@ -86,8 +86,8 @@ manifest variant=default_variant: rpm-ostree compose tree --print-only --repo=repo fedora-{{variant}}.yaml -# Perform dependency resolution for all official variants -compose-dry-run: +# Perform dependency resolution for a given variant (defaults to Silverblue) +compose-dry-run variant=default_variant: #!/bin/bash set -euxo pipefail @@ -98,9 +98,7 @@ compose-dry-run: popd > /dev/null || exit 1 fi - for v in "silverblue" "kinoite" "sericea" "onyx"; do - rpm-ostree compose tree --unified-core --repo=repo --dry-run "fedora-${v}.yaml" - done + rpm-ostree compose tree --unified-core --repo=repo --dry-run fedora-{{variant}}.yaml # Alias/shortcut for compose-image command compose variant=default_variant: (compose-image variant) From 44ba47cd29b4dc0ad13368cf7004f374422e1303 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Mar 01 2024 18:32:37 +0000 Subject: [PATCH 2/5] zuul: Fix YAML indentation --- diff --git a/.zuul.yaml b/.zuul.yaml index ddd3bf5..7c7f530 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,17 +1,17 @@ - - project: - check: - jobs: - - validate +- project: + check: + jobs: + - validate - - job: - name: validate - description: Validate manifests, scripts, dependencies and compose the base image - # Increased timeout as builds can take a while - timeout: 3600 - run: ci/validate.yaml - nodeset: - nodes: - - name: vm - # Replace by a fixed version label after branching, once available: - # https://fedora.softwarefactory-project.io/zuul/labels - label: cloud-fedora-rawhide-medium +- job: + name: validate + description: Validate manifests, scripts, dependencies and compose the base image + # Increased timeout as builds can take a while + timeout: 3600 + run: ci/validate.yaml + nodeset: + nodes: + - name: vm + # Replace by a fixed version label after branching, once available: + # https://fedora.softwarefactory-project.io/zuul/labels + label: cloud-fedora-rawhide-medium diff --git a/ci/validate.yaml b/ci/validate.yaml index 928cbc2..691695a 100644 --- a/ci/validate.yaml +++ b/ci/validate.yaml @@ -1,25 +1,25 @@ - - hosts: all - tasks: - - name: Install dependencies - package: - name: - - 'jq' - - 'just' - - 'ostree' - - 'python3-pyyaml' - - 'rpm-ostree' - - 'selinux-policy-targeted' - state: present - become: yes - - name: Validate manifests and scripts syntax - ansible.builtin.command: - chdir: "{{ zuul.project.src_dir }}" - cmd: just validate - - name: Perform dependency resolution for all variants - ansible.builtin.command: - chdir: "{{ zuul.project.src_dir }}" - cmd: just compose-dry-run - - name: Compose the base variant - ansible.builtin.command: - chdir: "{{ zuul.project.src_dir }}" - cmd: just compose-legacy base +- hosts: all + tasks: + - name: Install dependencies + package: + name: + - 'jq' + - 'just' + - 'ostree' + - 'python3-pyyaml' + - 'rpm-ostree' + - 'selinux-policy-targeted' + state: present + become: yes + - name: Validate manifests and scripts syntax + ansible.builtin.command: + chdir: "{{ zuul.project.src_dir }}" + cmd: just validate + - name: Perform dependency resolution for all variants + ansible.builtin.command: + chdir: "{{ zuul.project.src_dir }}" + cmd: just compose-dry-run + - name: Compose the base variant + ansible.builtin.command: + chdir: "{{ zuul.project.src_dir }}" + cmd: just compose-legacy base From 97c49fdb1652f5c8d2b51b0d8cb3209976433687 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Mar 01 2024 19:10:05 +0000 Subject: [PATCH 3/5] ci/validate: Also validate Zuul YAML config --- diff --git a/ci/validate b/ci/validate index 23a213c..d9b90af 100755 --- a/ci/validate +++ b/ci/validate @@ -22,17 +22,16 @@ def validate_shell(rootfd, name): for root, dirs, files, rootfd in os.fwalk('.'): # Skip .git, repo, cache, tmp, logs, fedora-comps - for d in ['.git', 'repo', 'cache', 'tmp', 'logs', 'fedora-comps', 'ci']: + for d in ['.git', 'repo', 'cache', 'tmp', 'logs', 'fedora-comps']: if d in dirs: dirs.remove(d) for name in files: - if name == ".zuul.yaml": - continue if name.endswith(('.yaml', '.yml')): print("Validating:", name) with open(os.open(name, dir_fd=rootfd, flags=os.O_RDONLY)) as f: yaml.safe_load(f) - result = subprocess.run(['grep', '-RniEv', '^( )*[a-z#/-]|^$|^#', name], encoding='UTF-8') + result = subprocess.run(['grep', '-RniEv', '^( )*[a-z#/-]|^$|^#', name], encoding='UTF-8', + preexec_fn=lambda: os.fchdir(rootfd)) if result.returncode == 0: raise Exception("Found likely invalid indentation in YAML file: {}".format(name)) validated +=1 @@ -47,7 +46,7 @@ for root, dirs, files, rootfd in os.fwalk('.'): if not stbuf.st_mode & stat.S_IXUSR: continue mimetype = subprocess.check_output(['file', '-b', '--mime-type', name], encoding='UTF-8', - preexec_fn=lambda: os.fchdir(rootfd)).strip() + preexec_fn=lambda: os.fchdir(rootfd)).strip() if mimetype == 'text/x-shellscript': print("Validating:", name) validate_shell(rootfd, name) From 8e0b7314f943f503045b762ae55e4e7e22ff735d Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Mar 01 2024 19:10:05 +0000 Subject: [PATCH 4/5] ci: Fix yaml lint --- diff --git a/ci/validate.yaml b/ci/validate.yaml index 691695a..122cc87 100644 --- a/ci/validate.yaml +++ b/ci/validate.yaml @@ -10,7 +10,7 @@ - 'rpm-ostree' - 'selinux-policy-targeted' state: present - become: yes + become: true - name: Validate manifests and scripts syntax ansible.builtin.command: chdir: "{{ zuul.project.src_dir }}" From 4b90deb10d13be81ecd256698a14f7e94ce16da6 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Mar 01 2024 19:10:05 +0000 Subject: [PATCH 5/5] ci: Split dependency check and run checks all on failures - Split the dependency check into a distinct step for each variant - If the dependency test failed, still run the compose base test to get results about it - Add a step at the end to make sure that if any step fails, the workflow still fail See: https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_error_handling.html#ignoring-failed-commands See discussion in: https://pagure.io/workstation-ostree-config/pull-request/493 --- diff --git a/ci/validate.yaml b/ci/validate.yaml index 122cc87..b07f753 100644 --- a/ci/validate.yaml +++ b/ci/validate.yaml @@ -11,15 +11,46 @@ - 'selinux-policy-targeted' state: present become: true + - name: Validate manifests and scripts syntax ansible.builtin.command: chdir: "{{ zuul.project.src_dir }}" cmd: just validate - - name: Perform dependency resolution for all variants - ansible.builtin.command: + + - name: Perform dependency resolution for Silverblue + ansible.builtin.shell: + chdir: "{{ zuul.project.src_dir }}" + cmd: "just compose-dry-run silverblue && touch .zuulci.silverblue" + # Still run the next step if this one fails + ignore_errors: true + + - name: Perform dependency resolution for Kinoite + ansible.builtin.shell: + chdir: "{{ zuul.project.src_dir }}" + cmd: "just compose-dry-run kinoite && touch .zuulci.kinoite" + # Still run the next step if this one fails + ignore_errors: true + + - name: Perform dependency resolution for Sway Atomic + ansible.builtin.shell: chdir: "{{ zuul.project.src_dir }}" - cmd: just compose-dry-run + cmd: "just compose-dry-run sericea && touch .zuulci.sericea" + # Still run the next step if this one fails + ignore_errors: true + + - name: Perform dependency resolution for Budgie Atomic + ansible.builtin.shell: + chdir: "{{ zuul.project.src_dir }}" + cmd: "just compose-dry-run onyx && touch .zuulci.onyx" + # Still run the next step if this one fails + ignore_errors: true + - name: Compose the base variant ansible.builtin.command: chdir: "{{ zuul.project.src_dir }}" cmd: just compose-legacy base + + - name: Check if any previous dependency resolution steps failed + ansible.builtin.shell: + chdir: "{{ zuul.project.src_dir }}" + cmd: "[[ -f .zuulci.silverblue ]] && [[ -f .zuulci.kinoite ]] && [[ -f .zuulci.sericea ]] && [[ -f .zuulci.onyx ]]"