From 022e5d4a572fc8d2ecd277fdca30c67c72d5b335 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:33:47 +0000
Subject: [PATCH 1/12] ci: Use just recipes for all tests


---

diff --git a/ci/validate.yaml b/ci/validate.yaml
index 4d8d78c..9483ff3 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -2,10 +2,10 @@
    tasks:
      - name: Install dependencies
        package:
-         name: ['python3-pyyaml']
+         name: ['just', 'python3-pyyaml']
          state: present
        become: yes
      - name: Validate manifests and scripts syntax
        ansible.builtin.command:
          chdir: "{{ zuul.project.src_dir }}"
-         cmd: ci/validate
+         cmd: just validate

From 916a3b2a841f2e702a9e1c7e1856d9761c40c23b Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:33:47 +0000
Subject: [PATCH 2/12] justfile: Add compose-dry-run recipe


To use in CI to perform dependency resolution for all official variants.

---

diff --git a/justfile b/justfile
index 4cedd9d..4fac76c 100644
--- a/justfile
+++ b/justfile
@@ -81,6 +81,22 @@ manifest variant=default_variant:
 
     rpm-ostree compose tree --print-only --repo=repo fedora-{{variant}}.yaml
 
+# Perform dependency resolution for all official variants
+compose-dry-run:
+    #!/bin/bash
+    set -euxo pipefail
+
+    mkdir -p repo cache logs
+    if [[ ! -f "repo/config" ]]; then
+        pushd repo > /dev/null || exit 1
+        ostree init --repo . --mode=bare-user
+        popd > /dev/null || exit 1
+    fi
+
+    for v in "silverblue" "kinoite" "sericea" "onyx"; do
+        rpm-ostree compose tree --unified-core --repo=repo --dry-run "fedora-${v}.yaml"
+    done
+
 # Alias/shortcut for compose-image command
 compose variant=default_variant: (compose-image variant)
 

From c327ad1193eee7d6f172353b62422f1938f1a447 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:33:47 +0000
Subject: [PATCH 3/12] ci: Perform dependency resolution for all variants


---

diff --git a/ci/validate.yaml b/ci/validate.yaml
index 9483ff3..038ecdb 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -2,10 +2,18 @@
    tasks:
      - name: Install dependencies
        package:
-         name: ['just', 'python3-pyyaml']
+         name:
+           - 'just'
+           - 'ostree'
+           - 'python3-pyyaml'
+           - 'rpm-ostree'
          state: present
        become: yes
      - name: Validate manifests and scripts syntax
        ansible.builtin.command:
          chdir: "{{ zuul.project.src_dir }}"
          cmd: just validate
+     - name: Perform dependency resolution for all variants
+       ansible.builtin.command:
+         chdir: "{{ zuul.project.src_dir }}"
+         cmd: just compose-dry-run

From d15cf547702a47bdd8520e941cc1481cef4adfe0 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:33:47 +0000
Subject: [PATCH 4/12] ci: Reformat package lists


---

diff --git a/ci/compose-base.yaml b/ci/compose-base.yaml
index fe067a0..a5017b0 100644
--- a/ci/compose-base.yaml
+++ b/ci/compose-base.yaml
@@ -2,7 +2,13 @@
    tasks:
      - name: Install dependencies
        package:
-         name: ['jq', 'just', 'ostree', 'python3-pyyaml', 'rpm-ostree', 'selinux-policy-targeted']
+         name:
+           - 'jq'
+           - 'just'
+           - 'ostree'
+           - 'python3-pyyaml'
+           - 'rpm-ostree'
+           - 'selinux-policy-targeted'
          state: present
        become: yes
      - name: Compose the base variant

From 5e7c19d3251113bcff539a6ce0f5227d4fbae653 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:33:47 +0000
Subject: [PATCH 5/12] Import GitLab CI config


This config is not used in Fedora infra (see `.zuul.yaml` instead) but
is used for testing in https://gitlab.com/fedora/ostree/ci-test.

Importing it here makes synchronizing changes accross the repos easier.

Co-authored by: Alexej Kowalew <616b2f@gmail.com>

---

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..2412ad8
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: MIT
+
+# Only used in https://gitlab.com/fedora/ostree/ci-test
+# For tests running in the Fedora infrastructure, see .zuul.yaml and
+# https://fedoraproject.org/wiki/Zuul-based-ci
+
+# See: https://gitlab.com/fedora/ostree/buildroot
+image: quay.io/fedora-ostree-desktops/buildroot
+
+stages:
+  - build
+
+# As those are not official images, we build all available variants
+build:
+  stage: build
+  script:
+    - just compose-image $VARIANT
+    - just upload-container $VARIANT
+  parallel:
+    matrix:
+    - VARIANT:
+      - silverblue
+      - kinoite
+      - sericea
+      - vauxite
+      - lazurite
+      - onyx
+      - base

From 8a1900bfc8f3712c75b29e9fbaea39b7dd5a0367 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Feb 29 2024 16:36:50 +0000
Subject: [PATCH 6/12] Zuul: Unify all jobs to save on CI costs


We need a VM for both the compose-dry-run and the compose-base tests
thus to avoid spawning two VMs in Zuul, let's keep everything in a
single job to save money.

---

diff --git a/.zuul.yaml b/.zuul.yaml
index 3590d56..f070ab8 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -2,22 +2,13 @@
      check:
        jobs:
          - validate
-         - compose-base
 
  - job:
      name: validate
-     description: Validate manifests and scripts syntax
-     run: ci/validate.yaml
-     nodeset:
-       nodes:
-         - name: container
-           label: zuul-worker-f39
-
- - job:
-     name: compose-base
-     description: Test composing the base variant
+     description: Validate manifests, scripts, dependencies and compose the base image
+     # Increased timeout as builds can take a while
      timeout: 3600
-     run: ci/compose-base.yaml
+     run: ci/validate.yaml
      nodeset:
        nodes:
          - name: vm
diff --git a/ci/compose-base.yaml b/ci/compose-base.yaml
deleted file mode 100644
index a5017b0..0000000
--- a/ci/compose-base.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
- - hosts: all
-   tasks:
-     - name: Install dependencies
-       package:
-         name:
-           - 'jq'
-           - 'just'
-           - 'ostree'
-           - 'python3-pyyaml'
-           - 'rpm-ostree'
-           - 'selinux-policy-targeted'
-         state: present
-       become: yes
-     - name: Compose the base variant
-       ansible.builtin.command:
-         chdir: "{{ zuul.project.src_dir }}"
-         cmd: just compose-legacy base
diff --git a/ci/validate.yaml b/ci/validate.yaml
index 038ecdb..928cbc2 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -3,10 +3,12 @@
      - name: Install dependencies
        package:
          name:
+           - 'jq'
            - 'just'
            - 'ostree'
            - 'python3-pyyaml'
            - 'rpm-ostree'
+           - 'selinux-policy-targeted'
          state: present
        become: yes
      - name: Validate manifests and scripts syntax
@@ -17,3 +19,7 @@
        ansible.builtin.command:
          chdir: "{{ zuul.project.src_dir }}"
          cmd: just compose-dry-run
+     - name: Compose the base variant
+       ansible.builtin.command:
+         chdir: "{{ zuul.project.src_dir }}"
+         cmd: just compose-legacy base

From e99c078bf707a140744ccdb11d48fd2e638bf7e3 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 01 2024 19:43:59 +0000
Subject: [PATCH 7/12] justfile: Run compose-dry-run for a single variant


We'll run it for all official variants in the Ansible playbook used for
the Zuul CI instead.

---

diff --git a/justfile b/justfile
index 4fac76c..6716f78 100644
--- a/justfile
+++ b/justfile
@@ -81,8 +81,8 @@ manifest variant=default_variant:
 
     rpm-ostree compose tree --print-only --repo=repo fedora-{{variant}}.yaml
 
-# Perform dependency resolution for all official variants
-compose-dry-run:
+# Perform dependency resolution for a given variant (defaults to Silverblue)
+compose-dry-run variant=default_variant:
     #!/bin/bash
     set -euxo pipefail
 
@@ -93,9 +93,7 @@ compose-dry-run:
         popd > /dev/null || exit 1
     fi
 
-    for v in "silverblue" "kinoite" "sericea" "onyx"; do
-        rpm-ostree compose tree --unified-core --repo=repo --dry-run "fedora-${v}.yaml"
-    done
+    rpm-ostree compose tree --unified-core --repo=repo --dry-run fedora-{{variant}}.yaml
 
 # Alias/shortcut for compose-image command
 compose variant=default_variant: (compose-image variant)

From 730349607bcce76c0caac0e2e11003702d1eb121 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 01 2024 19:44:27 +0000
Subject: [PATCH 8/12] zuul: Fix YAML indentation


---

diff --git a/.zuul.yaml b/.zuul.yaml
index f070ab8..a271c33 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,15 +1,15 @@
- - project:
-     check:
-       jobs:
-         - validate
+- project:
+    check:
+      jobs:
+        - validate
 
- - job:
-     name: validate
-     description: Validate manifests, scripts, dependencies and compose the base image
-     # Increased timeout as builds can take a while
-     timeout: 3600
-     run: ci/validate.yaml
-     nodeset:
-       nodes:
-         - name: vm
-           label: cloud-fedora-39-medium
+- job:
+    name: validate
+    description: Validate manifests, scripts, dependencies and compose the base image
+    # Increased timeout as builds can take a while
+    timeout: 3600
+    run: ci/validate.yaml
+    nodeset:
+      nodes:
+        - name: vm
+          label: cloud-fedora-39-medium
diff --git a/ci/validate.yaml b/ci/validate.yaml
index 928cbc2..691695a 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -1,25 +1,25 @@
- - hosts: all
-   tasks:
-     - name: Install dependencies
-       package:
-         name:
-           - 'jq'
-           - 'just'
-           - 'ostree'
-           - 'python3-pyyaml'
-           - 'rpm-ostree'
-           - 'selinux-policy-targeted'
-         state: present
-       become: yes
-     - name: Validate manifests and scripts syntax
-       ansible.builtin.command:
-         chdir: "{{ zuul.project.src_dir }}"
-         cmd: just validate
-     - name: Perform dependency resolution for all variants
-       ansible.builtin.command:
-         chdir: "{{ zuul.project.src_dir }}"
-         cmd: just compose-dry-run
-     - name: Compose the base variant
-       ansible.builtin.command:
-         chdir: "{{ zuul.project.src_dir }}"
-         cmd: just compose-legacy base
+- hosts: all
+  tasks:
+    - name: Install dependencies
+      package:
+        name:
+          - 'jq'
+          - 'just'
+          - 'ostree'
+          - 'python3-pyyaml'
+          - 'rpm-ostree'
+          - 'selinux-policy-targeted'
+        state: present
+      become: yes
+    - name: Validate manifests and scripts syntax
+      ansible.builtin.command:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: just validate
+    - name: Perform dependency resolution for all variants
+      ansible.builtin.command:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: just compose-dry-run
+    - name: Compose the base variant
+      ansible.builtin.command:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: just compose-legacy base

From 5cce0272face41fd68bf5b42468adc00cb48dfcd Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 01 2024 19:44:39 +0000
Subject: [PATCH 9/12] ci/validate: Also validate Zuul YAML config


---

diff --git a/ci/validate b/ci/validate
index 23a213c..d9b90af 100755
--- a/ci/validate
+++ b/ci/validate
@@ -22,17 +22,16 @@ def validate_shell(rootfd, name):
 
 for root, dirs, files, rootfd in os.fwalk('.'):
     # Skip .git, repo, cache, tmp, logs, fedora-comps
-    for d in ['.git', 'repo', 'cache', 'tmp', 'logs', 'fedora-comps', 'ci']:
+    for d in ['.git', 'repo', 'cache', 'tmp', 'logs', 'fedora-comps']:
         if d in dirs:
             dirs.remove(d)
     for name in files:
-        if name == ".zuul.yaml":
-            continue
         if name.endswith(('.yaml', '.yml')):
             print("Validating:", name)
             with open(os.open(name, dir_fd=rootfd, flags=os.O_RDONLY)) as f:
                 yaml.safe_load(f)
-            result = subprocess.run(['grep', '-RniEv', '^(  )*[a-z#/-]|^$|^#', name], encoding='UTF-8')
+            result = subprocess.run(['grep', '-RniEv', '^(  )*[a-z#/-]|^$|^#', name], encoding='UTF-8',
+                                    preexec_fn=lambda: os.fchdir(rootfd))
             if result.returncode == 0:
                 raise Exception("Found likely invalid indentation in YAML file: {}".format(name))
             validated +=1
@@ -47,7 +46,7 @@ for root, dirs, files, rootfd in os.fwalk('.'):
         if not stbuf.st_mode & stat.S_IXUSR:
             continue
         mimetype = subprocess.check_output(['file', '-b', '--mime-type', name], encoding='UTF-8',
-                                        preexec_fn=lambda: os.fchdir(rootfd)).strip()
+                                           preexec_fn=lambda: os.fchdir(rootfd)).strip()
         if mimetype == 'text/x-shellscript':
             print("Validating:", name)
             validate_shell(rootfd, name)

From 0747e68bf8cfff4a1bc302f186cc615d9fce196f Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 01 2024 19:44:39 +0000
Subject: [PATCH 10/12] ci: Fix yaml lint


---

diff --git a/ci/validate.yaml b/ci/validate.yaml
index 691695a..122cc87 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -10,7 +10,7 @@
           - 'rpm-ostree'
           - 'selinux-policy-targeted'
         state: present
-      become: yes
+      become: true
     - name: Validate manifests and scripts syntax
       ansible.builtin.command:
         chdir: "{{ zuul.project.src_dir }}"

From b926c07cf7269fe03e8e7b8216ad385ab4579277 Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 01 2024 19:44:39 +0000
Subject: [PATCH 11/12] ci: Split dependency check and run checks all on failures


- Split the dependency check into a distinct step for each variant
- If the dependency test failed, still run the compose base test to get
  results about it
- Add a step at the end to make sure that if any step fails, the
  workflow still fail

See: https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_error_handling.html#ignoring-failed-commands
See discussion in: https://pagure.io/workstation-ostree-config/pull-request/493

---

diff --git a/ci/validate.yaml b/ci/validate.yaml
index 122cc87..b07f753 100644
--- a/ci/validate.yaml
+++ b/ci/validate.yaml
@@ -11,15 +11,46 @@
           - 'selinux-policy-targeted'
         state: present
       become: true
+
     - name: Validate manifests and scripts syntax
       ansible.builtin.command:
         chdir: "{{ zuul.project.src_dir }}"
         cmd: just validate
-    - name: Perform dependency resolution for all variants
-      ansible.builtin.command:
+
+    - name: Perform dependency resolution for Silverblue
+      ansible.builtin.shell:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: "just compose-dry-run silverblue && touch .zuulci.silverblue"
+      # Still run the next step if this one fails
+      ignore_errors: true
+
+    - name: Perform dependency resolution for Kinoite
+      ansible.builtin.shell:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: "just compose-dry-run kinoite && touch .zuulci.kinoite"
+      # Still run the next step if this one fails
+      ignore_errors: true
+
+    - name: Perform dependency resolution for Sway Atomic
+      ansible.builtin.shell:
         chdir: "{{ zuul.project.src_dir }}"
-        cmd: just compose-dry-run
+        cmd: "just compose-dry-run sericea && touch .zuulci.sericea"
+      # Still run the next step if this one fails
+      ignore_errors: true
+
+    - name: Perform dependency resolution for Budgie Atomic
+      ansible.builtin.shell:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: "just compose-dry-run onyx && touch .zuulci.onyx"
+      # Still run the next step if this one fails
+      ignore_errors: true
+
     - name: Compose the base variant
       ansible.builtin.command:
         chdir: "{{ zuul.project.src_dir }}"
         cmd: just compose-legacy base
+
+    - name: Check if any previous dependency resolution steps failed
+      ansible.builtin.shell:
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: "[[ -f .zuulci.silverblue ]] && [[ -f .zuulci.kinoite ]] && [[ -f .zuulci.sericea ]] && [[ -f .zuulci.onyx ]]"

From 5113ecdd20ac8cd994b95545aac3d62ba9580c0c Mon Sep 17 00:00:00 2001
From: Timothée Ravier <tim@siosm.fr>
Date: Mar 26 2024 10:44:54 +0000
Subject: [PATCH 12/12] justfile: Remove set -x from common interactive commands


For those commands, the -x output is mostly noise.

---

diff --git a/justfile b/justfile
index 6716f78..82a14f1 100644
--- a/justfile
+++ b/justfile
@@ -17,7 +17,7 @@ validate:
 # Comps-sync, but without pulling latest
 sync:
     #!/bin/bash
-    set -euxo pipefail
+    set -euo pipefail
 
     if [[ ! -d fedora-comps ]]; then
         git clone https://pagure.io/fedora-comps.git
@@ -30,7 +30,7 @@ sync:
 # Sync the manifests with the content of the comps groups
 comps-sync:
     #!/bin/bash
-    set -euxo pipefail
+    set -euo pipefail
 
     if [[ ! -d fedora-comps ]]; then
         git clone https://pagure.io/fedora-comps.git
@@ -48,7 +48,7 @@ comps-sync:
 # Output the processed manifest for a given variant (defaults to Silverblue)
 manifest variant=default_variant:
     #!/bin/bash
-    set -euxo pipefail
+    set -euo pipefail
 
     variant={{variant}}
     case "${variant}" in