From 3de1b275bd05f6659fc36b0b0acf704a02b5a343 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 08 2025 13:49:00 +0000 Subject: [PATCH 1/9] justfile: Remove unused commands --- diff --git a/justfile b/justfile index a5f3b47..02313af 100644 --- a/justfile +++ b/justfile @@ -229,14 +229,6 @@ clean-repo: clean-cache: rm -rf ./cache -# Run from inside a container -podman: - podman run --rm -ti --volume $PWD:/srv:rw --workdir /srv --privileged quay.io/fedora-ostree-desktops/buildroot - -# Update the container image -podman-pull: - podman pull quay.io/fedora-ostree-desktops/buildroot - # Build an ISO lorax variant=default_variant: #!/bin/bash From c3a135f0beecaf26cfb3d7542b90d99a5384ec16 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 08 2025 13:49:00 +0000 Subject: [PATCH 2/9] justfile: Rework upload, multi-arch manifest & sign --- diff --git a/justfile b/justfile index 02313af..d85ebc8 100644 --- a/justfile +++ b/justfile @@ -324,7 +324,7 @@ lorax variant=default_variant: --add-template-var=ostree_update_ref=fedora/${version}/x86_64/${variant} \ ${pwd}/iso/linux -# Upload the containers to a registry (Quay.io) +# Upload a container to a registry and sign it. Used in CI upload-container variant=default_variant arch=default_arch: #!/bin/bash set -euxo pipefail @@ -339,13 +339,25 @@ upload-container variant=default_variant arch=default_arch: exit 1 fi + if [[ "${CI}" != "true" ]]; then + echo "Skipping: Not in CI" + exit 1 + fi + if [[ -z ${REGISTRY+x} ]] || [[ -z ${RELEASE_REPO+x} ]]; then + echo "Skipping: No REGISTRY or RELEASE_REPO set" + exit 1 + fi if [[ -z ${CI_REGISTRY_USER+x} ]] || [[ -z ${CI_REGISTRY_PASSWORD+x} ]]; then - echo "Skipping container upload: Not in CI" - exit 0 + echo "Skipping: No CI_REGISTRY_USER or CI_REGISTRY_PASSWORD set" + exit 1 fi - if [[ "${CI}" != "true" ]]; then - echo "Skipping container upload: Not in CI" - exit 0 + + buildid="" + if [[ -f ".buildid" ]]; then + buildid="$(< .buildid)" + else + echo "Skipping: No '.buildid' file" + exit 1 fi version="" @@ -355,18 +367,14 @@ upload-container variant=default_variant arch=default_arch: version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')" fi - buildid="" - if [[ -f ".buildid" ]]; then - buildid="$(< .buildid)" - else - buildid="$(date '+%Y%m%d.0')" - echo "${buildid}" > .buildid - fi - # Login to the registry - skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" quay.io + skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${REGISTRY}" + + # Login to the registry again for cosign + skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" \ + --authfile="${HOME}/.docker/config.json" "${REGISTRY}" - image="quay.io/fedora-ostree-desktops/${variant}" + image="${REGISTRY}/${RELEASE_REPO}/${variant}" # Only append arch suffix if requested suffix="" @@ -377,24 +385,22 @@ upload-container variant=default_variant arch=default_arch: # Support for the zstd:chunked format is not ready yet SKOPEO_ARGS="--retry-times 3 --dest-compress-format gzip" - # Push fully versioned tag (major version, build date/id) + # Push fully versioned tag (major version, build date/id, arch) skopeo copy ${SKOPEO_ARGS} \ "oci-archive:${variant}.ociarchive" \ "docker://${image}:${version}.${buildid}${suffix}" - # Update "un-versioned" tag (only major version) - skopeo copy ${SKOPEO_ARGS} \ - "docker://${image}:${version}.${buildid}${suffix}" \ - "docker://${image}:${version}${suffix}" + # Decode private key + printenv "COSIGN_PRIVATE_KEY" > private.key.b64 + base64 --decode private.key.b64 > private.key - if [[ "${variant}" == "kinoite-nightly" ]]; then - # Update latest tag for kinoite-nightly only - skopeo copy ${SKOPEO_ARGS} \ - "docker://${image}:${version}.${buildid}" \ - "docker://${image}:latest${suffix}" - fi + # Sign images recursively + cosign sign -y --key private.key ${image}:${version}.${buildid}${suffix} -# Create a multi-arch manifest for a given variant and push it to a registry + # Cleanup private key + rm private.key.b64 private.key + +# Create a multi-arch manifest for a given variant, push it to a registry and sign it multi-arch-manifest variant=default_variant: #!/bin/bash set -euxo pipefail @@ -408,58 +414,25 @@ multi-arch-manifest variant=default_variant: exit 1 fi - if [[ -z ${CI_REGISTRY_USER+x} ]] || [[ -z ${CI_REGISTRY_PASSWORD+x} ]]; then - echo "Skipping multi-arch-manifest: Not in CI" - exit 0 - fi if [[ "${CI}" != "true" ]]; then - echo "Skipping multi-arch-manifest: Not in CI" - exit 0 - fi - - version="" - if [[ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]] || [[ -f "fedora-rawhide.repo" ]]; then - version="rawhide" - else - version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')" + echo "Skipping: Not in CI" + exit 1 fi - - # Login to the registry - skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" quay.io - - image="quay.io/fedora-ostree-desktops/${variant}" - - # Create manifest - buildah manifest create "${image}:${version}" \ - "${image}:${version}-x86_64" \ - "${image}:${version}-aarch64" - - # Push to registry - buildah manifest push \ - "${image}:${version}" \ - "docker://${image}:${version}" - -# Sign containers using cosign (sigstore) -sign variant=default_variant: - #!/bin/bash - set -euxo pipefail - - variant={{variant}} - - declare -A pretty_names={{pretty_names}} - variant_pretty=${pretty_names[$variant]-} - if [[ -z $variant_pretty ]]; then - echo "Unknown variant" + if [[ -z ${REGISTRY+x} ]] || [[ -z ${RELEASE_REPO+x} ]]; then + echo "Skipping: No REGISTRY or RELEASE_REPO set" exit 1 fi - if [[ -z ${CI_REGISTRY_USER+x} ]] || [[ -z ${CI_REGISTRY_PASSWORD+x} ]]; then - echo "Skipping artifact archiving: Not in CI" - exit 0 + echo "Skipping: No CI_REGISTRY_USER or CI_REGISTRY_PASSWORD set" + exit 1 fi - if [[ "${CI}" != "true" ]]; then - echo "Skipping artifact archiving: Not in CI" - exit 0 + + buildid="" + if [[ -f ".buildid" ]]; then + buildid="$(< .buildid)" + else + echo "Skipping: No '.buildid' file" + exit 1 fi version="" @@ -470,19 +443,47 @@ sign variant=default_variant: fi # Login to the registry - skopeo login \ - --username "${CI_REGISTRY_USER}" \ - --password "${CI_REGISTRY_PASSWORD}" \ - --authfile="${HOME}/.docker/config.json" \ - quay.io + skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${REGISTRY}" + + # Login to the registry again for cosign + skopeo login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" \ + --authfile="${HOME}/.docker/config.json" "${REGISTRY}" + + image="${REGISTRY}/${RELEASE_REPO}/${variant}" + + # Create manifest with full version tags + buildah manifest create "${image}:${version}.${buildid}" \ + "${image}:${version}.${buildid}-x86_64" \ + "${image}:${version}.${buildid}-aarch64" # Decode private key printenv "COSIGN_PRIVATE_KEY" > private.key.b64 base64 --decode private.key.b64 > private.key - # Sign images recursively - image="quay.io/fedora-ostree-desktops/${variant}" - cosign sign -y --key private.key --recursive=true ${image}:${version} + # Push fully versioned dual arch manifest tag (major version, build date/id) + buildah manifest push \ + "${image}:${version}.${buildid}" \ + "docker://${image}:${version}.${buildid}" + + # Sign manifest + cosign sign -y --key private.key ${image}:${version}.${buildid} + + # Update "un-versioned" tag (only major version) + buildah manifest push \ + "${image}:${version}.${buildid}" \ + "docker://${image}:${version}" + + # Sign manifest + cosign sign -y --key private.key ${image}:${version} + + if [[ "${variant}" == "kinoite-nightly" ]]; then + # Update latest tag for kinoite-nightly only + buildah manifest push \ + "${image}:${version}.${buildid}" \ + "docker://${image}:latest" + # Sign manifest + cosign sign -y --key private.key ${image}:latest + fi # Cleanup private key rm private.key.b64 private.key From abde0e53333532e46ea5a71294384817d44505e9 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 09 2025 14:01:09 +0000 Subject: [PATCH 3/9] justfile: Use arrays for command arguments --- diff --git a/justfile b/justfile index d85ebc8..d1a040c 100644 --- a/justfile +++ b/justfile @@ -149,17 +149,20 @@ compose-legacy variant=default_variant: version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')" echo "Composing ${variant_pretty} ${version}.${buildid} ..." - ARGS="--repo=repo --cachedir=cache" - ARGS+=" --unified-core" + ARGS=( + "--repo=repo" + "--cachedir=cache" + "--unified-core" + ) if [[ {{force_nocache}} == "true" ]]; then - ARGS+=" --force-nocache" + ARGS+=(" --force-nocache") fi CMD="rpm-ostree" if [[ ${EUID} -ne 0 ]]; then CMD="sudo rpm-ostree" fi - ${CMD} compose tree ${ARGS} \ + ${CMD} compose tree "${ARGS[@]}" \ --add-metadata-string="version=${variant_pretty} ${version}.${buildid}" \ "${variant}-ostree.yaml" \ |& tee "logs/${variant}_${version}_${buildid}.${timestamp}.log" @@ -201,9 +204,13 @@ compose-image variant=default_variant: version="$(rpm-ostree compose tree --print-only --repo=repo ${variant}.yaml | jq -r '."mutate-os-release"')" echo "Composing ${variant_pretty} ${version}.${buildid} ..." - ARGS="--cachedir=cache --initialize" + ARGS=( + "--cachedir=cache" + "--initialize" + "--label=quay.expires-after=4w" + ) if [[ {{force_nocache}} == "true" ]]; then - ARGS+=" --force-nocache" + ARGS+=("--force-nocache") fi # To debug with gdb, use: gdb --args ... CMD="rpm-ostree" @@ -211,8 +218,7 @@ compose-image variant=default_variant: CMD="sudo rpm-ostree" fi - ${CMD} compose image ${ARGS} \ - --label="quay.expires-after=4w" \ + ${CMD} compose image "${ARGS[@]}" \ "${variant}.yaml" \ "${variant}.ociarchive" @@ -382,11 +388,16 @@ upload-container variant=default_variant arch=default_arch: suffix="-${arch}" fi + SKOPEO_ARGS=( + "--retry-times" "3" + ) + # Support for the zstd:chunked format is not ready yet - SKOPEO_ARGS="--retry-times 3 --dest-compress-format gzip" + SKOPEO_ARGS+=("--dest-compress-format") + SKOPEO_ARGS+=("gzip") # Push fully versioned tag (major version, build date/id, arch) - skopeo copy ${SKOPEO_ARGS} \ + skopeo copy "${SKOPEO_ARGS[@]}" \ "oci-archive:${variant}.ociarchive" \ "docker://${image}:${version}.${buildid}${suffix}" From e6707cea2862cf2cebd29b299b9097d262ec4b56 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 09 2025 14:01:09 +0000 Subject: [PATCH 4/9] justfile: Use 96 max layers for container images --- diff --git a/justfile b/justfile index d1a040c..56e32cb 100644 --- a/justfile +++ b/justfile @@ -208,6 +208,7 @@ compose-image variant=default_variant: "--cachedir=cache" "--initialize" "--label=quay.expires-after=4w" + "--max-layers=96" ) if [[ {{force_nocache}} == "true" ]]; then ARGS+=("--force-nocache") From f1bc3f0d2b9a1c9cb7c1fa7d6f48198197ec5fad Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 09 2025 14:01:09 +0000 Subject: [PATCH 5/9] justfile: Compress using zstd for F43+ container images --- diff --git a/justfile b/justfile index 56e32cb..75ba916 100644 --- a/justfile +++ b/justfile @@ -395,7 +395,11 @@ upload-container variant=default_variant arch=default_arch: # Support for the zstd:chunked format is not ready yet SKOPEO_ARGS+=("--dest-compress-format") - SKOPEO_ARGS+=("gzip") + if [[ ${version} == "rawhide" ]] || [[ ${version} == "43" ]]; then + SKOPEO_ARGS+=("zstd") + else + SKOPEO_ARGS+=("gzip") + fi # Push fully versioned tag (major version, build date/id, arch) skopeo copy "${SKOPEO_ARGS[@]}" \ From 07ce595e9257c38f5caefe3f21217e3d6addb790 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Sep 09 2025 14:01:09 +0000 Subject: [PATCH 6/9] justfile: Add retry logic to all skopeo/buildah commands Retry those commands 5 times with a 1 min interval between each try. Those are the most frequent source of pipeline failures due to quay.io infra flakes. --- diff --git a/justfile b/justfile index 75ba916..4c60541 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,28 @@ volume_id_substitutions := '( [cosmic-atomic]="CSMCA" )' +# Define a retry function for use in recipes +retry_function := ' +retry() { + if [[ "${#}" -lt 3 ]]; then + echo "retry usage: