From 3ef1e8dbabbf560b87695f18f6b1c8bac2813fba Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Apr 21 2022 14:12:39 +0000 Subject: [PATCH 1/2] systemd-git-update: Mirror in-progress PR branches to the repo --- diff --git a/systemd-git-update/systemd-git-update.sh b/systemd-git-update/systemd-git-update.sh index 7f779a2..64e58ca 100755 --- a/systemd-git-update/systemd-git-update.sh +++ b/systemd-git-update/systemd-git-update.sh @@ -9,10 +9,16 @@ export GIT_SSH_COMMAND="ssh -oBatchMode=yes -oControlMaster=no -o StrictHostKeyC echo 'Cloning upstream repo' git clone https://github.com/systemd/systemd.git "$workdir" -echo 'Adding Hyperscale remote' -(cd "$workdir" && git remote add hyperscale ssh://git@pagure.io/centos-sig-hyperscale/systemd.git) +echo 'Adding remotes' +git -C "$workdir" remote add hyperscale ssh://git@pagure.io/centos-sig-hyperscale/systemd.git +git -C "$workdir" remote add daandemeyer https://github.com/DaanDeMeyer/systemd.git +git -C "$workdir" fetch daandemeyer journal-compact-entry-item +git -C "$workdir" remote add plv https://github.com/plv/systemd.git +git -C "$workdir" fetch plv cg_mem_zswap_max echo 'Pushing changes to Hyperscale remote' -(cd "$workdir" && git push --tags hyperscale HEAD:main) +git -C "$workdir" push --tags hyperscale HEAD:main +git -C "$workdir" push -f hyperscale daandemeyer/journal-compact-entry-item:refs/heads/journal-compact-entry-item +git -C "$workdir" push -f hyperscale plv/cg_mem_zswap_max:refs/heads/cg_mem_zswap_max echo 'Cleaning up' exit 0 From b3e6e1a2086957883ebf49522e8572a59ddb5797 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Jun 10 2022 07:26:48 +0000 Subject: [PATCH 2/2] Fix systemd-cd script This commit updates the systemd-cd script to take into account the hyperscale systemd rpm changes. Included is a rewrite of the script from bash to python to keep things more manageable. We apply any downstream patches to the main branch before starting a build using cbs. Any upstream backports are skipped. --- diff --git a/systemd-cd/Dockerfile b/systemd-cd/Dockerfile index 3cff1fb..9012737 100644 --- a/systemd-cd/Dockerfile +++ b/systemd-cd/Dockerfile @@ -1,9 +1,9 @@ FROM quay.io/centos/centos:stream8 -# git-core and jq are for systemd-cd +# git-core and python are for systemd-cd # centos-packager is for cbs, see https://wiki.centos.org/HowTos/CentosPackager RUN dnf update -y -RUN dnf install -y epel-release git-core jq +RUN dnf install -y epel-release git-core python3 RUN dnf install -y 'dnf-command(copr)' RUN dnf copr enable -y arrfab/fasjson-client RUN dnf install -y centos-packager @@ -11,7 +11,7 @@ RUN dnf clean all ENV HOME /home/runner RUN useradd -r -g root -d $HOME -s /sbin/nologin -c "runner user" runner -COPY --chown=runner:root systemd-cd.sh /home/runner/systemd-cd.sh +COPY --chown=runner:root systemd-cd.py /home/runner/systemd-cd.py COPY --chown=runner:root runner.sh /home/runner/runner.sh RUN chmod g+w /home/runner RUN ln -s /run/secrets/hyperscalebot/centos.cert /home/runner/.centos.cert diff --git a/systemd-cd/runner.sh b/systemd-cd/runner.sh index 8d2d9ce..d7dcafc 100755 --- a/systemd-cd/runner.sh +++ b/systemd-cd/runner.sh @@ -2,10 +2,15 @@ export TZ='America/Los_Angeles' +# Always run once immediately for easy testing. +./systemd-cd.py + while true; do - sleep $(( ( $(printf 'tomorrow 10:00\nnow\n' | date -f - +%s-)0 )%86400 )) + TIME=$(( ( $(printf 'tomorrow 10:00\nnow\n' | date -f - +%s-)0 )%86400 )) + echo "Sleeping for $(date -d@$TIME -u +%H:%M:%S)" + sleep $TIME echo "Starting at $(date)" - ./systemd-cd.sh + ./systemd-cd.py echo "Done at $(date)" echo done diff --git a/systemd-cd/systemd-cd.py b/systemd-cd/systemd-cd.py new file mode 100755 index 0000000..87406e2 --- /dev/null +++ b/systemd-cd/systemd-cd.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import subprocess +import json +from datetime import datetime +from typing import Any + + +SYSTEMD_REPO = "https://pagure.io/centos-sig-hyperscale/systemd" +STABLE_REPO = "https://github.com/systemd/systemd-stable.git" +RPM_REPO = "https://git.centos.org/rpms/systemd.git" +GITHUB_ENDPOINT = "https://api.github.com/repos/systemd/systemd" +PAGURE_ENDPOINT = "https://pagure.io/api/0/centos-sig-hyperscale/systemd" +AUTHOR = "systemd-cd " +VERSION = "250.3" + + +def run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess: + return subprocess.run(*args, **kwargs, check=True, universal_newlines=True) + + +def do_build(stream: int, branch: str) -> None: + branches = run([ + "curl", + "-s", + f"{PAGURE_ENDPOINT}/git/branches?with_commits=true", + ], stdout=subprocess.PIPE).stdout + + last_commit = json.loads(branches)["branches"][branch] + short_commit = last_commit[:8] + print(f"Latest commit: {short_commit} ({last_commit})\n") + + print("Setting up packaging repo") + if Path("rpm").is_dir(): + run(["git", "reset", "--hard", "HEAD"], cwd="rpm") + run(["git", "checkout", f"c{stream}s-sig-hyperscale"], cwd="rpm") + run(["git", "pull"], cwd="rpm") + else: + run(["git", "clone", "-b", f"c{stream}s-sig-hyperscale", RPM_REPO, "rpm"]) + + print("Setting up source repo") + if Path("systemd").is_dir(): + run(["git", "reset", "--hard", "origin/main"], cwd="systemd") + run(["git", "pull"], cwd="systemd") + else: + run(["git", "clone", SYSTEMD_REPO, "systemd"]) + run(["git", "config", "user.name", "centos-sig-hyperscale"], cwd="systemd") + run(["git", "config", "user.email", "<>"], cwd="systemd") + + ### BACKPORTS BEGIN ### + + # Hyperscale hidepid commit + run(["git", "cherry-pick", "5e09b26fb1cf2599ce0dde574392890072996949", "--no-edit"], cwd="systemd") + + # Lower bpftool version requirement to 4.18.0 + run(["git", "merge", "origin/lower-bpftool-version-requirement", "--no-edit"], cwd="systemd") + + if "hs+fb" in branch: + # FB kernel version update for hidepid commit + run(["git", "cherry-pick", "5846e8617a04e3c71738079c5a0142633aaba1e7", "--no-edit"], cwd="systemd") + run(["git", "merge", "origin/cg_mem_zswap_max", "--no-edit"], cwd="systemd") + run(["git", "merge", "origin/journal-compact-entry-item", "--no-edit"], cwd="systemd") + + ### BACKPORTS END ### + + # For some reason the pagure archives are missing the 'v' prefix so we account for that. + name = f"systemd-{branch.replace('v', '')}" + print(f"Building tar file {name}.tar.gz") + run([ + "git", + "archive", + "--format=tar.gz", + "-o", f"../rpm/{name}.tar.gz", + # pagure removes the '+' from 'hs+fb' for the top level dir in the archive + # so we have to make sure we do the same here. + f"--prefix={name.replace('+', '')}/", + "HEAD", + ], cwd="systemd") + + print("Fixing up the specfile") + # the date is to ensure the release is always monotonically increasing + # e.g. 1.20190320.4bf953d + d = datetime.now() + release=f"1.{d.strftime('%Y%m%d.%H%M%S')}.{short_commit}" + version = branch.split('-')[1] + changelog=f"{d.strftime('%a %b %d %Y')} {AUTHOR} - {version}-{release}" + + run([ + "sed", + "-i", "rpm/systemd.spec", + "-e", "/^Patch:.*/d", + "-e", "/^Patch00.*/d", + "-e", "/^Patch01.*/d", + "-e", f"s/^%global hs_commit .*$/%global hs_commit {last_commit}/", + "-e", f"s/^Release:.*$/Release: {release}%{{?dist}}/", + "-e", f"/%changelog/a * {changelog}\\n- Auto build from {last_commit}\\n", + ]) + + run(["git", "--no-pager", "diff"], cwd="rpm") + print() + + print("Building src.rpm") + rpmbuild = run([ + "rpmbuild", + "--define", "_sourcedir .", + *(["--define", "%facebook 1"] if "hs+fb" in branch else []), + "-bs", "systemd.spec", + ], cwd="rpm", stdout=subprocess.PIPE) + + srcrpm = run(["awk", "/Wrote:/ { print $2 }"], input=rpmbuild.stdout, stdout=subprocess.PIPE).stdout.strip() + srcrpm = Path("rpm") / srcrpm + print(f"Wrote: {srcrpm}") + + target = "facebook" if "hs+fb" in branch else "main" + prefix = "hs+fb" if "hs+fb" in branch else "hs" + suffix = '' if stream == 8 else 's' + + run([ + "cbs", + "build", + "--wait", + "--skip-tag", + f"hyperscale{stream}s-packages-{target}-el{stream}{suffix}", + srcrpm, + ]) + run([ + "cbs", + "tag-build", + f"hyperscale{stream}s-packages-{target}-testing", + f"systemd-{version.replace('v', '')}-{release}.{prefix}.el{stream}", + ]) + + +def main() -> None: + for stream in (8, 9): + for branch in (f"hs-v{VERSION}", f"hs+fb-v{VERSION}"): + do_build(stream, branch) + + +if __name__ == "__main__": + main() diff --git a/systemd-cd/systemd-cd.sh b/systemd-cd/systemd-cd.sh deleted file mode 100755 index 8b27bdd..0000000 --- a/systemd-cd/systemd-cd.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh - -set -eu - -SYSTEMD_REPO='https://pagure.io/centos-sig-hyperscale/systemd' -GITHUB_ENDPOINT='https://api.github.com/repos/systemd/systemd' -PAGURE_ENDPOINT='https://pagure.io/api/0/centos-sig-hyperscale/systemd' -AUTHOR='systemd-cd ' - -echo "Fetching source repo information" -# TODO: move this to pagure once https://pagure.io/pagure/issue/5152 is resolved -last_tag=$(curl -s "${GITHUB_ENDPOINT}/tags" | jq -r '.[0].name') -echo "Latest tag: ${last_tag}" - -last_commit=$(curl -s "${PAGURE_ENDPOINT}/git/branches?with_commits=true" | jq -r '.branches.main') -short_commit=$(echo "$last_commit" | cut -c1-7) -echo "Latest commit: ${short_commit} (${last_commit})" -echo - -echo "Setting up packaging repo" -if [ -d systemd ]; then - (cd systemd && git checkout SPECS/systemd.spec && git pull) -else - git clone -b c8s-sig-hyperscale https://git.centos.org/rpms/systemd.git -fi -echo - -echo "Fetching tarball" -systemd_tarball="systemd/SOURCES/systemd-${short_commit}.tar.gz" -if [ ! -e "$systemd_tarball" ]; then - curl -s -o "$systemd_tarball" "${SYSTEMD_REPO}/archive/${last_commit}/systemd-${last_commit}.tar.gz" -fi -echo - -echo "Fixing up the specfile" -# v242-rc2 -> 242~rc2 -version=$(echo "${last_tag#?}" | tr '-' '~') -# the date is to ensure the release is always monotonically increasing -# e.g. 1.20190320.4bf953d -release="1.$(date +%Y%m%d).${short_commit}" -changelog="$(date '+%a %b %d %Y') ${AUTHOR} - ${version}-${release}" -sed -i "systemd/SPECS/systemd.spec" \ - -e "/^Patch00.*/d" \ - -e "/^Patch01.*/d" \ - -e "s/^#global commit .*$/%global commit ${last_commit}/" \ - -e "/^%global stable 1$/d" \ - -e "s/^Version:.*$/Version: ${version}/" \ - -e "s/^Release:.*$/Release: ${release}%{?dist}/" \ - -e "/%changelog/a * ${changelog}\\n- Auto build from ${last_commit}\\n" -(cd systemd && git --no-pager diff --no-color) -echo - -echo "Building src.rpm" -srcrpm=$(rpmbuild --define "%_topdir ${PWD}/systemd" -bs systemd/SPECS/systemd.spec | awk '/Wrote:/ { print $2 }') -echo "Wrote: $srcrpm" - -echo "Kicking off CBS builds" -cbs build --wait --skip-tag hyperscale8s-packages-main-el8 "$srcrpm" && \ - cbs tag-build hyperscale8s-packages-main-testing "systemd-${version}-${release}.hs.el8" -cbs build --wait --skip-tag hyperscale8s-packages-facebook-el8 "$srcrpm" && \ - cbs tag-build hyperscale8s-packages-facebook-testing "systemd-${version}-${release}.hs+fb.el8" - -exit 0