From 297bff1d78adb7d6da6fa4ef90676062d5864d55 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 25 2020 19:54:48 +0000 Subject: [PATCH 1/3] Adding initial revision of untag-trimmed-builds.py Signed-off-by: Merlin Mathesius --- diff --git a/untag-trimmed-builds.py b/untag-trimmed-builds.py new file mode 100755 index 0000000..27ebe51 --- /dev/null +++ b/untag-trimmed-builds.py @@ -0,0 +1,172 @@ +#!/usr/bin/python3 + +# SPDX-License-Identifier: MIT + +import click +import koji +import requests + + +DEBUG = False + + +def get_desired_packages(distro_url, distro_view, arches): + """ + Fetches the list of desired sources for 'distro-view' from 'distro-url' + for each of the given 'arches'. + + :param distro_url: top level of the content resolver + :type distro_url: str + :param distro_view: content resolver view + :type distro_view: str + :param arches: architectures to include + :type arches: tuple + :return: list of packages that are desired, merged for all 'arches' + :rtype: set + + """ + print( + "Downloading and merging desired {distro_view} sources" + " for arches: {arches}".format( + distro_view=distro_view, arches=", ".join(arches) + ) + ) + + merged_builds = set() + + for arch in arches: + url = ( + "{distro_url}" + "/view-source-package-name-list--view-{distro_view}--{arch}.txt" + ).format(distro_url=distro_url, distro_view=distro_view, arch=arch) + + if DEBUG: + print("downloading {url}".format(url=url)) + + r = requests.get(url, allow_redirects=True) + for line in r.text.splitlines(): + merged_builds.add(line) + + return merged_builds + + +def get_undesired_builds(session, koji_tag, desired_pkgs): + """ + Fetches the list of undesired builds currently tagged with + 'koji_tag'. + + :param session: Koji session + :type session: koji.ClientSession + :param koji_tag: Koji tag + :type koji_tag: str + :param desired_pkgs: list of packages to keep + :type desired_pkgs: set + :return: list of NVRs of builds that should be deleted from tag, + or None if an error occurred + :rtype: set + """ + print( + "Identifying undesirable builds in Koji tag {koji_tag}".format( + koji_tag=koji_tag + ) + ) + + tag = session.getTag(koji_tag) + if not tag: + print("No such tag {}".format(koji_tag)) + return None + + undesired_builds = set() + + # get all builds with tag + builds = session.listTagged(koji_tag) + + for binfo in builds: + pkg = binfo["package_name"] + nvr = binfo["nvr"] + + if DEBUG: + print("Found build of package {} with nvr {}".format(pkg, nvr)) + + if pkg not in desired_pkgs: + if DEBUG: + print("PACKAGE BUILD {} NEEDS TO BE REMOVED FROM TAG".format(nvr)) + + undesired_builds.add(nvr) + + return undesired_builds + + +def untag_builds(session, koji_tag, dry_run, builds): + """ + Untag given list of builds from 'koji_tag'. + + :param session: Koji session + :type session: koji.ClientSession + :param koji_tag: Koji tag + :type koji_tag: str + :param builds: list of builds to untag + :type desired_pkgs: set + """ + for nvr in sorted(builds): + if dry_run: + print("Would have untagged {}".format(nvr)) + else: + print("Untagging {}".format(nvr)) + session.untagBuild(koji_tag, nvr) + + +@click.command() +@click.option("--debug", + is_flag=True, + help="Output a lot of debugging information", + show_default=True, + default=False) +@click.option("--dry-run", + is_flag=True, + help="Do a trial run without making any changes", + show_default=True, + default=False) +@click.option("--koji-url", + help="The root of the Koji XMLRPC API", + show_default=True, + default="https://koji.fedoraproject.org/kojihub") +@click.option("--koji-tag", + help="The Koji tag to trim", + show_default=True, + default="eln") +@click.option("--distro-url", + help="The top level of the content resolver", + show_default=True, + default="https://tiny.distro.builders") +@click.option("--distro-view", + help="The content resolver view", + show_default=True, + default="eln-and-buildroot") +@click.option("--arches", "--arch", + multiple=True, + help="The architectures to include", + show_default=True, + default=["aarch64", "armv7hl", "ppc64le", "s390x", "x86_64"]) +def cli(dry_run, debug, koji_url, koji_tag, distro_url, distro_view, arches): + """ + Automated removal of packages from koji tag that have been trimmed from + distribution. + """ + global DEBUG + DEBUG = debug + + desired_pkgs = get_desired_packages(distro_url, distro_view, arches) + + session = koji.ClientSession(koji_url) + + builds_to_untag = get_undesired_builds(session, koji_tag, desired_pkgs) + + if DEBUG: + print("Builds to untag: {}".format(builds_to_untag)) + + untag_builds(session, koji_tag, dry_run, builds_to_untag) + + +if __name__ == "__main__": + cli() From 636ada1eec46ef7331bb10eebcf6f6ed2d637783 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 25 2020 22:02:24 +0000 Subject: [PATCH 2/3] Improvements: - Authenticate Koji session - Use Koji session multicall operation for efficiency - If number of builds to be untagged exceeds a pre-set threshold (50%), require a --force command line option Signed-off-by: Merlin Mathesius --- diff --git a/untag-trimmed-builds.py b/untag-trimmed-builds.py index 27ebe51..f075206 100755 --- a/untag-trimmed-builds.py +++ b/untag-trimmed-builds.py @@ -8,6 +8,10 @@ import requests DEBUG = False +# to prevent an unfortunate situation, if the ratio of undesired builds to +# the current total number of builds in the given tag exceeds FORCE_THRESHOLD, +# then the --force option must be supplied to complete the untagging operation +FORCE_THRESHOLD = 0.5 def get_desired_packages(distro_url, distro_view, arches): @@ -50,7 +54,7 @@ def get_desired_packages(distro_url, distro_view, arches): return merged_builds -def get_undesired_builds(session, koji_tag, desired_pkgs): +def get_undesired_builds(session, koji_tag, desired_pkgs, force): """ Fetches the list of undesired builds currently tagged with 'koji_tag'. @@ -80,6 +84,7 @@ def get_undesired_builds(session, koji_tag, desired_pkgs): # get all builds with tag builds = session.listTagged(koji_tag) + num_current_builds = len(builds) for binfo in builds: pkg = binfo["package_name"] @@ -94,6 +99,29 @@ def get_undesired_builds(session, koji_tag, desired_pkgs): undesired_builds.add(nvr) + num_undesired_builds = len(undesired_builds) + undesired_ratio = num_undesired_builds / num_current_builds + + print( + "Koji tag {} currently has {} builds, {} ({:.2%}) of which are undesired".format( + koji_tag, + num_current_builds, + num_undesired_builds, + undesired_ratio + ) + ) + + if undesired_ratio > FORCE_THRESHOLD: + print( + "WARNING: Undesired build ratio is above safety threshold" + " ({:.2%})".format(FORCE_THRESHOLD) + ) + if force: + print("--force option has been set. Proceeding.") + else: + print("Use --force option if you wish to proceed despite this warning.") + return None + return undesired_builds @@ -108,12 +136,13 @@ def untag_builds(session, koji_tag, dry_run, builds): :param builds: list of builds to untag :type desired_pkgs: set """ - for nvr in sorted(builds): - if dry_run: - print("Would have untagged {}".format(nvr)) - else: - print("Untagging {}".format(nvr)) - session.untagBuild(koji_tag, nvr) + with session.multicall() as m: + for nvr in sorted(builds): + if dry_run: + print("Would have untagged {}".format(nvr)) + else: + print("Untagging {}".format(nvr)) + m.untagBuild(koji_tag, nvr) @click.command() @@ -148,7 +177,14 @@ def untag_builds(session, koji_tag, dry_run, builds): help="The architectures to include", show_default=True, default=["aarch64", "armv7hl", "ppc64le", "s390x", "x86_64"]) -def cli(dry_run, debug, koji_url, koji_tag, distro_url, distro_view, arches): +@click.option("--force", + is_flag=True, + help=( + "Force untagging even if removal ratio exceeds" + " threshold ({:.2%})").format(FORCE_THRESHOLD), + show_default=True, + default=False) +def cli(dry_run, debug, koji_url, koji_tag, distro_url, distro_view, arches, force): """ Automated removal of packages from koji tag that have been trimmed from distribution. @@ -156,11 +192,24 @@ def cli(dry_run, debug, koji_url, koji_tag, distro_url, distro_view, arches): global DEBUG DEBUG = debug - desired_pkgs = get_desired_packages(distro_url, distro_view, arches) - session = koji.ClientSession(koji_url) + try: + session.gssapi_login() + except: + print("ERROR: an authentication has occurred") + if not session.logged_in: + print( + "Unable to log in to Koji." + " Did you forget to run 'kinit fasname@FEDORAPROJECT.ORG'?" + ) + return + + desired_pkgs = get_desired_packages(distro_url, distro_view, arches) - builds_to_untag = get_undesired_builds(session, koji_tag, desired_pkgs) + builds_to_untag = get_undesired_builds(session, koji_tag, desired_pkgs, force) + if not builds_to_untag: + print("No builds to untag") + return if DEBUG: print("Builds to untag: {}".format(builds_to_untag)) From 714fb94cfe038d42ba02973fa01e610cff4c7305 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 26 2020 18:58:17 +0000 Subject: [PATCH 3/3] Address review feedback and improvement - Use Python's "logging" facility for debug output - Split get_distro_packages() into a separate library/script Signed-off-by: Merlin Mathesius --- diff --git a/get_distro_packages.py b/get_distro_packages.py new file mode 100755 index 0000000..f072911 --- /dev/null +++ b/get_distro_packages.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: MIT + +import logging +import os +import requests +import sys + + +def get_distro_packages(distro_url, distro_view, arches, logger=None): + """ + Fetches the list of desired sources for 'distro-view' from 'distro-url' + for each of the given 'arches'. + + :param distro_url: top level of the content resolver + :type distro_url: str + :param distro_view: content resolver view + :type distro_view: str + :param arches: architectures to include + :type arches: iterable + :param logger: logger instance for debug output + :type logger: logging.Logger + :return: list of packages that are desired, merged for all 'arches' + :rtype: set + + """ + merged_packages = set() + + for arch in arches: + url = ( + "{distro_url}" + "/view-source-package-name-list--view-{distro_view}--{arch}.txt" + ).format(distro_url=distro_url, distro_view=distro_view, arch=arch) + + if logger: + logger.debug("downloading {url}".format(url=url)) + + r = requests.get(url, allow_redirects=True) + for line in r.text.splitlines(): + merged_packages.add(line) + + if logger: + logger.debug("Found a total of {} packages".format(len(merged_packages))) + + return merged_packages + + +if __name__ == "__main__": + logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s:%(message)s") + logger = logging.getLogger(os.path.basename(__file__)) + logger.setLevel(logging.DEBUG) + logger.debug("Debugging mode enabled") + + distro_url = "https://tiny.distro.builders" + distro_view = "eln-and-buildroot" + arches = ["aarch64", "armv7hl", "ppc64le", "s390x", "x86_64"] + + if len(sys.argv) > 1: + distro_url = sys.argv[1] + if len(sys.argv) > 2: + distro_view = sys.argv[2] + if len(sys.argv) > 3: + arches = sys.argv[3].split() + + pkgs = get_distro_packages(distro_url, distro_view, arches, logger) + + for pkg in sorted(pkgs): + print(pkg) diff --git a/untag-trimmed-builds.py b/untag-trimmed-builds.py index f075206..e825945 100755 --- a/untag-trimmed-builds.py +++ b/untag-trimmed-builds.py @@ -4,56 +4,22 @@ import click import koji -import requests +import logging +import os +import sys + +from get_distro_packages import get_distro_packages + + +logger = logging.getLogger(os.path.basename(__file__)) -DEBUG = False # to prevent an unfortunate situation, if the ratio of undesired builds to # the current total number of builds in the given tag exceeds FORCE_THRESHOLD, # then the --force option must be supplied to complete the untagging operation FORCE_THRESHOLD = 0.5 -def get_desired_packages(distro_url, distro_view, arches): - """ - Fetches the list of desired sources for 'distro-view' from 'distro-url' - for each of the given 'arches'. - - :param distro_url: top level of the content resolver - :type distro_url: str - :param distro_view: content resolver view - :type distro_view: str - :param arches: architectures to include - :type arches: tuple - :return: list of packages that are desired, merged for all 'arches' - :rtype: set - - """ - print( - "Downloading and merging desired {distro_view} sources" - " for arches: {arches}".format( - distro_view=distro_view, arches=", ".join(arches) - ) - ) - - merged_builds = set() - - for arch in arches: - url = ( - "{distro_url}" - "/view-source-package-name-list--view-{distro_view}--{arch}.txt" - ).format(distro_url=distro_url, distro_view=distro_view, arch=arch) - - if DEBUG: - print("downloading {url}".format(url=url)) - - r = requests.get(url, allow_redirects=True) - for line in r.text.splitlines(): - merged_builds.add(line) - - return merged_builds - - def get_undesired_builds(session, koji_tag, desired_pkgs, force): """ Fetches the list of undesired builds currently tagged with @@ -86,16 +52,18 @@ def get_undesired_builds(session, koji_tag, desired_pkgs, force): builds = session.listTagged(koji_tag) num_current_builds = len(builds) + # if no current builds, none are undesired so return empty set + if num_current_builds == 0: + return undesired_builds + for binfo in builds: pkg = binfo["package_name"] nvr = binfo["nvr"] - if DEBUG: - print("Found build of package {} with nvr {}".format(pkg, nvr)) + logger.debug("Found build of package {} with nvr {}".format(pkg, nvr)) if pkg not in desired_pkgs: - if DEBUG: - print("PACKAGE BUILD {} NEEDS TO BE REMOVED FROM TAG".format(nvr)) + logger.debug("PACKAGE BUILD {} NEEDS TO BE REMOVED FROM TAG".format(nvr)) undesired_builds.add(nvr) @@ -119,7 +87,10 @@ def get_undesired_builds(session, koji_tag, desired_pkgs, force): if force: print("--force option has been set. Proceeding.") else: - print("Use --force option if you wish to proceed despite this warning.") + print( + "Use --force option if you wish to proceed despite this warning.", + file=sys.stderr + ) return None return undesired_builds @@ -189,30 +160,40 @@ def cli(dry_run, debug, koji_url, koji_tag, distro_url, distro_view, arches, for Automated removal of packages from koji tag that have been trimmed from distribution. """ - global DEBUG - DEBUG = debug + if debug: + logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s:%(message)s") + logger.setLevel(logging.DEBUG) + logger.debug("Debugging mode enabled") + else: + logging.basicConfig() session = koji.ClientSession(koji_url) try: session.gssapi_login() except: - print("ERROR: an authentication has occurred") + print("ERROR: an authentication error has occurred", file=sys.stderr) if not session.logged_in: print( "Unable to log in to Koji." - " Did you forget to run 'kinit fasname@FEDORAPROJECT.ORG'?" + " Did you forget to run 'kinit fasname@FEDORAPROJECT.ORG'?", + file=sys.stderr ) return - desired_pkgs = get_desired_packages(distro_url, distro_view, arches) + print( + "Downloading and merging desired {distro_view} sources" + " for arches: {arches}".format( + distro_view=distro_view, arches=", ".join(arches) + ) + ) + desired_pkgs = get_distro_packages(distro_url, distro_view, arches, logger=logger) builds_to_untag = get_undesired_builds(session, koji_tag, desired_pkgs, force) if not builds_to_untag: print("No builds to untag") return - if DEBUG: - print("Builds to untag: {}".format(builds_to_untag)) + logger.debug("Builds to untag: {}".format(builds_to_untag)) untag_builds(session, koji_tag, dry_run, builds_to_untag)