From 3cf711f5045e68ecc66771daddf148ecf6edf2a2 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 19 2021 09:35:21 +0000 Subject: [PATCH 1/5] Faster srpm pairing with rpms Instead of parsing huge number of RPMs to perform the calculation, use the DNF repodata. This actually also fixes an issue where we removed the source RPM for each RPM, even when **not all** the binary RPMs from the given build were subject of removal. This use-case is now tested. Fix: #7 --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index 8477fb2..2e0f734 100755 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -8,6 +8,8 @@ import time import shutil import logging +from prunerepo.pair_srpm_rpm import RPMToSRPMPairs + log = logging.getLogger(__name__) @@ -60,20 +62,6 @@ def get_rpms(repoquery_cmd, path, dry_run): return abs_rpms_paths -def get_srpm(rpm, get_all_packages_cmd, dry_run): - """ - Get matching srpm in the same directory as given rpm (described by its absolute path) - """ - get_srpm_cmd = get_all_packages_cmd + ["--srpm", os.path.splitext(os.path.basename(rpm))[0]] - output = run_cmd(get_srpm_cmd, dry_run) - if not output: - return - - srpm_name = os.path.basename(output[0]) - srpm_path = os.path.abspath(os.path.join(os.path.dirname(rpm), srpm_name)) - return srpm_path - - def prune_packages(path, days, log_level, dry_run): """ Remove obsoleted packages @@ -178,13 +166,22 @@ def get_rpms_to_remove(directory, log_level='INFO', days=0): if not latest_rpms: return [] all_rpms = get_rpms(get_all_packages_cmd, directory, dry_run=False) + + repodir = os.path.abspath(directory) + pair_lookup = RPMToSRPMPairs(repodir, log) + to_remove_rpms = set(all_rpms) - set(latest_rpms) rpm_list = [] for rpm in to_remove_rpms: log.debug("Checking age of the '%s' file" % os.path.split(rpm)[1]) - if time.time() - get_package_build_time(rpm, dry_run=False) > days * 24 * 3600: - srpm = get_srpm(rpm, get_all_packages_cmd, dry_run=False) - if srpm: - rpm_list.append(srpm) - rpm_list.append(rpm) + if time.time() - get_package_build_time(rpm, dry_run=False) < days * 24 * 3600: + continue + rpm_list.append(rpm) + + rel_rpm = os.path.normpath(os.path.relpath(rpm, repodir)) + rel_srpm = pair_lookup.srpm_to_be_removed_for_rpm(rel_rpm) + if not rel_srpm: + continue + rpm_list.append(os.path.join(repodir, rel_srpm)) + return rpm_list diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py new file mode 100644 index 0000000..d34e1a6 --- /dev/null +++ b/prunerepo/pair_srpm_rpm.py @@ -0,0 +1,141 @@ +#! /usr/bin/python3 + +""" +When source RPM and the corresponding binary RPMs are put in the same result +directory, this script is able to parse the repository metadata and pair the +existing source RPM with set of binary RPMs. Per discussion in: +https://pagure.io/prunerepo/issue/7 +https://github.com/praiskup/dnf-hacks/blob/main/find-srpm-to-rpm-pairs.py +""" + +import re +import os + +import dnf + + +def url_to_repoid(repo_url): + """ + Taken from: https://pagure.io/copr/copr/blob/\ + 0ea325a9249fd4570e2d380f30432ff8f90290e8/f/\ + frontend/coprs_frontend/coprs/helpers.py#_477-482 + """ + repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) + repo_url = re.sub("(__*)", '_', repo_url) + repo_url = re.sub("(_*$)|^_*", '', repo_url) + return repo_url + + +def get_initialized_dnf(repo, log): + """ + Return DNF Base object pre-configured to work with the given REPO. + """ + base = dnf.Base() + base.read_all_repos() + + # disable all pre-configured repos + base.repos.get_matching('*').disable() + + found = 0 + reponame = url_to_repoid(repo) + log.debug("Using repo ID %s => %s", reponame, repo) + found += 1 + base.repos.add_new_repo( + reponame, + base.conf, + baseurl=(repo,), + metadata_expire=1, + module_hotfixes=1, + ) + + # read the metadata + base.fill_sack() + return base + + +def get_mapping(repo, log): + """ + Read the repository metadata and find what RPMs were built from which SRPMs, + and map the source RPM name to SRPMs and vice versa. + """ + # query the metadata + base = get_initialized_dnf(repo, log) + query = base.sack.query() + remote = query.filter(reponame__neq="@System") + available_packages = list(remote) + + found_srpms = set() + map_srpm_to_rpms = {} + map_rpm_to_srpm = {} + + # list all packages + for package in available_packages: + if package.sourcerpm: + # handling source RPMs only for now + continue + + # handling source RPM + found_srpms.add(os.path.normpath(package.relativepath)) + + # group the binary RPMs + for package in available_packages: + if not package.sourcerpm: + continue # only binary RPMs now.. + + dirname = os.path.dirname(os.path.normpath(package.relativepath)) + expected_source_rpm = os.path.normpath( + os.path.join(dirname, package.sourcerpm)) + + if not expected_source_rpm: + log.error("%s has no SRPM header", package.relativepath) + continue + + if expected_source_rpm not in found_srpms: + log.error("%s has no source RPM in the directory", + package.relativepath) + continue + + if not expected_source_rpm in map_srpm_to_rpms: + map_srpm_to_rpms[expected_source_rpm] = set() + + rpm = os.path.normpath(package.relativepath) + map_srpm_to_rpms[expected_source_rpm].add(rpm) + map_rpm_to_srpm[rpm] = expected_source_rpm + + return map_srpm_to_rpms, map_rpm_to_srpm + + +class RPMToSRPMPairs: # pylint: disable=too-few-public-methods + """ Search/query MAP of SRPM => RPMS and RPM => SRPM objects. """ + + def __init__(self, repo, log): + self.log = log + self.repo = repo + self.srpm_map, self.rpm_map = get_mapping(repo, log) + + def srpm_to_be_removed_for_rpm(self, rpm): + """ + Detect if we should remove also the SRPM for the given RPM. We don't + remove SRPM as long as there's at least one binary RPM (sub-package) + generated from SRPM. So, when we aim to remove an RPM, we call this + method to check if it is the last RPM generated from the corresponding + SRPM, and if yes - we return the relative path of SRPM. So when we + return string (not None), the SRPM should be removed. + """ + rpm = os.path.normpath(rpm) + if rpm not in self.rpm_map: + return None + + srpm = self.rpm_map[rpm] + self.log.debug("removing %f from set of RPMs in %s", rpm, srpm) + + # Drop the reference to RPM, as it is being removed. + self.srpm_map[srpm].remove(rpm) + + if self.srpm_map[srpm]: + # some RPM(s) still exist for the SRPM, we can't remove the SRPM + return None + + # Return non-None value, to notify caller that the SRPM should be + # removed. + return srpm diff --git a/tests/keep-srpm-for-rpm/dummy-pkg-blah-2-1.fc34.x86_64.rpm b/tests/keep-srpm-for-rpm/dummy-pkg-blah-2-1.fc34.x86_64.rpm new file mode 100644 index 0000000..e7a8433 Binary files /dev/null and b/tests/keep-srpm-for-rpm/dummy-pkg-blah-2-1.fc34.x86_64.rpm differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/0cd63b6c93f0da84a7e1821b4c3cdcf4126e49fae9abdbc8cc3a24b92f8f942f-other.xml.gz b/tests/keep-srpm-for-rpm/orig-repo/repodata/0cd63b6c93f0da84a7e1821b4c3cdcf4126e49fae9abdbc8cc3a24b92f8f942f-other.xml.gz new file mode 100644 index 0000000..404af45 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/0cd63b6c93f0da84a7e1821b4c3cdcf4126e49fae9abdbc8cc3a24b92f8f942f-other.xml.gz differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/3e30d76461ec06fce6754eb804495099aeb7b8dbd6b0e1351a88e5229bcda016-primary.xml.gz b/tests/keep-srpm-for-rpm/orig-repo/repodata/3e30d76461ec06fce6754eb804495099aeb7b8dbd6b0e1351a88e5229bcda016-primary.xml.gz new file mode 100644 index 0000000..dd397a9 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/3e30d76461ec06fce6754eb804495099aeb7b8dbd6b0e1351a88e5229bcda016-primary.xml.gz differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/45a45a37a87b8af865592d996b8428ad2a2bb434468949c21fb1de1ae6c7e962-filelists.xml.gz b/tests/keep-srpm-for-rpm/orig-repo/repodata/45a45a37a87b8af865592d996b8428ad2a2bb434468949c21fb1de1ae6c7e962-filelists.xml.gz new file mode 100644 index 0000000..3d6b30f Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/45a45a37a87b8af865592d996b8428ad2a2bb434468949c21fb1de1ae6c7e962-filelists.xml.gz differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/577e6c44780ad51d9b7ca74983382e52dcddf5cb758b1f7bf6687cfb401fd460-filelists.sqlite.bz2 b/tests/keep-srpm-for-rpm/orig-repo/repodata/577e6c44780ad51d9b7ca74983382e52dcddf5cb758b1f7bf6687cfb401fd460-filelists.sqlite.bz2 new file mode 100644 index 0000000..3fdae38 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/577e6c44780ad51d9b7ca74983382e52dcddf5cb758b1f7bf6687cfb401fd460-filelists.sqlite.bz2 differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/70822375125809e33077c4947867d6042edf7852355b615d61132cd6179ff246-other.sqlite.bz2 b/tests/keep-srpm-for-rpm/orig-repo/repodata/70822375125809e33077c4947867d6042edf7852355b615d61132cd6179ff246-other.sqlite.bz2 new file mode 100644 index 0000000..6b3eddd Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/70822375125809e33077c4947867d6042edf7852355b615d61132cd6179ff246-other.sqlite.bz2 differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/edcb33b99554d8904ddd624662e94281c554dc158ee5cf39f1df286f7e7aaf7c-primary.sqlite.bz2 b/tests/keep-srpm-for-rpm/orig-repo/repodata/edcb33b99554d8904ddd624662e94281c554dc158ee5cf39f1df286f7e7aaf7c-primary.sqlite.bz2 new file mode 100644 index 0000000..250e732 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/repodata/edcb33b99554d8904ddd624662e94281c554dc158ee5cf39f1df286f7e7aaf7c-primary.sqlite.bz2 differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/repodata/repomd.xml b/tests/keep-srpm-for-rpm/orig-repo/repodata/repomd.xml new file mode 100644 index 0000000..095373a --- /dev/null +++ b/tests/keep-srpm-for-rpm/orig-repo/repodata/repomd.xml @@ -0,0 +1,55 @@ + + + 1618666027 + + 3e30d76461ec06fce6754eb804495099aeb7b8dbd6b0e1351a88e5229bcda016 + 769d614017e8f8cd1d309b6d5ac40f3bce3530f8683260872e6ab218c2d7beac + + 1618666027 + 996 + 5435 + + + 45a45a37a87b8af865592d996b8428ad2a2bb434468949c21fb1de1ae6c7e962 + c24f16585853cb90ffd08c90daa872dfc0b24824b66d2eef6412be04dd01f07f + + 1618666027 + 505 + 1320 + + + 0cd63b6c93f0da84a7e1821b4c3cdcf4126e49fae9abdbc8cc3a24b92f8f942f + 58c7ab042a1c2829f5add1f871a0e89a33cfb672b175d40812abb32686995b30 + + 1618666027 + 509 + 1545 + + + edcb33b99554d8904ddd624662e94281c554dc158ee5cf39f1df286f7e7aaf7c + ff93abc01ea8914ca509548ea53090be8ea3b83cdb069e41a697622a4507f464 + + 1618666027 + 2509 + 106496 + 10 + + + 577e6c44780ad51d9b7ca74983382e52dcddf5cb758b1f7bf6687cfb401fd460 + 0513ac62fc6a7c2010e0c4fcc746a11f4cfbdf8057127524b3ed47f963e407d9 + + 1618666027 + 1213 + 28672 + 10 + + + 70822375125809e33077c4947867d6042edf7852355b615d61132cd6179ff246 + 6f42a311d229e004ee29c510e9dff10eb1cf1541a04e73e16bef89ff9821fdf9 + + 1618666027 + 1114 + 24576 + 10 + + diff --git a/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.src.rpm b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.src.rpm new file mode 100644 index 0000000..ed0d8ff Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.src.rpm differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.x86_64.rpm b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.x86_64.rpm new file mode 100644 index 0000000..97085f9 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-1-1.fc34.x86_64.rpm differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-blah-1-1.fc34.x86_64.rpm b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-blah-1-1.fc34.x86_64.rpm new file mode 100644 index 0000000..3d8c765 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/subdir1/dummy-pkg-blah-1-1.fc34.x86_64.rpm differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.src.rpm b/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.src.rpm new file mode 100644 index 0000000..9ef10cf Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.src.rpm differ diff --git a/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.x86_64.rpm b/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.x86_64.rpm new file mode 100644 index 0000000..5190616 Binary files /dev/null and b/tests/keep-srpm-for-rpm/orig-repo/subdir2/dummy-pkg-2-1.fc34.x86_64.rpm differ diff --git a/tests/keep-srpm-for-rpm/test.sh b/tests/keep-srpm-for-rpm/test.sh new file mode 100755 index 0000000..1f0c1ab --- /dev/null +++ b/tests/keep-srpm-for-rpm/test.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# The newer build contains only one RPM (subpackage), while the old bulid +# contains two RPMs. So one RPM from the newer build obsoletes the build from +# older build, but the second RPM should stay, together with the source RPM. + +export testdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export origrepo=$testdir/orig-repo +export testrepo=$testdir/repo-test + +source $testdir/../testlib.sh + +echo "============================ test --cleancopr --days ============================"; + +setup + +persistent_files=" + subdir1/dummy-pkg-blah-1-1.fc34.x86_64.rpm + subdir1/dummy-pkg-1-1.fc34.src.rpm + subdir2/dummy-pkg-2-1.fc34.src.rpm + subdir2/dummy-pkg-2-1.fc34.x86_64.rpm +" + +removed_files=" + subdir1/dummy-pkg-1-1.fc34.x86_64.rpm +" + +for i in $persistent_files $removed_files; do run "ls $i" || die ; done +runcmd --days 0 . +for i in $persistent_files; do run "ls $i" || die ; done +for i in $removed_files; do run "ls $i" && die; done + +cd .. +setup + +cp ../dummy-pkg-blah-2-1.fc34.x86_64.rpm subdir2 +createrepo_c . + +persistent_files=" + subdir2/dummy-pkg-2-1.fc34.src.rpm + subdir2/dummy-pkg-2-1.fc34.x86_64.rpm + subdir2/dummy-pkg-blah-2-1.fc34.x86_64.rpm +" + +removed_files=" + subdir1/dummy-pkg-1-1.fc34.src.rpm + subdir1/dummy-pkg-1-1.fc34.x86_64.rpm + subdir1/dummy-pkg-blah-1-1.fc34.x86_64.rpm +" + +for i in $persistent_files $removed_files; do run "ls $i" || die ; done +runcmd --days 0 . +for i in $persistent_files; do run "ls $i" || die ; done +for i in $removed_files; do run "ls $i" && die; done + +echo success. + +exit 0 From ec57c811c9e35d97d3fa59e0926ddc85b775e448 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 19 2021 09:35:25 +0000 Subject: [PATCH 2/5] Enhance the logging mechanism a bit For Copr usecase, it is easier to not pass "log_level" down to "get_rpms_to_remove" method, but the logger directly (the one we have in copr is the one that is sending log records to Redis). Also, not all the methods needed the `dry_run` option. --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index 2e0f734..bdf7b8b 100755 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -10,14 +10,12 @@ import logging from prunerepo.pair_srpm_rpm import RPMToSRPMPairs -log = logging.getLogger(__name__) - def is_srpm(package): return package.endswith(".src.rpm") -def rm_file(path, dry_run): +def rm_file(path, dry_run, log): """ Remove file given its absolute path """ @@ -28,13 +26,13 @@ def rm_file(path, dry_run): os.remove(path) -def run_cmd(cmd, dry_run): +def run_cmd(cmd, log, dry_run=False): """ Run given command in a subprocess """ log.debug("Executing: " + ' '.join(cmd)) if dry_run: - return + return [] process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() sys.stderr.write(stderr.decode(encoding='utf-8')) @@ -43,54 +41,52 @@ def run_cmd(cmd, dry_run): return stdout.decode(encoding='utf-8').splitlines() -def get_package_build_time(package_path, dry_run): +def get_package_build_time(package_path, log): """ Get build time by reading package metadata """ query_cmd = ["/usr/bin/rpm", "-qp", "--queryformat", "%{BUILDTIME}"] + [package_path] - stdout = run_cmd(query_cmd, dry_run) + stdout = run_cmd(query_cmd, log) return int(stdout[0]) -def get_rpms(repoquery_cmd, path, dry_run): +def get_rpms(repoquery_cmd, path, log): """ Get paths to rpm packages in the repository according to given repoquery_cmd """ - stdout = run_cmd(repoquery_cmd, dry_run) # returns srpms as well + stdout = run_cmd(repoquery_cmd, log) # returns srpms as well rel_rpms_paths = [relpath for relpath in stdout if not is_srpm(relpath)] abs_rpms_paths = [os.path.abspath(os.path.join(path, relpath)) for relpath in rel_rpms_paths] return abs_rpms_paths -def prune_packages(path, days, log_level, dry_run): +def prune_packages(path, days, dry_run, log): """ Remove obsoleted packages """ - if not set_logging_level(log_level): - sys.exit(1) log.debug('Removing obsoleted packages...') - rpms = get_rpms_to_remove(path, log_level, days) + rpms = get_rpms_to_remove(path, days, log) was_deletion = False if not rpms: log.error("No RPMs available") return was_deletion for rpm in rpms: - rm_file(rpm, dry_run) + rm_file(rpm, dry_run, log) was_deletion = True return was_deletion -def recreate_repo(path, dry_run): +def recreate_repo(path, dry_run, log): """ Recreate the repository by using createrepo_c """ log.debug("Recreating repository...") createrepo_cmd = ['/usr/bin/createrepo_c', '--database', '--update', '--local-sqlite', '--cachedir', '/tmp/', '--workers', '8'] + [path] - return run_cmd(createrepo_cmd, dry_run) + return run_cmd(createrepo_cmd, log, dry_run) -def clean_copr(path, days, dry_run): +def clean_copr(path, days, dry_run, log): """ Remove whole copr build dirs if they no longer contain a srpm/rpm file """ @@ -110,41 +106,41 @@ def clean_copr(path, days, dry_run): continue log.info('Removing: ' + dir_path) - shutil.rmtree(dir_path) + if not dry_run: + shutil.rmtree(dir_path) # also remove the associated log in the main dir build_id = os.path.basename(dir_path).split('-')[0] buildlog_name = 'build-' + build_id + '.log' buildlog_path = os.path.abspath(os.path.join(path, buildlog_name)) - rm_file(os.path.join(path, buildlog_path), dry_run) + rm_file(os.path.join(path, buildlog_path), dry_run, log) -def set_logging_level(log_level): +def get_logger(log_level="INFO", module=None): """ Set logging level """ + log = logging.getLogger(module or __name__) + if log.handlers: + # repeated call + return log + handler = logging.StreamHandler(stream=sys.stderr) handler.setFormatter(logging.Formatter("%(message)s")) - try: - handler.setLevel(log_level.upper()) - log.addHandler(handler) - log.setLevel(handler.level) - return True - except ValueError as error: - print(str(error), file=sys.stderr) - except TypeError as error: - print(str(error), file=sys.stderr) - return False + handler.setLevel(log_level.upper()) + log.addHandler(handler) + log.setLevel(handler.level) + return log -def get_rpms_to_remove(directory, log_level='INFO', days=0): +def get_rpms_to_remove(directory, days=0, log=None): """ Returns a list of (s)rpm path names that should be removed. 0 days means that (s)rpm will be removed regardless of when the package was built. :param directory: local path to a yum repository - :param log_level: set logging to desired level (error, info or debug) :param days: how old are the packages to be removed, in the number of days + :param log: logger to use, if not specified an INFO stderr logger is created :return: a list of (s)RPM path names that should be removed """ get_all_packages_cmd = [ @@ -157,15 +153,19 @@ def get_rpms_to_remove(directory, log_level='INFO', days=0): "--quiet", "--setopt=skip_if_unavailable=False", ] - if not log.handlers: - if not set_logging_level(log_level): - return [] - log.info("Checking '%s' directory for removal candidates older than %s days" % (os.path.abspath(directory), days)) + + if not log: + log = get_logger() + + log.info("Checking '%s' repo for removal candidates older than %s days", + os.path.abspath(directory), days) + get_latest_packages_cmd = get_all_packages_cmd + ['--latest-limit=1'] - latest_rpms = get_rpms(get_latest_packages_cmd, directory, dry_run=False) + latest_rpms = get_rpms(get_latest_packages_cmd, directory, log) if not latest_rpms: return [] - all_rpms = get_rpms(get_all_packages_cmd, directory, dry_run=False) + + all_rpms = get_rpms(get_all_packages_cmd, directory, log) repodir = os.path.abspath(directory) pair_lookup = RPMToSRPMPairs(repodir, log) @@ -173,8 +173,8 @@ def get_rpms_to_remove(directory, log_level='INFO', days=0): to_remove_rpms = set(all_rpms) - set(latest_rpms) rpm_list = [] for rpm in to_remove_rpms: - log.debug("Checking age of the '%s' file" % os.path.split(rpm)[1]) - if time.time() - get_package_build_time(rpm, dry_run=False) < days * 24 * 3600: + log.debug("Checking age of the '%s' file", os.path.split(rpm)[1]) + if time.time() - get_package_build_time(rpm, log) < days * 24 * 3600: continue rpm_list.append(rpm) diff --git a/prunerepo/main.py b/prunerepo/main.py index 7059663..d6e0da7 100755 --- a/prunerepo/main.py +++ b/prunerepo/main.py @@ -2,7 +2,12 @@ import argparse -from prunerepo.helpers import prune_packages, recreate_repo, clean_copr +from prunerepo.helpers import ( + prune_packages, + recreate_repo, + clean_copr, + get_logger, +) def get_parser(): @@ -36,12 +41,14 @@ def get_parser(): def main(): args = get_parser() - was_deletion = prune_packages(args.path, args.days, args.log_level, args.dry_run) + log = get_logger(args.log_level) + + was_deletion = prune_packages(args.path, args.days, args.dry_run, log) if (was_deletion or args.alwayscreaterepo) and not args.nocreaterepo: - recreate_repo(args.path, args.dry_run) + recreate_repo(args.path, args.dry_run, log) if args.cleancopr: - clean_copr(args.path, args.days, args.dry_run) + clean_copr(args.path, args.days, args.dry_run, log) if __name__ == "__main__": From 57e9e64a067a5fc50f2c626acdbc8e8bb011691d Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 19 2021 09:35:25 +0000 Subject: [PATCH 3/5] Fix (or silence) the remaining PyLint warnings And remove the executable bit from non-executable scripts. --- diff --git a/prunerepo/__init__.py b/prunerepo/__init__.py index 7c17009..a46daee 100644 --- a/prunerepo/__init__.py +++ b/prunerepo/__init__.py @@ -1 +1,3 @@ +""" Prunerepo API methods """ + from .helpers import get_rpms_to_remove diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py old mode 100755 new mode 100644 index bdf7b8b..e450de9 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -1,5 +1,10 @@ #!/usr/bin/python3 +""" +Set of helper methods for the /usr/bin/prunerepo command. These methods are not +supposed to be library calls so please never import anything from this file. +""" + import subprocess import sys import os @@ -12,6 +17,7 @@ from prunerepo.pair_srpm_rpm import RPMToSRPMPairs def is_srpm(package): + """ Check if the PACKAGE string ends with src.rpm """ return package.endswith(".src.rpm") diff --git a/prunerepo/main.py b/prunerepo/main.py old mode 100755 new mode 100644 index d6e0da7..f3cf5a5 --- a/prunerepo/main.py +++ b/prunerepo/main.py @@ -1,5 +1,7 @@ #!/usr/bin/python3 +""" /usr/bin/prunerepo script """ + import argparse from prunerepo.helpers import ( @@ -10,26 +12,36 @@ from prunerepo.helpers import ( ) -def get_parser(): - parser = argparse.ArgumentParser(description='Remove old packages from rpm-md repository') - parser.add_argument('path', action='store', - help='local path to a yum repository') - parser.add_argument('--days', type=int, action='store', default=0, - help='only remove packages (and build directories when --cleancopr is used)\ - that are DAYS old or older (for packages by their build date, for directories\ - the last modification time is considered') - parser.add_argument('--cleancopr', action='store_true', - help='additionaly remove whole copr build dirs and logs if the associated package gets deleted') - parser.add_argument('--alwayscreaterepo', action='store_true', - help='Recreate repository even when there was no change in data.') - parser.add_argument('--nocreaterepo', action='store_true', - help='repository is not automatically recreated (not even after data deletion). ' - 'Supresses --alwayscreaterepo.') - parser.add_argument('--log-level', type=str, default='INFO', - help='set logging to desired level') - parser.add_argument('--dry-run', action='store_true', - help='do not remove anything from the repository and print the actions instead. ' - 'Verbose mode will be set.') +def _get_parser(): + parser = argparse.ArgumentParser( + description='Remove old packages from rpm-md repository') + parser.add_argument( + 'path', action='store', + help='local path to a yum repository') + parser.add_argument( + '--days', type=int, action='store', default=0, + help="Only remove packages (and build directories when --cleancopr is " + "used) that are DAYS old or older (for packages by their build " + "date, for directories the last modification time is considered" + ) + parser.add_argument( + '--cleancopr', action='store_true', + help="additionaly remove whole copr build dirs and " + "logs if the associated package gets deleted") + parser.add_argument( + '--alwayscreaterepo', action='store_true', + help='Recreate repository even when there was no change in data.') + parser.add_argument( + '--nocreaterepo', action='store_true', + help="repository is not automatically recreated (not even after data " + "deletion). Supresses --alwayscreaterepo.") + parser.add_argument( + '--log-level', type=str, default='INFO', + help='set logging to desired level') + parser.add_argument( + '--dry-run', action='store_true', + help='do not remove anything from the repository and print the actions instead. ' + 'Verbose mode will be set.') parser.add_argument('-v', '--version', action='version', version='1.5', help='print program version and exit') @@ -40,7 +52,8 @@ def get_parser(): def main(): - args = get_parser() + """ entrypoint """ + args = _get_parser() log = get_logger(args.log_level) was_deletion = prune_packages(args.path, args.days, args.dry_run, log) diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 From 3690cd8b078265b7e7782925df4f6f1420ee0aa8 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 19 2021 09:35:25 +0000 Subject: [PATCH 4/5] Bump release to beat the current Fedora releases --- diff --git a/prunerepo.spec b/prunerepo.spec index 4f7a884..62a8837 100644 --- a/prunerepo.spec +++ b/prunerepo.spec @@ -1,7 +1,7 @@ Name: prunerepo Version: 1.18 Summary: Remove old packages from rpm-md repository -Release: 1%{?dist} +Release: 5%{?dist} Url: https://pagure.io/prunerepo # Source is created by: From 1e1128c25c2bee56a885644a2ee84f6c67956469 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 19 2021 09:35:25 +0000 Subject: [PATCH 5/5] api: work with relative paths to the repo The absolute paths we returned is unnecessarily hard to process by the calling code that will eventually do `createrepo_c --excludes ...`. --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index e450de9..6e096f7 100644 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -77,7 +77,8 @@ def prune_packages(path, days, dry_run, log): log.error("No RPMs available") return was_deletion for rpm in rpms: - rm_file(rpm, dry_run, log) + remove = os.path.abspath(os.path.join(path, rpm)) + rm_file(remove, dry_run, log) was_deletion = True return was_deletion @@ -182,12 +183,12 @@ def get_rpms_to_remove(directory, days=0, log=None): log.debug("Checking age of the '%s' file", os.path.split(rpm)[1]) if time.time() - get_package_build_time(rpm, log) < days * 24 * 3600: continue - rpm_list.append(rpm) - rel_rpm = os.path.normpath(os.path.relpath(rpm, repodir)) + rpm_list.append(rel_rpm) + rel_srpm = pair_lookup.srpm_to_be_removed_for_rpm(rel_rpm) if not rel_srpm: continue - rpm_list.append(os.path.join(repodir, rel_srpm)) + rpm_list.append(rel_srpm) return rpm_list diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index d34e1a6..d657ca3 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -106,7 +106,11 @@ def get_mapping(repo, log): class RPMToSRPMPairs: # pylint: disable=too-few-public-methods - """ Search/query MAP of SRPM => RPMS and RPM => SRPM objects. """ + """ + Search/query MAP of SRPM => RPMS and RPM => SRPM objects. + The paths we work with, and return are relative to the "repo" directory we + get in the constructor. + """ def __init__(self, repo, log): self.log = log