From 37d4ae443ac690f936446f950455394769bf5d0c Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Jun 06 2023 15:36:42 +0000 Subject: [PATCH 1/2] Read BUILDTIME from the metadata, rather than the RPM file This reduces the I/O requirements a lot, per: https://github.com/fedora-copr/copr/issues/2750 --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index 53e4f82..c10f1c3 100644 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -14,7 +14,7 @@ import shutil import logging import tempfile -from prunerepo.pair_srpm_rpm import RPMToSRPMPairs +from prunerepo.pair_srpm_rpm import PruneRepoAnalyzer class PrunerepoException(Exception): @@ -56,15 +56,6 @@ def run_cmd(cmd, log, dry_run=False): return stdout.decode(encoding='utf-8').splitlines() -def get_package_build_time(package_path, log): - """ - Get build time by reading package metadata - """ - query_cmd = ["/usr/bin/rpm", "-qp", "--nosignature", "--queryformat", "%{BUILDTIME}"] + [package_path] - stdout = run_cmd(query_cmd, log) - return int(stdout[0]) - - def get_rpms(repoquery_cmd, log): """ Get paths to rpm packages in the repository according to given repoquery_cmd @@ -209,18 +200,21 @@ def _get_rpms_to_remove_internal(directory, days, log, cachedir): all_rpms = get_rpms(get_all_packages_cmd, log) repodir = os.path.abspath(directory) - pair_lookup = RPMToSRPMPairs(repodir, log) + repo_analyzer = PruneRepoAnalyzer(repodir, log) to_remove_rpms = set(all_rpms) - set(latest_rpms) rpm_list = [] + + time_now = time.time() for rpm in to_remove_rpms: log.debug("Checking age of the '%s' file", os.path.split(rpm)[1]) - if time.time() - get_package_build_time(rpm, log) < days * 24 * 3600: - continue rel_rpm = os.path.normpath(os.path.relpath(rpm, repodir)) + if time_now - repo_analyzer.get_build_time(rel_rpm) < days * 24 * 3600: + continue + rpm_list.append(rel_rpm) - rel_srpm = pair_lookup.srpm_to_be_removed_for_rpm(rel_rpm) + rel_srpm = repo_analyzer.srpm_to_be_removed_for_rpm(rel_rpm) if not rel_srpm: continue rpm_list.append(rel_srpm) diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index 8d190e7..668361e 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -59,7 +59,7 @@ def initialized_dnf(repo, log): base.close() -def get_mapping(repo, log): +def _get_mapping(repo, log): """ Read the repository metadata and find what RPMs were built from which SRPMs, and map the source RPM name to SRPMs and vice versa. @@ -73,15 +73,21 @@ def get_mapping(repo, log): found_srpms = set() map_srpm_to_rpms = {} map_rpm_to_srpm = {} + map_rpm_to_buildtime = {} # list all packages for package in available_packages: + # remove leading ./ etc. + normalized_pkg_path = os.path.normpath(package.relativepath) + + map_rpm_to_buildtime[normalized_pkg_path] = package.buildtime + if package.sourcerpm: # handling source RPMs only for now continue # handling source RPM - found_srpms.add(os.path.normpath(package.relativepath)) + found_srpms.add(normalized_pkg_path) # group the binary RPMs for package in available_packages: @@ -108,20 +114,20 @@ def get_mapping(repo, log): map_srpm_to_rpms[expected_source_rpm].add(rpm) map_rpm_to_srpm[rpm] = expected_source_rpm - return map_srpm_to_rpms, map_rpm_to_srpm + return map_srpm_to_rpms, map_rpm_to_srpm, map_rpm_to_buildtime -class RPMToSRPMPairs: # pylint: disable=too-few-public-methods +class PruneRepoAnalyzer: # pylint: disable=too-few-public-methods """ - Search/query MAP of SRPM => RPMS and RPM => SRPM objects. - The paths we work with, and return are relative to the "repo" directory we - get in the constructor. + Search/query MAP of "SRPM => RPMS", "RPM => SRPM", and "(S)RPM => + buildtime". The paths we work with, and return are relative to the "repo" + directory we get in the constructor. """ def __init__(self, repo, log): self.log = log self.repo = repo - self.srpm_map, self.rpm_map = get_mapping(repo, log) + self.srpm_map, self.rpm_map, self.buildtime_map = _get_mapping(repo, log) def srpm_to_be_removed_for_rpm(self, rpm): """ @@ -149,3 +155,9 @@ class RPMToSRPMPairs: # pylint: disable=too-few-public-methods # Return non-None value, to notify caller that the SRPM should be # removed. return srpm + + def get_build_time(self, rpm): + """ + Get the BUILDTIME stored in RPM, per previous repository analysis. + """ + return self.buildtime_map[rpm] From 5662bba38600e532c1f38edf0bae19a9818b3435 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Jun 06 2023 16:23:16 +0000 Subject: [PATCH 2/2] PruneRepoAnalyzer to use the temporary cachedir --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index c10f1c3..05e8348 100644 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -200,7 +200,7 @@ def _get_rpms_to_remove_internal(directory, days, log, cachedir): all_rpms = get_rpms(get_all_packages_cmd, log) repodir = os.path.abspath(directory) - repo_analyzer = PruneRepoAnalyzer(repodir, log) + repo_analyzer = PruneRepoAnalyzer(repodir, cachedir, log) to_remove_rpms = set(all_rpms) - set(latest_rpms) rpm_list = [] diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index 668361e..db83882 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -28,12 +28,13 @@ def url_to_repoid(repo_url): @contextmanager -def initialized_dnf(repo, log): +def initialized_dnf(repo, cachedir, log): """ Prepare and yield DNF Base object pre-configured to work with the given REPO. Make sure you run this in `with` context. """ base = dnf.Base() + base.conf.cachedir = cachedir base.read_all_repos() # disable all pre-configured repos @@ -59,13 +60,13 @@ def initialized_dnf(repo, log): base.close() -def _get_mapping(repo, log): +def _get_mapping(repo, cachedir, log): """ Read the repository metadata and find what RPMs were built from which SRPMs, and map the source RPM name to SRPMs and vice versa. """ # query the metadata - with initialized_dnf(repo, log) as base: + with initialized_dnf(repo, cachedir, log) as base: query = base.sack.query() remote = query.filter(reponame__neq="@System") available_packages = list(remote) @@ -124,10 +125,11 @@ class PruneRepoAnalyzer: # pylint: disable=too-few-public-methods directory we get in the constructor. """ - def __init__(self, repo, log): + def __init__(self, repo, cachedir, log): self.log = log self.repo = repo - self.srpm_map, self.rpm_map, self.buildtime_map = _get_mapping(repo, log) + self.srpm_map, self.rpm_map, self.buildtime_map = _get_mapping( + repo, cachedir, log) def srpm_to_be_removed_for_rpm(self, rpm): """