From a78322f675bcf92fa7e53a7793771defac749245 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Feb 15 2025 13:32:45 +0000 Subject: [PATCH 1/5] pair_srpm_rpm: prepare for the DNF5 movement The way we work with libdnf5 and dnf objects is very different, but we can somehow work with both on query level. While doing this, let's hide the new _get_dnf_query() symbol (shouldn't be used externally). Use `location` attribute, as `relativepath` has been removed in DNF5. --- diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index db83882..ccf43fe 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -28,10 +28,10 @@ def url_to_repoid(repo_url): @contextmanager -def initialized_dnf(repo, cachedir, log): +def _get_dnf_query(repo, cachedir, log): """ - Prepare and yield DNF Base object pre-configured to work with the given - REPO. Make sure you run this in `with` context. + Prepare and yield a prepared DNF4 query object pre-configured to work with + the given REPO. Make sure you run this in `with` context. """ base = dnf.Base() base.conf.cachedir = cachedir @@ -55,7 +55,9 @@ def initialized_dnf(repo, cachedir, log): # read the metadata base.fill_sack() try: - yield base + query = base.sack.query() + remote = query.filter(reponame__neq="@System") + yield remote finally: base.close() @@ -65,12 +67,12 @@ def _get_mapping(repo, cachedir, log): Read the repository metadata and find what RPMs were built from which SRPMs, and map the source RPM name to SRPMs and vice versa. """ - # query the metadata - with initialized_dnf(repo, cachedir, log) as base: - query = base.sack.query() - remote = query.filter(reponame__neq="@System") - available_packages = list(remote) + with _get_dnf_query(repo, cachedir, log) as query: + available_packages = list(query) + return _available_pkgs_to_mapping(available_packages, log) + +def _available_pkgs_to_mapping(available_packages, log): found_srpms = set() map_srpm_to_rpms = {} map_rpm_to_srpm = {} @@ -79,7 +81,7 @@ def _get_mapping(repo, cachedir, log): # list all packages for package in available_packages: # remove leading ./ etc. - normalized_pkg_path = os.path.normpath(package.relativepath) + normalized_pkg_path = os.path.normpath(package.location) map_rpm_to_buildtime[normalized_pkg_path] = package.buildtime @@ -95,23 +97,23 @@ def _get_mapping(repo, cachedir, log): if not package.sourcerpm: continue # only binary RPMs now.. - dirname = os.path.dirname(os.path.normpath(package.relativepath)) + dirname = os.path.dirname(os.path.normpath(package.location)) expected_source_rpm = os.path.normpath( os.path.join(dirname, package.sourcerpm)) if not expected_source_rpm: - log.error("%s has no SRPM header", package.relativepath) + log.error("%s has no SRPM header", package.location) continue if expected_source_rpm not in found_srpms: log.error("%s has no source RPM in the directory", - package.relativepath) + package.location) continue if not expected_source_rpm in map_srpm_to_rpms: map_srpm_to_rpms[expected_source_rpm] = set() - rpm = os.path.normpath(package.relativepath) + rpm = os.path.normpath(package.location) map_srpm_to_rpms[expected_source_rpm].add(rpm) map_rpm_to_srpm[rpm] = expected_source_rpm From 6ebb9c01bbdba097034c3bbecb3084400ec4178a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Feb 15 2025 14:13:44 +0000 Subject: [PATCH 2/5] pair_srpm_rpm: use correct specifier for logging strings This bug was not normally exposed because dnf did not turn loglevel DEBUG on, but libdnf5 seems to. --- diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index ccf43fe..18891f8 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -147,7 +147,7 @@ class PruneRepoAnalyzer: # pylint: disable=too-few-public-methods return None srpm = self.rpm_map[rpm] - self.log.debug("removing %f from set of RPMs in %s", rpm, srpm) + self.log.debug("removing %s from set of RPMs in %s", rpm, srpm) # Drop the reference to RPM, as it is being removed. self.srpm_map[srpm].remove(rpm) From afa915573f2f30364493ecb86408f2029f13883e Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Feb 15 2025 14:13:46 +0000 Subject: [PATCH 3/5] pair_srpm_rpm: migrate to DNF5, keep DNF4 backward compat --- diff --git a/prunerepo/pair_srpm_rpm.py b/prunerepo/pair_srpm_rpm.py index 18891f8..2a2636f 100644 --- a/prunerepo/pair_srpm_rpm.py +++ b/prunerepo/pair_srpm_rpm.py @@ -12,7 +12,15 @@ import re import os from contextlib import contextmanager -import dnf + +try: + import libdnf5 + DNF5_USED = True +except ImportError: + # We can not import both, if we do - dnf query mechanism behaves + # differently. + DNF5_USED = False + import dnf def url_to_repoid(repo_url): @@ -28,6 +36,29 @@ def url_to_repoid(repo_url): @contextmanager +def _get_dnf5_query(repo, cachedir, log): + """ + Prepare and yield a prepared DNF5 query object pre-configured to work with + the given REPO. Make sure you run this in `with` context. + """ + base = libdnf5.base.Base() + base_config = base.get_config() + base_config.plugins = False + base_config.cachedir = cachedir + base.load_config() + base.setup() + + repo_sack = base.get_repo_sack() + repo_obj = repo_sack.create_repo(url_to_repoid(repo)) + repo_obj.get_config().baseurl = repo + repo_sack.load_repos(libdnf5.repo.Repo.Type_AVAILABLE) + query = libdnf5.rpm.PackageQuery(base) + try: + yield query + finally: + pass + +@contextmanager def _get_dnf_query(repo, cachedir, log): """ Prepare and yield a prepared DNF4 query object pre-configured to work with @@ -67,11 +98,30 @@ def _get_mapping(repo, cachedir, log): Read the repository metadata and find what RPMs were built from which SRPMs, and map the source RPM name to SRPMs and vice versa. """ - with _get_dnf_query(repo, cachedir, log) as query: - available_packages = list(query) + get_query = _get_dnf5_query if DNF5_USED else _get_dnf_query + with get_query(repo, cachedir, log) as query: + available_packages = [_Pkg(p) for p in query] return _available_pkgs_to_mapping(available_packages, log) +class _Pkg: + """ + Make the Package backward compatible with DNF4. Remove once we have no + DNF4 support (RHEL11+). + """ + def __init__(self, pkg): + self.pkg = pkg + def __getattr__(self, attr): + try: + return getattr(self.pkg, attr) + except AttributeError: + if attr == "get_build_time": + # get_build_time() was buildtime in DNF4 + return lambda: self.pkg.buildtime + attr = attr[4:] + return lambda: getattr(self.pkg, attr) + + def _available_pkgs_to_mapping(available_packages, log): found_srpms = set() map_srpm_to_rpms = {} @@ -81,11 +131,11 @@ def _available_pkgs_to_mapping(available_packages, log): # list all packages for package in available_packages: # remove leading ./ etc. - normalized_pkg_path = os.path.normpath(package.location) + normalized_pkg_path = os.path.normpath(package.get_location()) - map_rpm_to_buildtime[normalized_pkg_path] = package.buildtime + map_rpm_to_buildtime[normalized_pkg_path] = package.get_build_time() - if package.sourcerpm: + if package.get_sourcerpm(): # handling source RPMs only for now continue @@ -94,26 +144,26 @@ def _available_pkgs_to_mapping(available_packages, log): # group the binary RPMs for package in available_packages: - if not package.sourcerpm: + if not package.get_sourcerpm(): continue # only binary RPMs now.. - dirname = os.path.dirname(os.path.normpath(package.location)) + dirname = os.path.dirname(os.path.normpath(package.get_location())) expected_source_rpm = os.path.normpath( - os.path.join(dirname, package.sourcerpm)) + os.path.join(dirname, package.get_sourcerpm())) if not expected_source_rpm: - log.error("%s has no SRPM header", package.location) + log.error("%s has no SRPM header", package.get_location()) continue if expected_source_rpm not in found_srpms: log.error("%s has no source RPM in the directory", - package.location) + package.get_location()) continue if not expected_source_rpm in map_srpm_to_rpms: map_srpm_to_rpms[expected_source_rpm] = set() - rpm = os.path.normpath(package.location) + rpm = os.path.normpath(package.get_location()) map_srpm_to_rpms[expected_source_rpm].add(rpm) map_rpm_to_srpm[rpm] = expected_source_rpm From 205839dd3c779b0332c9757c34e58cf75da038e9 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Feb 15 2025 14:13:49 +0000 Subject: [PATCH 4/5] Use file:// in baseurls explicitly This works for both DNF4 and DNF5, while the previous code would only work with DNF4. Relates: https://github.com/rpm-software-management/dnf5/issues/2048 --- diff --git a/prunerepo/helpers.py b/prunerepo/helpers.py index 05e8348..f388ccd 100644 --- a/prunerepo/helpers.py +++ b/prunerepo/helpers.py @@ -175,9 +175,9 @@ def get_rpms_to_remove(directory, days=0, log=None): def _get_rpms_to_remove_internal(directory, days, log, cachedir): get_all_packages_cmd = [ - "dnf-3", + "dnf", "repoquery", - "--repofrompath=prunerepo_query," + os.path.abspath(directory), + "--repofrompath=prunerepo_query,file://" + os.path.abspath(directory), "--repo=prunerepo_query", "--refresh", "--location", diff --git a/tests/testlib.sh b/tests/testlib.sh index e4dee6a..7cd34b5 100644 --- a/tests/testlib.sh +++ b/tests/testlib.sh @@ -10,7 +10,7 @@ function runcmd { } function listpkgsbyrepo { - dnf-3 repoquery --repofrompath=test_prunerepo,$testrepo --setopt=cachedir=prunerepo-tests-dnf-cache --repo=test_prunerepo --refresh --quiet --location --setopt='skip_if_unavailable=False' | sed 's|file://||' | sort + dnf repoquery --repofrompath=test_prunerepo,file://$testrepo --setopt=cachedir=prunerepo-tests-dnf-cache --repo=test_prunerepo --refresh --quiet --location --setopt='skip_if_unavailable=False' | sed 's|file://||' | sort } function listpkgsbyfs { From 34da43e7d35c82a577f27ce0647588dc6688584b Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Feb 15 2025 14:39:51 +0000 Subject: [PATCH 5/5] rpm: depend either on dnf5 or dnf4 (not both) --- diff --git a/prunerepo.spec b/prunerepo.spec index 7ebd80d..b95a6cc 100644 --- a/prunerepo.spec +++ b/prunerepo.spec @@ -4,6 +4,13 @@ Summary: Remove old packages from rpm-md repository Release: 1%{?dist} Url: https://pagure.io/prunerepo + +%if 0%{?rhel} > 10 || 0%{?fedora} > 40 +%bcond_without dnf5 +%else +%bcond_with dnf5 +%endif + # Source is created by: # git clone %%url && cd prunerepo # tito build --tgz --tag %%name-%%version-%%release @@ -18,11 +25,20 @@ BuildRequires: python3-rpm BuildRequires: createrepo_c BuildRequires: asciidoc BuildRequires: findutils -BuildRequires: python3-dnf +%if %{with dnf5} +BuildRequires: dnf5-command(repoquery) +BuildRequires: python3-libdnf5 +Requires: dnf5-command(repoquery) +%else +BuildRequires: dnf-command(repoquery) BuildRequires: dnf-plugins-core +Requires: dnf-command(repoquery) +# F40 needs this explicit requirement +Requires: /usr/bin/dnf +BuildRequires: /usr/bin/dnf +%endif BuildRequires: coreutils Requires: createrepo_c -Requires: dnf-plugins-core Requires: python3-rpm Requires: python3