From 6e623523180c0e18a627593d5c82410c36b054c1 Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 1/7] feat: function that gets packages provided by requested package Signed-off-by: Anton Medvedev --- diff --git a/conf/etc/rpkg/fedpkg.conf b/conf/etc/rpkg/fedpkg.conf index fa08a8a..7412461 100644 --- a/conf/etc/rpkg/fedpkg.conf +++ b/conf/etc/rpkg/fedpkg.conf @@ -83,3 +83,7 @@ url = https://greenwave.fedoraproject.org/ [fedpkg.distgit] apibaseurl = https://src.fedoraproject.org token = + +[fedpkg.mdapi] +apibaseurl = https://mdapi.fedoraproject.org/ +package_requires = https://mdapi.fedoraproject.org/%(branch)s/requires/%(repo_name)s diff --git a/fedpkg/cli.py b/fedpkg/cli.py index e5ed34b..fd9ad42 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -43,6 +43,7 @@ from fedpkg.utils import (assert_new_tests_repo, assert_valid_epel_package, config_get_safely, disable_monitoring, do_add_remote, do_fork, expand_release, get_dist_git_url, get_fedora_release_state, get_pagure_branches, + get_packages_provided_by_package, get_release_branches, get_stream_branches, is_epel, new_pagure_issue, sl_list_to_dict, verify_sls) @@ -1541,6 +1542,19 @@ class fedpkgClient(cliClient): Runs the rpkg retire command after check. Check includes reading the state of Fedora release. """ + # todo: write check on retirement + # the goal here is to recieve the names of packages that are depend on package the is wanted to be retired + # and notify the user that it would be better to retire them all + repo_name = self.cmd.repo_name + # 1. version using mdapi for getting info about dependencies, it works with rpms packages + if self.cmd.ns in ('rpms'): + # getting list of packages that requested package provides + provided_packages = get_packages_provided_by_package(repo_name, self.name) + for package in provided_packages: + print(f"Requested package provide `{package}` conside to retire it as well.") + # getting list of packages that requested package requires + + # 2. version # Allow retiring in epel if is_epel(self.cmd.branch_merge): super(fedpkgClient, self).retire() diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 1ce9177..1f9eb9b 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -657,3 +657,41 @@ def disable_monitoring(logger, base_url, token, repo_name, namespace, cli_name): raise rpkgError(base_error_msg.format(rv_error)) logger.info("Monitoring of the project was sucessfully disabled.") + +def get_packages_provided_by_package(repo_name, cli_name): + """ + Getting list of packages provided by a package. + :param repo_name: a string of the repository name + """ + mdapi_url = f"https://mdapi.fedoraproject.org/rawhide/requires/{repo_name}" + try: + mdapi_url = config.get('{0}.mdapi'.format(cli_name), + 'package_requires', + vars={'branch': 'rawhide', + 'repo_name': repo_name}) + except (ValueError, NoOptionError, NoSectionError) as e: + raise rpkgError('Could not get mdapi endpoint for repository' + '({0}): {1}.'.format(repo_name, str(e))) + + try: + rv = requests.get(mdapi_url, timeout=60) + except ConnectionError as error: + error_msg = ('The connection to Mdapi failed.' + ' The error was: {0}'.format(str(error))) + raise rpkgError(error_msg) + + if rv.status_code == 404: + # release wasn't found + return None + elif not rv.ok: + base_error_msg = ('The following error occurred while trying to ' + 'get the information about package on mdapi') + raise rpkgError(base_error_msg.format(rv.text)) + + provides = rv.json()[0]["provides"] + packages_provided_by_package = [package["name"] for package in provides] + print(packages_provided_by_package) + return packages_provided_by_package + +def get_packages_that_the_package_requires(): + pass From c15d72ce0f730fc2302fc56f6e15f3d351bed39c Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 2/7] feat: getting provided packages by subprocess Signed-off-by: Anton Medvedev --- diff --git a/fedpkg/cli.py b/fedpkg/cli.py index fd9ad42..d0067e4 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -1546,6 +1546,7 @@ class fedpkgClient(cliClient): # the goal here is to recieve the names of packages that are depend on package the is wanted to be retired # and notify the user that it would be better to retire them all repo_name = self.cmd.repo_name + # 1. version using mdapi for getting info about dependencies, it works with rpms packages if self.cmd.ns in ('rpms'): # getting list of packages that requested package provides @@ -1555,6 +1556,7 @@ class fedpkgClient(cliClient): # getting list of packages that requested package requires # 2. version + # Allow retiring in epel if is_epel(self.cmd.branch_merge): super(fedpkgClient, self).retire() diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 1f9eb9b..1ec9fff 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -12,6 +12,7 @@ import json import re +import subprocess from datetime import datetime, timezone import git @@ -693,5 +694,22 @@ def get_packages_provided_by_package(repo_name, cli_name): print(packages_provided_by_package) return packages_provided_by_package -def get_packages_that_the_package_requires(): - pass +def get_packages_provided_by_package2(repo_name): + try: + result = subprocess.run( + [ + "dnf", "repoquery", "--whatrequires", repo_name, "--qf", "%{name}" + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + if result.returncode != 0: + raise RuntimeError(f"Error running dnf command: {result.stderr.strip()}") + + dependent_packages = result.stdout.strip().split("\n") + return [pkg for pkg in dependent_packages if pkg] + + except FileNotFoundError: + raise RuntimeError("dnf command not found. Ensure dnf is installed and in the PATH.") From 103179d93671741fc63743af1403b0dc20f54cad Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 3/7] feat: small changes Signed-off-by: Anton Medvedev --- diff --git a/fedpkg/cli.py b/fedpkg/cli.py index d0067e4..a4d35c3 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -1550,7 +1550,7 @@ class fedpkgClient(cliClient): # 1. version using mdapi for getting info about dependencies, it works with rpms packages if self.cmd.ns in ('rpms'): # getting list of packages that requested package provides - provided_packages = get_packages_provided_by_package(repo_name, self.name) + provided_packages = get_packages_provided_by_package(self.config, repo_name, self.name) for package in provided_packages: print(f"Requested package provide `{package}` conside to retire it as well.") # getting list of packages that requested package requires diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 1ec9fff..8d56f5b 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -659,7 +659,7 @@ def disable_monitoring(logger, base_url, token, repo_name, namespace, cli_name): logger.info("Monitoring of the project was sucessfully disabled.") -def get_packages_provided_by_package(repo_name, cli_name): +def get_packages_provided_by_package(config, repo_name, cli_name): """ Getting list of packages provided by a package. :param repo_name: a string of the repository name From 359f17d456508443c3af2d4e7bb66cf1da7e39f4 Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 4/7] feat: getting list of packages that need to be retired with requested package Signed-off-by: Anton Medvedev --- diff --git a/conf/etc/rpkg/fedpkg.conf b/conf/etc/rpkg/fedpkg.conf index 7412461..0af120d 100644 --- a/conf/etc/rpkg/fedpkg.conf +++ b/conf/etc/rpkg/fedpkg.conf @@ -86,4 +86,4 @@ token = [fedpkg.mdapi] apibaseurl = https://mdapi.fedoraproject.org/ -package_requires = https://mdapi.fedoraproject.org/%(branch)s/requires/%(repo_name)s +package_info = https://mdapi.fedoraproject.org/%(branch)s/pkg/%(repo_name)s diff --git a/fedpkg/cli.py b/fedpkg/cli.py index a4d35c3..c050b0a 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -43,7 +43,7 @@ from fedpkg.utils import (assert_new_tests_repo, assert_valid_epel_package, config_get_safely, disable_monitoring, do_add_remote, do_fork, expand_release, get_dist_git_url, get_fedora_release_state, get_pagure_branches, - get_packages_provided_by_package, + get_copackages_of_pkg, dnf_repoquery, get_release_branches, get_stream_branches, is_epel, new_pagure_issue, sl_list_to_dict, verify_sls) @@ -1542,20 +1542,53 @@ class fedpkgClient(cliClient): Runs the rpkg retire command after check. Check includes reading the state of Fedora release. """ - # todo: write check on retirement + # todo: get copackages + # todo: get provided packages + # todo: remove packages that are not requested by others + # todo: remove packages that are provided by others + # todo: remove package that are in comps + # todo: remove packages that are in kickstarts + # todo: remove packages that somewhere else # the goal here is to recieve the names of packages that are depend on package the is wanted to be retired # and notify the user that it would be better to retire them all - repo_name = self.cmd.repo_name + pkg_name = self.cmd.repo_name # 1. version using mdapi for getting info about dependencies, it works with rpms packages if self.cmd.ns in ('rpms'): - # getting list of packages that requested package provides - provided_packages = get_packages_provided_by_package(self.config, repo_name, self.name) - for package in provided_packages: - print(f"Requested package provide `{package}` conside to retire it as well.") - # getting list of packages that requested package requires + # getting co-packages of package + copackages = get_copackages_of_pkg(self.config, pkg_name, self.name) + package_and_copakages:list = copackages.append(pkg_name) + + # getting packages provided by package and his copackages + provided_packages = package_and_copakages.copy() + for pkg_name in package_and_copakages: + provided_by_pkg = dnf_repoquery(pkg_name, "--provides") + provided_packages.extend(provided_by_pkg) + + # getting provided lst to be unique + unique_provided_packages = list(set(provided_packages)) + + # remove packages that are not required by others + packages_required_by_others = unique_provided_packages.copy() + for pkg_name in unique_provided_packages: + what_require_pkg = dnf_repoquery(pkg_name, "--whatrequires") + if len(what_require_pkg) == 0: + packages_required_by_others.remove(pkg_name) + elif len(what_require_pkg) == 1: + if what_require_pkg[0] == "": + packages_required_by_others.remove(pkg_name) + + # remove packages that are provided by others + packages_not_provided_by_others = packages_required_by_others.copy() + for pkg_name in packages_required_by_others: + what_provides_pkg = dnf_repoquery(pkg_name, "--whatprovides") + if len(what_provides_pkg) != 1: + packages_not_provided_by_others.remove(pkg_name) + + print("List of packages that need to be retired with requested package:\n" + "{0}".format(packages_not_provided_by_others)) + - # 2. version # Allow retiring in epel if is_epel(self.cmd.branch_merge): diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 8d56f5b..7730a02 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -659,20 +659,23 @@ def disable_monitoring(logger, base_url, token, repo_name, namespace, cli_name): logger.info("Monitoring of the project was sucessfully disabled.") -def get_packages_provided_by_package(config, repo_name, cli_name): +def get_copackages_of_pkg(config, pkg_name, cli_name) -> list: """ Getting list of packages provided by a package. - :param repo_name: a string of the repository name + :param config: config + :param pkg_name: package name + :param cli_name: cli name + :return: list of package names """ - mdapi_url = f"https://mdapi.fedoraproject.org/rawhide/requires/{repo_name}" + mdapi_url = f"https://mdapi.fedoraproject.org/rawhide/pkg/{pkg_name}" try: mdapi_url = config.get('{0}.mdapi'.format(cli_name), - 'package_requires', + 'package_info', vars={'branch': 'rawhide', - 'repo_name': repo_name}) + 'repo_name': pkg_name}) except (ValueError, NoOptionError, NoSectionError) as e: raise rpkgError('Could not get mdapi endpoint for repository' - '({0}): {1}.'.format(repo_name, str(e))) + '({0}): {1}.'.format(pkg_name, str(e))) try: rv = requests.get(mdapi_url, timeout=60) @@ -689,27 +692,28 @@ def get_packages_provided_by_package(config, repo_name, cli_name): 'get the information about package on mdapi') raise rpkgError(base_error_msg.format(rv.text)) - provides = rv.json()[0]["provides"] - packages_provided_by_package = [package["name"] for package in provides] - print(packages_provided_by_package) - return packages_provided_by_package + copackages = rv.json()["co-packages"] + return copackages -def get_packages_provided_by_package2(repo_name): +def dnf_repoquery(pkg_name, option): + """ + Getting list of packages by repoquery request. + :param pkg_name: a string of the repository name + :param option: a string of the option for requests + :return: a list of package names + """ try: + # Run the dnf repoquery command result = subprocess.run( - [ - "dnf", "repoquery", "--whatrequires", repo_name, "--qf", "%{name}" - ], + ['dnf', 'repoquery', option, pkg_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True + text=True, + check=True ) - - if result.returncode != 0: - raise RuntimeError(f"Error running dnf command: {result.stderr.strip()}") - - dependent_packages = result.stdout.strip().split("\n") - return [pkg for pkg in dependent_packages if pkg] - - except FileNotFoundError: - raise RuntimeError("dnf command not found. Ensure dnf is installed and in the PATH.") + # Split the output into lines and return as a list + provided_packages = result.stdout.strip().split('\n') + return provided_packages + except subprocess.CalledProcessError as e: + print(f"Error: {e.stderr.strip()}") + return [] From 89d9016ad02c9de3fb668fb6824da2986b08144b Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 5/7] feat: getting packages from manifests and check if requested package in this list Signed-off-by: Anton Medvedev --- diff --git a/fedpkg/cli.py b/fedpkg/cli.py index c050b0a..e760dfb 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -43,7 +43,7 @@ from fedpkg.utils import (assert_new_tests_repo, assert_valid_epel_package, config_get_safely, disable_monitoring, do_add_remote, do_fork, expand_release, get_dist_git_url, get_fedora_release_state, get_pagure_branches, - get_copackages_of_pkg, dnf_repoquery, + get_copackages_of_pkg, dnf_repoquery, get_manifested_packages, get_release_branches, get_stream_branches, is_epel, new_pagure_issue, sl_list_to_dict, verify_sls) @@ -1542,22 +1542,13 @@ class fedpkgClient(cliClient): Runs the rpkg retire command after check. Check includes reading the state of Fedora release. """ - # todo: get copackages - # todo: get provided packages - # todo: remove packages that are not requested by others - # todo: remove packages that are provided by others - # todo: remove package that are in comps - # todo: remove packages that are in kickstarts - # todo: remove packages that somewhere else - # the goal here is to recieve the names of packages that are depend on package the is wanted to be retired - # and notify the user that it would be better to retire them all - pkg_name = self.cmd.repo_name - - # 1. version using mdapi for getting info about dependencies, it works with rpms packages + requested_pkg_name = self.cmd.repo_name + packages_that_need_to_be_retired = [requested_pkg_name] + if self.cmd.ns in ('rpms'): # getting co-packages of package - copackages = get_copackages_of_pkg(self.config, pkg_name, self.name) - package_and_copakages:list = copackages.append(pkg_name) + copackages = get_copackages_of_pkg(self.config, requested_pkg_name, self.name) + package_and_copakages:list = copackages.append(requested_pkg_name) # getting packages provided by package and his copackages provided_packages = package_and_copakages.copy() @@ -1588,7 +1579,9 @@ class fedpkgClient(cliClient): print("List of packages that need to be retired with requested package:\n" "{0}".format(packages_not_provided_by_others)) + packages_that_need_to_be_retired = packages_not_provided_by_others.copy() + clean_packages_list = [pkg.split()[0] for pkg in packages_that_need_to_be_retired] # Allow retiring in epel if is_epel(self.cmd.branch_merge): @@ -1596,9 +1589,25 @@ class fedpkgClient(cliClient): else: state = get_fedora_release_state(self.config, self.name, self.cmd.branch_merge) + fedora_version = self.cmd.branch_merge + # getting packages from comps, kickstarts, workstation-ostree-config, lorax and FCOS manifests + manifested_packages = get_manifested_packages(fedora_version) + package_list_for_retirement = [] + for pkg_name in clean_packages_list: + if pkg_name not in manifested_packages: + package_list_for_retirement.append(pkg_name) + else: + if pkg_name is requested_pkg_name: + print("ERROR: Requested package {0} is listed in manifests.".format(pkg_name)) + return False + + print("Please retire all packages listed here `{0}`".format(package_list_for_retirement)) + + # Allow retiring in Rawhide and Branched until Final Freeze if state is None or state == 'pending': super(fedpkgClient, self).retire() + else: self.log.error("Fedora release (%s) is in state '%s' - retire operation " "is not allowed." % (self.cmd.branch_merge, state)) diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 7730a02..dda2650 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -11,16 +11,21 @@ # the full text of the license. import json +import os import re import subprocess +import tempfile from datetime import datetime, timezone import git import requests +import yaml +from git import Repo from pyrpkg import rpkgError from requests.exceptions import ConnectionError from configparser import NoOptionError, NoSectionError from urllib.parse import urlparse +from xml.etree import ElementTree def query_bodhi(server_url, timeout=60): @@ -695,6 +700,129 @@ def get_copackages_of_pkg(config, pkg_name, cli_name) -> list: copackages = rv.json()["co-packages"] return copackages +def get_manifested_packages(fedora_version): + """ + Getting list of packages that are listed in + comps, or kickstarts, or workstation-ostree-config, or FCOS manifests, or lorax's lists. + No need to get packages listed in kickstarts and lorax's manifests, because it + use packages listed in comps. + + :param fedora_version: fedora version + :return: list of package names listed in manifests + """ + manifested_packages = [] + with tempfile.TemporaryDirectory() as tmp_dir: + # TODO write error handling for cloning repo + print(f"Temporary directory created: {tmp_dir}") + + # Creating subdirectories + comps_dir = os.path.join(tmp_dir, "fedora-comps") + workstation_ostree_dir = os.path.join(tmp_dir, "workstation-ostree-config") + fcos_manifests_dir = os.path.join(tmp_dir, "fcos-manifests") + + os.mkdir(comps_dir) + os.mkdir(workstation_ostree_dir) + os.mkdir(fcos_manifests_dir) + + # getting comps packages + comps_repo_url = "https://pagure.io/fedora-comps.git" + Repo.clone_from(comps_repo_url, comps_dir) + + file_name = "comps-{0}.xml.in".format(fedora_version) + file_path = os.path.join(comps_dir, file_name) + + if os.path.exists(file_path): + with open(file_path, "r") as f: + content = f.read() + else: + print(f"{file_path} does not exist in the repository.") + + xml_parse = ElementTree.fromstring(content) + + # find all packagereq + package_requirements = xml_parse.findall(".//packagereq") + comps_packages = [pkg.text for pkg in package_requirements] + + manifested_packages.extend(comps_packages) + + # getting workstation-ostree-config packages + workstation_ostree_repo_url = "https://pagure.io/workstation-ostree-config.git" + Repo.clone_from(workstation_ostree_repo_url, workstation_ostree_dir) + + # Walk through the repo and list files + file_list = os.listdir(workstation_ostree_dir) + + files_with_packages_common = [file for file in file_list + if file.endswith("common.yaml")] + print("Files with packages common:") + print(files_with_packages_common) + + files_with_packages = [file for file in file_list + if file.endswith("packages.yaml")] + print("Files with packages:") + print(files_with_packages) + + workstation_ostree_packages = [] + + def get_packages_from_worksation_ostree(files): + packages = [] + for file_name in files: + file_path = os.path.join(workstation_ostree_dir, file_name) + if os.path.exists(file_path): + with open(file_path, "r") as f: + content = yaml.safe_load(f) + packages.extend(content["packages"]) + return packages + + packages_from_common = get_packages_from_worksation_ostree(files_with_packages_common) + packages_from_package_files = get_packages_from_worksation_ostree(files_with_packages) + + workstation_ostree_packages.extend(packages_from_common) + workstation_ostree_packages.extend(packages_from_package_files) + workstation_ostree_packages_clean = list(set(workstation_ostree_packages)) + + workstation_ostree_unique_packages = [] + + for package in workstation_ostree_packages_clean: + if package not in manifested_packages: + workstation_ostree_unique_packages.append(package) + + manifested_packages.extend(workstation_ostree_unique_packages) + + # Getting packages from FCOS manifest + fcos_repo_url = "https://github.com/coreos/fedora-coreos-config.git" + Repo.clone_from(fcos_repo_url, fcos_manifests_dir) + + folder_name = "manifests" + folder_path = os.path.join(fcos_manifests_dir, folder_name) + file_list = os.listdir(folder_path) + yaml_files = [file for file in file_list if file.endswith(".yaml")] + + fcos_manifest_packages = [] + + for file_name in yaml_files: + file_path = os.path.join(folder_path, file_name) + if os.path.exists(file_path): + with open(file_path, "r") as f: + content = yaml.safe_load(f) + if "packages" in content.keys(): + fcos_manifest_packages.extend(content["packages"]) + + + fcos_manifest_packages_clean = list(set(fcos_manifest_packages)) + + fcos_manifest_unique_packages = [] + + for package in fcos_manifest_packages_clean: + if package not in manifested_packages: + fcos_manifest_unique_packages.append(package) + + print(len(fcos_manifest_unique_packages)) + + manifested_packages.extend(fcos_manifest_unique_packages) + + return manifested_packages + def dnf_repoquery(pkg_name, option): """ Getting list of packages by repoquery request. From 87dffbbefb075ba7ab8489c7d4c007de80cb1c32 Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 07 2025 07:45:47 +0000 Subject: [PATCH 6/7] feat(retire): added --disable-retirement-checks flag Signed-off-by: Anton Medvedev --- diff --git a/fedpkg/cli.py b/fedpkg/cli.py index e760dfb..51f6e43 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -138,6 +138,7 @@ class fedpkgClient(cliClient): self.register_set_distgit_token() self.register_set_pagure_token() self.register_do_disable_monitoring() + self.register_retire() def setup_completers(self): """ @@ -786,6 +787,9 @@ class fedpkgClient(cliClient): super(fedpkgClient, self).register_retire() retire_parser = self.subparsers.choices['retire'] + retire_parser.add_argument('--disable-retirement-checks', + action='store_true', + help='Disable retirement checks, useful for orphaned packages.') retire_parser.formatter_class = argparse.RawDescriptionHelpFormatter retire_parser.description = textwrap.dedent(''' {0} From 43f5c2c146d66fceea5368d2c7363a6bb12fbd15 Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Jul 09 2025 09:34:36 +0000 Subject: [PATCH 7/7] todos for future --- diff --git a/fedpkg/utils.py b/fedpkg/utils.py index dda2650..1e1bbab 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -672,6 +672,7 @@ def get_copackages_of_pkg(config, pkg_name, cli_name) -> list: :param cli_name: cli name :return: list of package names """ + # todo: remove one of options mdapi_url = f"https://mdapi.fedoraproject.org/rawhide/pkg/{pkg_name}" try: mdapi_url = config.get('{0}.mdapi'.format(cli_name),