From 55bd8046e2605d9616338703c225a5ac2cecba0a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 06 2019 13:38:28 +0000 Subject: [PATCH 1/6] Drop useless double-quote in --queryformat --- diff --git a/prunerepo b/prunerepo index e9b6cfa..aa8f7fb 100755 --- a/prunerepo +++ b/prunerepo @@ -42,7 +42,7 @@ get_all_packages_cmd = [ '--repofrompath=prunerepo_query,'+os.path.abspath(args.path), '--repo=prunerepo_query', '--refresh', - '--queryformat="%{location}"', + '--queryformat=%{location}', '--quiet', ] @@ -83,7 +83,7 @@ def run_cmd(cmd, silent=False, dry_run=False): if process.returncode != 0: print(stderr.decode(encoding='utf-8'), file=sys.stderr) sys.exit(1) - return [line.strip('"') for line in stdout.decode(encoding='utf-8').split()] # NOTE: for some reason the get_all_packages_cmd gives output as b'"..."\n"..."\n', hence line.strip('"') + return stdout.decode(encoding='utf-8').split() def get_package_build_time(package_path): @@ -100,7 +100,7 @@ def get_rpms(repoquery_cmd): Get paths to rpm packages in the repository according to given repoquery_cmd """ stdout = run_cmd(repoquery_cmd) # returns srpms as well - rel_rpms_paths = [relpath.strip('"') for relpath in stdout if not is_srpm(relpath)] + rel_rpms_paths = [relpath for relpath in stdout if not is_srpm(relpath)] abs_rpms_paths = [os.path.abspath(os.path.join(args.path, relpath)) for relpath in rel_rpms_paths] return abs_rpms_paths From 5dbb28bc4a426111c455dc4dd5a7666b329e8485 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 06 2019 13:38:31 +0000 Subject: [PATCH 2/6] Always dump stderr of repoquery (not only in error case) --- diff --git a/prunerepo b/prunerepo index aa8f7fb..ab7d433 100755 --- a/prunerepo +++ b/prunerepo @@ -80,8 +80,8 @@ def run_cmd(cmd, silent=False, dry_run=False): return process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() + print(stderr.decode(encoding='utf-8'), file=sys.stderr) if process.returncode != 0: - print(stderr.decode(encoding='utf-8'), file=sys.stderr) sys.exit(1) return stdout.decode(encoding='utf-8').split() From a36158d92c11ce436754d317a68ac28bbdf90f51 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 12 2019 10:01:00 +0000 Subject: [PATCH 3/6] Set skip_if_unavailable=False to not loose the data This probably caused massive data loss in https://pagure.io/copr/copr/issue/1090 The skip_if_unavailable=True used to be the default on Fedora 30: https://fedoraproject.org/wiki/Changes/Set_skip_if_unavailable_default_to_false --- diff --git a/prunerepo b/prunerepo index ab7d433..916419b 100755 --- a/prunerepo +++ b/prunerepo @@ -44,6 +44,7 @@ get_all_packages_cmd = [ '--refresh', '--queryformat=%{location}', '--quiet', + '--setopt=skip_if_unavailable=False', ] get_latest_packages_cmd = get_all_packages_cmd + [ '--latest-limit=1' ] From c6c27e7ee18b0eab360b4dc2e9ab0f55984462e1 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 12 2019 10:01:33 +0000 Subject: [PATCH 4/6] Use splitlines instead of split for repoquery parsing Even though there shouldn't be any other whitespaces in the query format output, it's better to not risk it. --- diff --git a/prunerepo b/prunerepo index 916419b..0445031 100755 --- a/prunerepo +++ b/prunerepo @@ -84,7 +84,7 @@ def run_cmd(cmd, silent=False, dry_run=False): print(stderr.decode(encoding='utf-8'), file=sys.stderr) if process.returncode != 0: sys.exit(1) - return stdout.decode(encoding='utf-8').split() + return stdout.decode(encoding='utf-8').splitlines() def get_package_build_time(package_path): From a0f856f538532afcfe6a4f84c98c7f67169346f2 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 12 2019 10:01:33 +0000 Subject: [PATCH 5/6] skip prunerepo if set(latest_rpms) is empty This is trivial safety check against repoquery bugs; such situation could lead to empty set of latest_rpms which would mean in turn removing of all packages in repository. --- diff --git a/prunerepo b/prunerepo index 0445031..c1a7530 100755 --- a/prunerepo +++ b/prunerepo @@ -132,6 +132,9 @@ def prune_packages(): log_info('Removing obsoleted packages...') was_deletion = False latest_rpms = get_rpms(get_latest_packages_cmd) + if not latest_rpms: + log_info("No RPMs available") + return was_deletion all_rpms = get_rpms(get_all_packages_cmd) to_remove_rpms = set(all_rpms) - set(latest_rpms) for rpm in to_remove_rpms: From ee280d87ed6edb460c495d9816e3d9aca7c9161e Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Nov 19 2019 07:02:57 +0000 Subject: [PATCH 6/6] avoid additional newlines in stderr --- diff --git a/prunerepo b/prunerepo index c1a7530..1a9cf22 100755 --- a/prunerepo +++ b/prunerepo @@ -81,7 +81,7 @@ def run_cmd(cmd, silent=False, dry_run=False): return process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = process.communicate() - print(stderr.decode(encoding='utf-8'), file=sys.stderr) + sys.stderr.write(stderr.decode(encoding='utf-8')) if process.returncode != 0: sys.exit(1) return stdout.decode(encoding='utf-8').splitlines()