From 4a7ff1e300aae5025c75df8ab960132d63b56831 Mon Sep 17 00:00:00 2001 From: Lenka Segura Date: Feb 12 2024 12:33:15 +0000 Subject: Remove pdc calls, use bodhi instead Signed-off-by: Lenka Segura --- diff --git a/conf/etc/rpkg/fedpkg-stage.conf b/conf/etc/rpkg/fedpkg-stage.conf index eeae285..772ae8d 100644 --- a/conf/etc/rpkg/fedpkg-stage.conf +++ b/conf/etc/rpkg/fedpkg-stage.conf @@ -55,6 +55,7 @@ results_dir=root # Refer to fedpkg.conf staging = True releases_service = https://bodhi.stg.fedoraproject.org/releases/%(release)s +url = bodhi.stg.fedoraproject.org/ [fedpkg-stage.mbs] auth_method = oidc @@ -71,12 +72,9 @@ url = https://bugzilla.redhat.com/ [fedpkg-stage.pagure] url = https://stg.pagure.io/ -[fedpkg-stage.pdc] -url = https://pdc.stg.fedoraproject.org/ - [fedpkg-stage.greenwave] url = https://greenwave.stg.fedoraproject.org/ [fedpkg-stage.distgit] apibaseurl = https://src.stg.fedoraproject.org -token = +token = diff --git a/conf/etc/rpkg/fedpkg.conf b/conf/etc/rpkg/fedpkg.conf index f975f71..4b897e8 100644 --- a/conf/etc/rpkg/fedpkg.conf +++ b/conf/etc/rpkg/fedpkg.conf @@ -57,6 +57,7 @@ results_dir=root # bodhi, and production is used without providing --staging. staging = False releases_service = https://bodhi.fedoraproject.org/releases/%(release)s +url = https://bodhi.fedoraproject.org/ [fedpkg.mbs] auth_method = oidc @@ -73,12 +74,9 @@ url = https://bugzilla.redhat.com/ url = https://pagure.io/ token = -[fedpkg.pdc] -url = https://pdc.fedoraproject.org/ - [fedpkg.greenwave] url = https://greenwave.fedoraproject.org/ [fedpkg.distgit] apibaseurl = https://src.fedoraproject.org -token = +token = diff --git a/fedpkg/cli.py b/fedpkg/cli.py index 5dd69b6..093dea5 100644 --- a/fedpkg/cli.py +++ b/fedpkg/cli.py @@ -1143,7 +1143,7 @@ class fedpkgClient(cliClient): if not (branch or all_releases) and active_branch: branch = active_branch - pdc_url = config.get('{0}.pdc'.format(name), 'url') + bodhi_url = config.get('{0}.bodhi'.format(name), 'url') if branch: if is_epel(branch): assert_valid_epel_package(repo_name, branch) @@ -1156,7 +1156,7 @@ class fedpkgClient(cliClient): 'underscores, and pluses are allowed in {0} branch ' 'names'.format('flatpak' if ns == 'flatpaks' else 'module')) release_branches = list(itertools.chain( - *list(get_release_branches(pdc_url).values()))) + *list(get_release_branches(bodhi_url).values()))) # treat epel*-playground the same as epel* release branches playground_match = re.match(r'^(epel\d+)-playground$', branch) # treat epel*-next the same as epel* release branches @@ -1169,14 +1169,14 @@ class fedpkgClient(cliClient): # If service levels were provided, verify them if service_levels: sl_dict = sl_list_to_dict(service_levels) - verify_sls(pdc_url, sl_dict) + verify_sls(bodhi_url, sl_dict) pagure_section = '{0}.pagure'.format(name) pagure_url = config_get_safely(config, pagure_section, 'url') pagure_token = config_get_safely(config, pagure_section, 'token') if all_releases: release_branches = list(itertools.chain( - *list(get_release_branches(pdc_url).values()))) + *list(get_release_branches(bodhi_url).values()))) branches = [b for b in release_branches if re.match(r'^(f\d+)$', b)] else: @@ -1367,25 +1367,23 @@ class fedpkgClient(cliClient): :raises rpkgError: if branch is a stream branch but it is inactive. """ for branch_info in stream_branches: - if branch_info['name'] != name: - continue - if branch_info['active']: + if branch_info == name: return True - else: - raise rpkgError('Cannot build from stream branch {0} as it is ' - 'inactive.'.format(name)) return False def _build(self, sets=None): if hasattr(self.args, 'chain') or self.args.scratch: return super(fedpkgClient, self)._build(sets) - server_url = self.config.get('{0}.pdc'.format(self.name), 'url') + server_url = self.config.get('{0}.bodhi'.format(self.name), 'url') + distgit_section = '{0}.distgit'.format(self.name) + apibaseurl = config_get_safely(self.config, distgit_section, "apibaseurl") + logger = self.log - stream_branches = get_stream_branches(server_url, self.cmd.repo_name) + stream_branches = get_stream_branches(server_url, self.cmd.repo_name, apibaseurl, logger) self.log.debug( 'Package %s has stream branches: %r', - self.cmd.repo_name, [item['name'] for item in stream_branches]) + self.cmd.repo_name, [item for item in stream_branches]) if not self.is_stream_branch(stream_branches, self.cmd.branch_merge): return super(fedpkgClient, self)._build(sets) @@ -1414,7 +1412,7 @@ class fedpkgClient(cliClient): return task_ids def show_releases_info(self): - server_url = self.config.get('{0}.pdc'.format(self.name), 'url') + server_url = self.config.get('{0}.bodhi'.format(self.name), 'url') releases = get_release_branches(server_url) def _join(ln): diff --git a/fedpkg/utils.py b/fedpkg/utils.py index 301bdab..925c00a 100644 --- a/fedpkg/utils.py +++ b/fedpkg/utils.py @@ -22,36 +22,28 @@ from six.moves.configparser import NoOptionError, NoSectionError from six.moves.urllib.parse import urlencode, urlparse -def query_pdc(server_url, endpoint, params, timeout=60): - api_url = '{0}/rest_api/v1/{1}/'.format( - server_url.rstrip('/'), endpoint.strip('/')) - query_args = params - while True: +def query_bodhi(server_url, timeout=60): + active_releases = ["current", "pending", "frozen"] + for item in active_releases: + query_arg = '/?state={0}'.format(item) + api_url = '{0}/releases/{1}'.format(server_url.rstrip('/'), query_arg) try: - rv = requests.get(api_url, params=query_args, timeout=60) + rv = requests.get('api_url{0}'.format(api_url), timeout=60) except ConnectionError as error: - error_msg = ('The connection to PDC failed while trying to get ' + error_msg = ('The connection to BODHI failed while trying to get ' 'the active release branches. The error was: {0}' .format(str(error))) raise rpkgError(error_msg) if not rv.ok: base_error_msg = ('The following error occurred while trying to ' - 'get the active release branches in PDC: {0}') + 'get the active release branches in Bodhi: {0}') raise rpkgError(base_error_msg.format(rv.text)) rv_json = rv.json() - for item in rv_json['results']: - yield item - - if rv_json['next']: - # Clear the query_args because they are baked into the "next" URL - query_args = {} - api_url = rv_json['next'] - else: - # We've gone through every page, so we can return the found - # branches - break + if rv_json['releases']: + for branch in rv_json['releases']: + yield branch['branch'] def get_sl_type(url, sl_name): @@ -287,30 +279,15 @@ def get_release_branches(server_url): :return: a mapping containing the active Fedora releases and EPEL branches. :rtype: dict """ - query_args = { - 'fields': ['short', 'version'], - 'active': True - } releases = {} - - for product_version in query_pdc( - server_url, 'product-versions', params=query_args): - short_name = product_version['short'] - version = product_version['version'] - - # If the version is not a digit we can ignore it (e.g. rawhide) - if not version.isdigit(): + for product_version in query_bodhi( + server_url): + if product_version == "rawhide": continue + short_name = "fedora" if product_version.startswith("f") else "epel" + releases.setdefault(short_name, set()).add(product_version) - if short_name == 'epel': - prefix = 'el' if version == '6' else 'epel' - elif short_name == 'fedora': - prefix = 'f' - - release = '{0}{1}'.format(prefix, version) - releases.setdefault(short_name, []).append(release) - - return releases + return {key: list(value) for key, value in releases.items()} def sl_list_to_dict(sls): @@ -494,46 +471,43 @@ def get_dist_git_url(anongiturl): return '{0}://{1}'.format(parsed_url.scheme, parsed_url.netloc) -def get_stream_branches(server_url, package_name): +def get_stream_branches(server_url, package_name, apibaseurl, logger): """Get a package's stream branches - :param str server_url: PDC server URL. + :param str server_url: Bodhi server URL. :param str package_name: package name. Generally for RPM packages, this is the repository name without namespace. - :return: a list of stream branches. Each element in the list is a dict - containing branch property name and active. - :rtype: list[dict] + :param str apibaseurl: Distgit url (src.fedoraproject.org) + :param obj logger: Log object + :return: a list of stream branches. Each element in the list is an active + release branch name. + :rtype: list """ - query_args = { - 'global_component': package_name, - 'fields': ['name', 'active'], - } - branches = query_pdc( - server_url, 'component-branches', params=query_args) - # When write this method, endpoint component-branches contains not only - # stream branches, but also regular release branches, e.g. rawhide/main, f28. - # Please remember to review the data regularly, there are only stream - # branches, or some new replacement of PDC fixes the issue as well, it - # should be ok to remove if from this list. + active_branches = [] + for active_branch in query_bodhi(server_url): + if active_branch not in active_branches: + active_branches.append(active_branch) + + package_branches = get_pagure_branches(logger, apibaseurl, "rpms", package_name) + + # Stream branches are the intersection between active releases in Bodhi + # (active_branches) and package_branches + intersection = list(set(active_branches)&set(package_branches)) + stream_branches = [] - for item in branches: - if item['name'] in ('rawhide', 'main'): + for item in intersection: + if item in ('rawhide', 'main'): continue - elif re.match(r'^(f|el)\d+$', item['name']): + elif re.match(r'^(f|el)\d+$', item): continue # epel7 is regular release branch # epel8 and above should be considered a stream branch to use # package.cfg file in the branch. - elif 'epel7' == item['name']: - continue - # epel8-playground and above playground branches should be considered - # as release branches so that it will use epelX-playground-candidate - # target to build. - elif re.match(r'^epel\d+-playground$', item['name']): + elif 'epel7' == item: continue # epel8-next and above branches should be considered as release branches # so that it will use epelX-next-candidate target to build. - elif re.match(r'^epel\d+-next$', item['name']): + elif re.match(r'^epel\d+-next$', item): continue else: stream_branches.append(item) diff --git a/test/fedpkg-stage.conf b/test/fedpkg-stage.conf index 49089c4..14748a1 100644 --- a/test/fedpkg-stage.conf +++ b/test/fedpkg-stage.conf @@ -12,6 +12,7 @@ kerberos_realms = STG.FEDORAPROJECT.ORG [fedpkg-stage.bodhi] staging = True +url = https://bodhi.stg.example.com [fedpkg-stage.bugzilla] url = https://bugzilla.stg.example.com @@ -19,8 +20,6 @@ url = https://bugzilla.stg.example.com [fedpkg-stage.pagure] url = https://pagure.stg.example.com/ -[fedpkg-stage.pdc] -url = https://pdc.stg.example.com/ - [fedpkg-stage.distgit] apibaseurl = https://src.example.com +token = aabbccdd diff --git a/test/fedpkg-test.conf b/test/fedpkg-test.conf index 6774f31..85acc2c 100644 --- a/test/fedpkg-test.conf +++ b/test/fedpkg-test.conf @@ -13,6 +13,7 @@ kerberos_realms = FEDORAPROJECT.ORG [fedpkg.bodhi] staging = False releases_service = https://bodhi.fedoraproject.org/releases/%(release)s +url = https://bodhi.example.org [fedpkg.bugzilla] url = https://bugzilla.example.com @@ -20,8 +21,6 @@ url = https://bugzilla.example.com [fedpkg.pagure] url = https://pagure.example.com/ -[fedpkg.pdc] -url = https://pdc.example.com/ - [fedpkg.distgit] apibaseurl = https://src.example.com +token = aabbccdd diff --git a/test/test_cli.py b/test/test_cli.py index 43bd20a..df263d3 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -2357,21 +2357,12 @@ class TestIsStreamBranch(CliTestCase): 'f28') self.assertFalse(result) - def test_stream_branch_is_inactive(self): - with patch('sys.argv', new=self.fake_cmd): - cli = self.new_cli() - - six.assertRaisesRegex( - self, rpkgError, 'Cannot build from stream branch', - cli.is_stream_branch, [{'name': '10', 'active': False}], '10') - def test_branch_is_stream_branch(self): with patch('sys.argv', new=self.fake_cmd): cli = self.new_cli() result = cli.is_stream_branch( - [{'name': '8', 'active': True}, {'name': '10', 'active': True}], - '8') + ['epel8', 'epel9'], 'epel8') self.assertTrue(result) @@ -2449,9 +2440,9 @@ class TestBuildFromStreamBranch(CliTestCase): _build): get_release_branches.return_value = { 'fedora': ['f28', 'f27'], - 'epel': ['el6', 'epel7'], + 'epel': ['epel7', 'epel8', 'epel9'], } - get_stream_branches.return_value = [{'name': '8', 'active': True}] + get_stream_branches.return_value = ['8', 'epel9'] _build.side_effect = [1, 2] self.checkout_branch(git.Repo(self.cloned_repo_path), '8') diff --git a/test/test_utils.py b/test/test_utils.py index 49dc1da..7bc5f87 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -132,29 +132,16 @@ class TestUtils(unittest.TestCase): """ mock_rv = Mock() mock_rv.ok = True - # This abbreviated data returned from the product-versions PDC API - mock_rv.json.return_value = { - 'count': 7, - 'next': None, - 'previous': None, - 'results': [ - {'short': 'epel', 'version': '6'}, - {'short': 'epel', 'version': '7'}, - {'short': 'fedora', 'version': '25'}, - {'short': 'fedora', 'version': '26'}, - {'short': 'fedora', 'version': '27'}, - {'short': 'fedora', 'version': '28'}, - {'short': 'fedora', 'version': 'rawhide'} - ] - } + # This abbreviated data returned from the Bodhi API active releases + mock_rv.json.return_value = {'releases': [{'name': 'EPEL-7', 'branch': 'epel7'}, {'name': 'EPEL-8', 'branch': 'epel8'}, {'name': 'EPEL-8N', 'long_name': 'Fedora EPEL 8 Next', 'version': '8', 'id_prefix': 'FEDORA-EPEL-NEXT', 'branch': 'epel8-next'}, {'name': 'EPEL-9', 'branch': 'epel9'}, {'name': 'EPEL-9N', 'branch': 'epel9-next'}, {'name': 'F38', 'branch': 'f38'}, {'name': 'F38C', 'branch': 'f38'}, {'name': 'F38F', 'branch': 'f38'}, {'name': 'F38M', 'branch': 'f38m'}, {'name': 'F39', 'branch': 'f39'}, {'name': 'F39C', 'branch': 'f39'}, {'name': 'F39F', 'branch': 'f39'}], 'page': 1, 'pages': 1, 'rows_per_page': 20, 'total': 12} mock_request_get.return_value = mock_rv - expected = set(['el6', 'epel7', 'f25', 'f26', 'f27', 'f28']) expected = { - 'epel': ['el6', 'epel7'], - 'fedora': ['f25', 'f26', 'f27', 'f28'], + 'epel': ['epel7', 'epel8', 'epel8-next', 'epel9', 'epel9-next'], + 'fedora': ['f38', 'f38m', 'f39'], } - actual = utils.get_release_branches('http://pdc.local') - self.assertEqual(expected, actual) + actual = utils.get_release_branches('http://src.local') + actual_sorted = {key: sorted(value) for key, value in sorted(actual.items())} + self.assertDictEqual(expected, actual_sorted) @patch('requests.get') @@ -436,20 +423,20 @@ class TestNewPagureIssue(unittest.TestCase): @patch('requests.get') class TestQueryPDC(unittest.TestCase): - """Test utils.query_pdc""" + """Test utils.query_bodhi""" def test_connection_error(self, get): get.side_effect = ConnectionError - result = utils.query_pdc('http://localhost/', 'endpoint', {}) + result = utils.query_bodhi('http://localhost/') six.assertRaisesRegex( - self, rpkgError, 'The connection to PDC failed', + self, rpkgError, 'The connection to BODHI failed', list, result) def test_response_not_ok(self, get): get.return_value.ok = False - result = utils.query_pdc('http://localhost/', 'endpoint', {}) + result = utils.query_bodhi('http://localhost/') six.assertRaisesRegex( self, rpkgError, 'The following error occurred', list, result) @@ -458,39 +445,47 @@ class TestQueryPDC(unittest.TestCase): rv = Mock() rv.ok = True rv.json.side_effect = [ - {'results': ['item1', 'item2'], - 'next': 'http://localhost/?page=2'}, - {'results': ['item3'], 'next': None} + {'releases': [ + {'name': 'item1', 'branch': 'item2'}, + {'name': 'items5', 'branch': 'item6'} + ]}, + {'releases': [ + {'name': 'item3', 'branch': 'item4'} + ]} ] get.return_value = rv - result = utils.query_pdc('http://localhost/', 'endpoint', {}) - self.assertEqual(['item1', 'item2', 'item3'], list(result)) + result = utils.query_bodhi('http://localhost/') + v = next(result) + self.assertEqual('item2', v) + v = next(result) + self.assertEqual('item6', v) + v = next(result) + self.assertEqual('item4', v) + class TestGetStreamBranches(unittest.TestCase): """Test get_stream_branches""" + @patch('fedpkg.utils.get_pagure_branches') @patch('requests.get') - def test_fedora_and_epel_branches_are_filtered_out(self, get): + def test_fedora_and_epel_branches_are_filtered_out(self, get, pagure_branches): + logger = Mock() + apibaseurl = "https://bodhiurl" rv = Mock(ok=True) - rv.json.return_value = { - 'results': [ - {'name': '8'}, - {'name': '10'}, - {'name': 'f28'}, - {'name': 'epel7'}, - {'name': 'rawhide'}, - {'name': 'epel8'}, - {'name': 'epel8-playground'} - ], - 'next': None - } + rv.json.return_value = {'releases': [ + {'name': 'ELN', 'branch': 'eln'}, + {'name': 'F40', 'branch': 'rawhide'}, + {'name': 'F40C', 'branch': 'f40'}, + {'name': 'epel8', 'branch': 'epel8'}, + ], 'page': 1, 'pages': 1, 'rows_per_page': 20, 'total': 3} + {'releases': [], 'page': 1, 'pages': 0, 'rows_per_page': 20, 'total': 0} get.return_value = rv + pagure_branches.return_value = ["epel7", "epel8", "epel9", "f38", "f39"] - result = utils.get_stream_branches('http://localhost/', 'pkg') - self.assertEqual([{'name': '8'}, {'name': '10'}, {'name': 'epel8'}], list(result)) - + result = utils.get_stream_branches('http://localhost/', 'pkg', apibaseurl, logger) + self.assertEqual(['epel8'], list(result)) class TestExpandRelease(unittest.TestCase): """Test expand_release"""