From e53c66c03f49b48045ef83bda3476af465f7ae55 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Oct 02 2017 11:51:02 +0000 Subject: Move lightblue filter_fnc filtering before we call image.resolve_commit() to save some time by not resolving images we would filter out later. --- diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index c039ba8..789ffcf 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -433,7 +433,9 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): builds = builds or {} nvrs = errata.get_builds(errata_id) for nvr in nvrs: - if nvr.endswith(".rpm"): + # Container images builds end with ".tar.gz", so do not treat + # them as RPMs here. + if not nvr.endswith(".tar.gz"): srpm_name = self._find_build_srpm_name(nvr) batches = lb.find_images_to_rebuild( srpm_name, content_sets, diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 61c95ca..6218814 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -606,13 +606,19 @@ class LightBlue(object): def find_images_with_package_from_content_set( self, srpm_name, content_sets, published=True, deprecated=False, - release_category="Generally Available"): + release_category="Generally Available", filter_fnc=None): """Query lightblue and find containers which contain given package from one of content sets :param str srpm_name: srpm_name (source rpm name) to look for :param list content_sets: list of strings (content sets) to consider when looking for the packages + :param function filter_fnc: Function called as + filter_fnc(container_image) with container_image being + ContainerImage instance. If this function returns True, the image + will not be considered for a rebuild as well as its parent images. + This function is used to filter out images not allowed by + Freshmaker configuration. :return: a list of dictionaries with three keys - repository, commit and srpm_nevra. Repository is a name git repository including the @@ -630,6 +636,11 @@ class LightBlue(object): images = self.find_images_with_included_srpm(repos, srpm_name, published=published) + + # Filter out images based on the filter_fnc. + if filter_fnc: + images = [image for image in images if not filter_fnc(image)] + for image in images: image.resolve_commit(srpm_name) return images @@ -657,11 +668,8 @@ class LightBlue(object): Freshmaker configuration. """ images = self.find_images_with_package_from_content_set( - srpm_name, content_sets, published, deprecated, release_category) - - # Filter out images based on the filter_fnc. - if filter_fnc: - images = [image for image in images if not filter_fnc(image)] + srpm_name, content_sets, published, deprecated, release_category, + filter_fnc=filter_fnc) def _get_images_to_rebuild(image): """ diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index 98146f7..cd9de58 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -609,6 +609,9 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): cont_images.assert_called_with(expected_image_request) self.assertEqual(ret, cont_images.return_value) + def _filter_fnc(self, image): + return image["brew"]["build"].startswith("filtered_") + @patch('freshmaker.lightblue.LightBlue.find_container_repositories') @patch('freshmaker.lightblue.LightBlue.find_container_images') @patch('freshmaker.kojiservice.KojiService.get_build') @@ -620,15 +623,17 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): exists.return_value = True cont_repos.return_value = self.fake_repositories_with_content_sets - cont_images.return_value = self.fake_container_images + # "filtered_x-1-23" image will be filtered by filter_fnc. + cont_images.return_value = self.fake_container_images + [ + ContainerImage.create({"brew": {"build": "filtered_x-1-23"}})] koji_task_request.side_effect = self.fake_koji_task_requests koji_get_build.side_effect = self.fake_koji_builds lb = LightBlue(server_url=self.fake_server_url, cert=self.fake_cert_file, private_key=self.fake_private_key) - ret = lb.find_images_with_package_from_content_set("openssl", - ["dummy-content-set-1"]) + ret = lb.find_images_with_package_from_content_set( + "openssl", ["dummy-content-set-1"], filter_fnc=self._filter_fnc) self.assertEqual(2, len(ret)) self.assertEqual(ret, @@ -727,9 +732,6 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): self.assertEqual(1, len(ret)) self.assertEqual(ret[0]["brew"]["package"], "package-name-1") - def _filter_fnc(self, image): - return image["brew"]["build"].startswith("filtered_") - @patch('freshmaker.lightblue.LightBlue.find_images_with_package_from_content_set') @patch('freshmaker.lightblue.LightBlue.find_parent_images_with_package') @patch('freshmaker.lightblue.LightBlue.find_unpublished_image_for_build') @@ -743,12 +745,8 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): "parsed_data": {"layers": None}}) child2 = ContainerImage.create({'brew': {'package': 'child2', 'build': 'child2'}, "parsed_data": {"layers": None}}) - # This "filtered_child" will be filtered by self._filter_fnc. - filtered_child = ContainerImage.create( - {'brew': {'package': 'filtered_child', 'build': 'filtered_child'}, - "parsed_data": {"layers": None}}) - cont_images.return_value = [child1, child2, filtered_child] - unpublished_image.side_effect = [child1, child2, filtered_child] + cont_images.return_value = [child1, child2] + unpublished_image.side_effect = [child1, child2] child1_parent1 = ContainerImage.create( {'brew': {'package': 'child1_parent1', 'build': 'child1_parent1'}}) @@ -780,8 +778,7 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): lb = LightBlue(server_url=self.fake_server_url, cert=self.fake_cert_file, private_key=self.fake_private_key) - ret = lb.find_images_to_rebuild("dummy", "dummy", - filter_fnc=self._filter_fnc) + ret = lb.find_images_to_rebuild("dummy", "dummy") self.assertEqual([len(x) for x in ret], [1, 2, 2, 1, 1, 1]) self.assertEqual(set(ret[0]), set([child1_parent4])) self.assertEqual(set(ret[1]), set([child1_parent3, child2_parent2]))