From b97780ebab4284fc6dcc8b0b429aa8f600b423e5 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Aug 09 2017 11:25:09 +0000 Subject: Add handler for brew build container task state change event In BrewSignRpmHandler, we record all the planned Docker image builds into database. They have the state PLANNED and they have the ArtifactBuild.dep_on set properly according to dependencies between these images. When the build container task state is changed in brew, we should update the build state in db, and if the task closed successfully, we should rebuild containers that depends on it. At this moment, we only log the builds which need to be rebuilt. --- diff --git a/conf/configrh.py b/conf/configrh.py index 0922d7e..82980ba 100644 --- a/conf/configrh.py +++ b/conf/configrh.py @@ -22,10 +22,12 @@ class BaseConfiguration(config.BaseConfiguration): PARSERS = [ 'freshmaker.parsers.brew.sign_rpm:BrewSignRpmParser', + 'freshmaker.parsers.brew:BrewTaskStateChangeParser', ] HANDLERS = [ 'freshmaker.handlers.brew:BrewSignRPMHandler', + 'freshmaker.handlers.brew:BrewContainerTaskStateChangeHandler', ] KOJI_PROFILE = 'brew' diff --git a/freshmaker/events.py b/freshmaker/events.py index 4aba462..aa8a55f 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -260,3 +260,21 @@ class BrewSignRPMEvent(BaseEvent): @property def search_key(self): return str(self.nvr) + + +class BrewContainerTaskStateChangeEvent(BaseEvent): + """ + Represents the message sent by Brew when a container task state is changed. + """ + def __init__(self, msg_id, container, branch, target, task_id, old_state, new_state): + super(BrewContainerTaskStateChangeEvent, self).__init__(msg_id) + self.container = container + self.branch = branch + self.target = target + self.task_id = task_id + self.old_state = old_state + self.new_state = new_state + + @property + def search_key(self): + return str(self.task_id) diff --git a/freshmaker/handlers/brew/__init__.py b/freshmaker/handlers/brew/__init__.py index 23428e4..c77742c 100644 --- a/freshmaker/handlers/brew/__init__.py +++ b/freshmaker/handlers/brew/__init__.py @@ -20,3 +20,4 @@ # SOFTWARE. from .sign_rpm import BrewSignRPMHandler # noqa +from .container_task_state_change import BrewContainerTaskStateChangeHandler # noqa diff --git a/freshmaker/handlers/brew/container_task_state_change.py b/freshmaker/handlers/brew/container_task_state_change.py new file mode 100644 index 0000000..5fd26f2 --- /dev/null +++ b/freshmaker/handlers/brew/container_task_state_change.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker import log +from freshmaker import db +from freshmaker.events import BrewContainerTaskStateChangeEvent +from freshmaker.models import ArtifactBuild +from freshmaker.handlers import BaseHandler +from freshmaker.types import ArtifactType, ArtifactBuildState + + +class BrewContainerTaskStateChangeHandler(BaseHandler): + """Rebuild container when a dependecy container is built in Brew""" + + name = 'BrewContainerTaskStateChangeHandler' + + def can_handle(self, event): + return isinstance(event, BrewContainerTaskStateChangeEvent) + + def handle(self, event): + """ + When build container task state changed in brew, update build state in db and + rebuild containers depend on the success build as necessary. + """ + + build_id = event.task_id + + # check db to see whether this build exists in db + found_build = db.session.query(ArtifactBuild).filter_by(type=ArtifactType.IMAGE.value, + build_id=build_id).one_or_none() + if found_build is not None: + # update build state in db + if event.new_state == 'CLOSED': + found_build.state = ArtifactBuildState.DONE.value + if event.new_state == 'FAILED': + found_build.state = ArtifactBuildState.FAILED.value + db.session.commit() + + if found_build.state == ArtifactBuildState.DONE.value: + # check db to see whether there is any planned image build depends on this build + planned_builds = db.session.query(ArtifactBuild).filter_by(type=ArtifactType.IMAGE.value, + state=ArtifactBuildState.PLANNED.value, + dep_on=found_build).all() + for build in planned_builds: + # TODO: enable rebuild for these containers + log.info("Build %s depends on build %s" % (str(build), str(found_build))) diff --git a/freshmaker/parsers/brew/__init__.py b/freshmaker/parsers/brew/__init__.py index e69de29..e641469 100644 --- a/freshmaker/parsers/brew/__init__.py +++ b/freshmaker/parsers/brew/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .sign_rpm import BrewSignRpmParser # noqa +from .task_state_change import BrewTaskStateChangeParser # noqa diff --git a/freshmaker/parsers/brew/task_state_change.py b/freshmaker/parsers/brew/task_state_change.py new file mode 100644 index 0000000..c97e4f8 --- /dev/null +++ b/freshmaker/parsers/brew/task_state_change.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import re + +from freshmaker.parsers import BaseParser +from freshmaker.events import BrewContainerTaskStateChangeEvent + + +class BrewTaskStateChangeParser(BaseParser): + """ + Parser parsing task state change message from Brew. + + Unlike koji, Brew sends such messages with topics of 'brew.task.closed' + and 'brew.task.failed'. + """ + + name = "BrewTaskStateChangeParser" + topic_suffixes = ["eng.brew.task.closed", 'eng.brew.task.failed'] + + def can_parse(self, topic, msg): + return any([topic.endswith(s) for s in self.topic_suffixes]) + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + inner_msg = msg.get('msg') + old_state = inner_msg.get('old') + new_state = inner_msg.get('new') + task_info = inner_msg.get('info', {}) + task_id = task_info.get('id') + task_method = task_info.get('method') + + if task_method == 'buildContainer': + request = task_info.get('request') + (git_url, target, opts) = request + branch = opts.get('git_branch', None) + m = re.match(r".*/(?P[^#]*)", git_url) + container = m.group('container') + return BrewContainerTaskStateChangeEvent(msg_id, container, branch, target, task_id, old_state, new_state) diff --git a/tests/fedmsgs/brew_container_task_closed b/tests/fedmsgs/brew_container_task_closed new file mode 100644 index 0000000..dcc3138 --- /dev/null +++ b/tests/fedmsgs/brew_container_task_closed @@ -0,0 +1,74 @@ +{ + "username": null, + "source_name": "datanommer", + "certificate": null, + "i": 0, + "timestamp": 1501051181.0, + "msg_id": "ID:messaging-example.com-44140-1499406995819-2:36670:0:0:1", + "crypto": null, + "topic": "/topic/VirtualTopic.eng.brew.task.closed", + "headers": { + "content-length": "1313", + "expires": "0", + "old": "OPEN", + "JMS_AMQP_MESSAGE_FORMAT": "0", + "parent": "null", + "JMS_AMQP_NATIVE": "false", + "destination": "/topic/VirtualTopic.eng.brew.task.closed", + "method": "buildContainer", + "priority": "4", + "message-id": "ID:messaging-example.com-44140-1499406995819-2:36670:0:0:1", + "timestamp": "0", + "attribute": "state", + "new": "CLOSED", + "JMS_AMQP_FirstAcquirer": "false", + "type": "TaskStateChange", + "id": "20124784", + "subscription": "/queue/Consumer.client-datanommer.mikeb-dev.VirtualTopic.eng.>" + }, + "signature": null, + "source_version": "0.7.0", + "msg": { + "info": { + "weight": 2.0, + "parent": null, + "completion_time": "2017-07-26 06:39:54.907075", + "start_ts": 1501050630.82726, + "start_time": "2017-07-26 06:30:30.827262", + "request": [ + "git://dist-git.example.com/rpms/test-product-docker#d3670cde469aca43f1ff6768257420eb80e147db", + "rhel-docker-candidate", + { + "scratch": false, + "yum_repourls": null, + "git_branch": "rhel-docker" + } + ], + "waiting": null, + "awaited": null, + "label": null, + "priority": 20, + "channel_id": 20, + "state": 2, + "create_time": "2017-07-26 06:30:18.225661", + "create_ts": 1501050618.22566, + "owner": 3447, + "host_id": 191, + "method": "buildContainer", + "completion_ts": 1501051194.90707, + "arch": "noarch", + "id": 20124784, + "result": { + "repositories": [ + "docker.example.com:8888/rhel7/test-product-docker:rhel-docker-candidate-10587-20170726063100" + ], + "koji_builds": [ + "1016224" + ] + } + }, + "attribute": "state", + "old": "OPEN", + "new": "CLOSED" + } +} diff --git a/tests/fedmsgs/brew_container_task_failed b/tests/fedmsgs/brew_container_task_failed new file mode 100644 index 0000000..18fae3d --- /dev/null +++ b/tests/fedmsgs/brew_container_task_failed @@ -0,0 +1,74 @@ +{ + "username": null, + "source_name": "datanommer", + "certificate": null, + "i": 0, + "timestamp": 1501051181.0, + "msg_id": "ID:messaging-example.com-44140-1499406995819-2:36670:0:0:1", + "crypto": null, + "topic": "/topic/VirtualTopic.eng.brew.task.closed", + "headers": { + "content-length": "1313", + "expires": "0", + "old": "OPEN", + "JMS_AMQP_MESSAGE_FORMAT": "0", + "parent": "null", + "JMS_AMQP_NATIVE": "false", + "destination": "/topic/VirtualTopic.eng.brew.task.closed", + "method": "buildContainer", + "priority": "4", + "message-id": "ID:messaging-example.com-44140-1499406995819-2:36670:0:0:1", + "timestamp": "0", + "attribute": "state", + "new": "CLOSED", + "JMS_AMQP_FirstAcquirer": "false", + "type": "TaskStateChange", + "id": "20124784", + "subscription": "/queue/Consumer.client-datanommer.mikeb-dev.VirtualTopic.eng.>" + }, + "signature": null, + "source_version": "0.7.0", + "msg": { + "info": { + "weight": 2.0, + "parent": null, + "completion_time": "2017-07-26 06:39:54.907075", + "start_ts": 1501050630.82726, + "start_time": "2017-07-26 06:30:30.827262", + "request": [ + "git://dist-git.example.com/rpms/test-product-docker#d3670cde469aca43f1ff6768257420eb80e147db", + "rhel-docker-candidate", + { + "scratch": false, + "yum_repourls": null, + "git_branch": "rhel-docker" + } + ], + "waiting": null, + "awaited": null, + "label": null, + "priority": 20, + "channel_id": 20, + "state": 2, + "create_time": "2017-07-26 06:30:18.225661", + "create_ts": 1501050618.22566, + "owner": 3447, + "host_id": 191, + "method": "buildContainer", + "completion_ts": 1501051194.90707, + "arch": "noarch", + "id": 20124784, + "result": { + "repositories": [ + "docker.example.com:8888/rhel7/test-product-docker:rhel-docker-candidate-10587-20170726063100" + ], + "koji_builds": [ + "1016224" + ] + } + }, + "attribute": "state", + "old": "OPEN", + "new": "FAILED" + } +} diff --git a/tests/test_brew_container_task_state_change_handler.py b/tests/test_brew_container_task_state_change_handler.py new file mode 100644 index 0000000..f75b45d --- /dev/null +++ b/tests/test_brew_container_task_state_change_handler.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import mock +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # noqa +from tests import get_fedmsg, helpers + +from freshmaker import db, events, models +from freshmaker.parsers.brew import BrewTaskStateChangeParser +from freshmaker.handlers.brew import BrewContainerTaskStateChangeHandler +from freshmaker.types import ArtifactType, ArtifactBuildState + + +class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + events.BaseEvent.register_parser(BrewTaskStateChangeParser) + self.handler = BrewContainerTaskStateChangeHandler() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_can_handle_brew_container_task_closed_event(self): + """ + Tests handler can handle brew build container task closed event. + """ + event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) + self.assertTrue(self.handler.can_handle(event)) + + def test_can_handle_brew_container_task_failed_event(self): + """ + Tests handler can handle brew build container task failed event. + """ + event = self.get_event_from_msg(get_fedmsg('brew_container_task_failed')) + self.assertTrue(self.handler.can_handle(event)) + + @mock.patch('freshmaker.handlers.brew.container_task_state_change.log') + def test_build_containers_when_dependency_container_is_built(self, log): + """ + Tests when dependency container is built, rebuild containers depend on it. + """ + e1 = models.Event.create(db.session, "test_msg_id", "RHSA-2018-001", events.TestingEvent) + event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) + + base_build = models.ArtifactBuild.create(db.session, e1, 'test-product-docker', ArtifactType.IMAGE.value, event.task_id) + + build_0 = models.ArtifactBuild.create(db.session, e1, 'docker-up-0', ArtifactType.IMAGE.value, 0, + dep_on=base_build, state=ArtifactBuildState.PLANNED.value) + build_1 = models.ArtifactBuild.create(db.session, e1, 'docker-up-1', ArtifactType.IMAGE.value, 0, + dep_on=base_build, state=ArtifactBuildState.PLANNED.value) + build_2 = models.ArtifactBuild.create(db.session, e1, 'docker-up-2', ArtifactType.IMAGE.value, 0, + dep_on=base_build, state=ArtifactBuildState.PLANNED.value) + + self.handler.handle(event) + self.assertEqual(base_build.state, ArtifactBuildState.DONE.value) + # we only log the builds at this moment + log.info.assert_has_calls([ + mock.call('Build %s depends on build %s' % (str(build_0), str(base_build))), + mock.call('Build %s depends on build %s' % (str(build_1), str(base_build))), + mock.call('Build %s depends on build %s' % (str(build_2), str(base_build))), + ]) + + @mock.patch('freshmaker.handlers.brew.container_task_state_change.log') + def test_not_build_containers_when_dependency_container_build_task_failed(self, log): + """ + Tests when dependency container build task failed in brew, only update build state in db. + """ + e1 = models.Event.create(db.session, "test_msg_id", "RHSA-2018-001", events.TestingEvent) + event = self.get_event_from_msg(get_fedmsg('brew_container_task_failed')) + + base_build = models.ArtifactBuild.create(db.session, e1, 'test-product-docker', ArtifactType.IMAGE.value, event.task_id) + + models.ArtifactBuild.create(db.session, e1, 'docker-up', ArtifactType.IMAGE.value, 0, + dep_on=base_build, state=ArtifactBuildState.PLANNED.value) + self.handler.handle(event) + self.assertEqual(base_build.state, ArtifactBuildState.FAILED.value) + log.info.assert_not_called() + + +if __name__ == '__main__': + unittest.main()