From 4a7c8dfbd94bf5cadc8219caa377c98d272b582d Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Jun 07 2017 03:12:49 +0000 Subject: Connect to UMB and parse brew.sign.rpm UMB related configuration are put into dedicated config files, fedmsg.d/freshmaker-rh.py.disable and conf/configrh.py. Name freshmaker-rh.py.disable because fedmsg-hub listens on fedmsg by default when start freshmaker. When need to connect to UMB, freshmaker.py should have same content as freshmaker-rh.py.disable. Configure fedmsg-hub to use STOMP to connect to UMB and receive messages. Broker URLs, certificate and private key files can be configured via several new FRESHMAKER_STOMP_* environment variables. * FRESHMAKER_STOMP_URI, URIs of brokers separated by comma. * FRESHMAKER_STOMP_SSL_CRT, path to certificate file. * FRESHMAKER_STOMP_SSL_KEY, path to private key file. * FRESHMAKER_MESSAGING_TOPIC_PREFIX, part of queue name from the beginning to word VirtualTopic, e.g. /queue/Consumer.client-me.queue.VirtualTopic Major changes: * Make event parsers configurable along with handlers in config * Reuse load_classes to load event parsers and handlers from config * Modify several config to add internal configuration for event parsers and handlers * BrewRPMSignEvent is renamed to BrewSignRPMEvent, which is natrual to follow the part of topic name, that is brew.sign.rpm * Add event parse for BrewSignRPMEvent Signed-off-by: Chenxiong Qi So no need of environment variable FRESHMAKER_INSIDE_RH Signed-off-by: Chenxiong Qi --- diff --git a/conf/config.py b/conf/config.py index c1445af..90f890c 100644 --- a/conf/config.py +++ b/conf/config.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from os import path @@ -26,7 +28,6 @@ class BaseConfiguration(object): SYSTEM = 'koji' MESSAGING = 'fedmsg' # or amq - MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] PDC_URL = 'http://modularity.fedorainfracloud.org:8080/rest_api/v1' PDC_INSECURE = True PDC_DEVELOP = True @@ -40,6 +41,16 @@ class BaseConfiguration(object): # Available log levels are: debug, info, warn, error. LOG_LEVEL = 'info' + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] + + # Parsers defined for parse specific messages + PARSERS = [ + 'freshmaker.parsers.bodhi:BodhiUpdateCompleteStableParser', + 'freshmaker.parsers.git:GitReceiveParser', + 'freshmaker.parsers.koji:KojiTaskStateChangeParser', + 'freshmaker.parsers.mbs:MBSModuleStateChangeParser', + ] + # List of enabled composing handlers. HANDLERS = [ "freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler", diff --git a/conf/configrh.py b/conf/configrh.py new file mode 100644 index 0000000..cd80b17 --- /dev/null +++ b/conf/configrh.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +import os + +from conf import config + + +class BaseConfiguration(config.BaseConfiguration): + MESSAGING_TOPIC_PREFIX = [ + # This is the queue name to receive messages from UMB. + # Generally, it has format Consumer.client-[name].*.VirtualTopic.> + # + # - name is deteremined by the certificate requested. For example, a + # certificate is requested with client name msg-client-bob, then in + # queue name, it name should be bob, that is Consumer.client-bob. + # + # - * is any word you choose, which should be descriptive to this queue + # + # - > represents the hierarchy of topic name, e.g. eng.brew.sign.rpm + os.environ['FRESHMAKER_MESSAGING_TOPIC_PREFIX'], + ] + + PARSERS = [ + 'freshmaker.parsers.brew.sign_rpm:BrewSignRpmParser', + ] + + HANDLERS = [ + ] + + +class DevConfiguration(BaseConfiguration): + DEBUG = True + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_VERIFY_SSL = False + + +class TestConfiguration(BaseConfiguration): + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + DEBUG = True + + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format( + os.path.join(config.dbdir, 'tests', 'test_freshmaker.db')) + + MESSAGING = 'in_memory' + PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + MBS_AUTH_TOKEN = "testingtoken" + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_SERVER_URL = '' # replace with real dev server url + LIGHTBLUE_VERIFY_SSL = False diff --git a/fedmsg.d/freshmaker-rh.py.disabled b/fedmsg.d/freshmaker-rh.py.disabled new file mode 100644 index 0000000..49bfdfa --- /dev/null +++ b/fedmsg.d/freshmaker-rh.py.disabled @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +import os + +config = { + 'zmq_enabled': False, + 'stomp_heartbeat': 1000, + 'stomp_uri': os.environ['FRESHMAKER_STOMP_URI'], + 'stomp_ssl_crt': os.environ['FRESHMAKER_STOMP_SSL_CRT'], + 'stomp_ssl_key': os.environ['FRESHMAKER_STOMP_SSL_KEY'], + 'validate_signatures': False, +} diff --git a/fedmsg.d/freshmaker.py b/fedmsg.d/freshmaker.py index c46ea1d..7f958ab 100644 --- a/fedmsg.d/freshmaker.py +++ b/fedmsg.d/freshmaker.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import os config = { diff --git a/freshmaker/config.py b/freshmaker/config.py index 8c0e2c1..7e02826 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -50,9 +50,11 @@ def init_config(app): # try getting config_file from os.environ if 'FRESHMAKER_CONFIG_FILE' in os.environ: config_file = os.environ['FRESHMAKER_CONFIG_FILE'] + # try getting config_section from os.environ if 'FRESHMAKER_CONFIG_SECTION' in os.environ: config_section = os.environ['FRESHMAKER_CONFIG_SECTION'] + # TestConfiguration shall only be used for running tests, otherwise... if any(['nosetests' in arg or 'noserunner.py' in arg or 'py.test' in arg or 'pytest.py' in arg for arg in sys.argv]): config_section = 'TestConfiguration' @@ -64,17 +66,22 @@ def init_config(app): # package -> /conf/config.py. elif ('FRESHMAKER_DEVELOPER_ENV' in os.environ and - os.environ['FRESHMAKER_DEVELOPER_ENV'].lower() in ( - '1', 'on', 'true', 'y', 'yes')): + os.environ['FRESHMAKER_DEVELOPER_ENV'].lower() in ('1', 'on', 'true', 'y', 'yes')): config_section = 'DevConfiguration' - from conf import config - config_module = config + if 'FRESHMAKER_CONFIG_FILE' in os.environ: + config_file = os.environ['FRESHMAKER_CONFIG_FILE'] + config_module = None + else: + from conf import config + config_module = config + # try loading configuration from file if not config_module: try: config_module = imp.load_source('freshmaker_runtime_config', config_file) except: + raise raise SystemError("Configuration file {} was not found." .format(config_file)) @@ -132,6 +139,10 @@ class Config(object): 'type': int, 'default': 30, 'desc': 'Global network retry interval for read/write operations, in seconds.'}, + 'parsers': { + 'type': list, + 'default': [], + 'desc': 'Parsers defined for parse specific messages.'}, 'handlers': { 'type': list, 'default': ["freshmaker.handlers.mbs:MBSModuleStateChangeHandler"], diff --git a/freshmaker/consumer.py b/freshmaker/consumer.py index 9c81a4b..c3ffca3 100644 --- a/freshmaker/consumer.py +++ b/freshmaker/consumer.py @@ -27,13 +27,9 @@ to use. import fedmsg.consumers import moksha.hub -import freshmaker.handlers -import freshmaker.parsers.mbs -import freshmaker.parsers.git -import freshmaker.parsers.bodhi -import freshmaker.parsers.koji from freshmaker import log, conf, messaging, events +from freshmaker.utils import load_classes class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): @@ -45,7 +41,6 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): def __init__(self, hub): # set topic before super, otherwise topic will not be subscribed - self.handlers = list(freshmaker.handlers.load_handlers()) self.register_parsers() super(FreshmakerConsumer, self).__init__(hub) @@ -63,10 +58,9 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): self.incoming.put(msg) def register_parsers(self): - events.BaseEvent.register_parser(freshmaker.parsers.bodhi.BodhiUpdateCompleteStableParser) - events.BaseEvent.register_parser(freshmaker.parsers.git.GitReceiveParser) - events.BaseEvent.register_parser(freshmaker.parsers.koji.KojiTaskStateChangeParser) - events.BaseEvent.register_parser(freshmaker.parsers.mbs.MBSModuleStateChangeParser) + parser_classes = load_classes(conf.parsers) + for parser_class in parser_classes: + events.BaseEvent.register_parser(parser_class) log.debug("Parser classes: %r", events.BaseEvent._parsers) self.topic = events.BaseEvent.get_parsed_topics() @@ -112,19 +106,19 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): def get_abstracted_msg(self, message): # Convert the message to an abstracted message - if conf.messaging == 'fedmsg' or conf.messaging == 'in_memory': - msg = events.BaseEvent.from_fedmsg( - message['topic'], message) - else: - raise ValueError('The messaging format "{0}" is not supported' - .format(conf.messaging)) - return msg + if 'topic' not in message: + raise ValueError( + 'The messaging format "{}" is not supported'.format(conf.messaging)) + + return events.BaseEvent.from_fedmsg(message['topic'], message) def process_event(self, msg): log.debug('Received a message with an ID of "{0}" and of type "{1}"' .format(getattr(msg, 'msg_id', None), type(msg).__name__)) - for handler in self.handlers: + for handler_class in load_classes(conf.handlers): + handler = handler_class() + if not handler.can_handle(msg): continue diff --git a/freshmaker/errata.py b/freshmaker/errata.py index d60cc73..671647b 100644 --- a/freshmaker/errata.py +++ b/freshmaker/errata.py @@ -24,7 +24,7 @@ import requests from requests_kerberos import HTTPKerberosAuth -from freshmaker.events import BrewRPMSignEvent +from freshmaker.events import BrewSignRPMEvent class ErrataAdvisory(object): @@ -87,7 +87,7 @@ class Errata(object): :return: List of ErrataAdvisory instances :rtype: list """ - if isinstance(event, BrewRPMSignEvent): + if isinstance(event, BrewSignRPMEvent): build = self._errata_rest_get("/build/%s" % str(event.nvr)) if "all_errata" not in build: return [] diff --git a/freshmaker/events.py b/freshmaker/events.py index b9ad881..d056b0c 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -249,12 +249,12 @@ class KojiTaskStateChangeEvent(BaseEvent): self.task_state = task_state -class BrewRPMSignEvent(BaseEvent): +class BrewSignRPMEvent(BaseEvent): """ Represents the message sent by Brew when RPM is signed. """ def __init__(self, msg_id, nvr): - super(BrewRPMSignEvent, self).__init__(msg_id) + super(BrewSignRPMEvent, self).__init__(msg_id) self.nvr = nvr @property diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 383fa70..bc1456a 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -23,35 +23,12 @@ import abc import re -import sys from freshmaker import conf, log, db, models from freshmaker.mbs import MBS from freshmaker.kojiservice import koji_service -def load_class(location): - """ Take a string of the form 'fedmsg.consumers.ircbot:IRCBotConsumer' - and return the IRCBotConsumer class. - """ - mod_name, cls_name = location.strip().split(':') - - __import__(mod_name) - - try: - return getattr(sys.modules[mod_name], cls_name) - except AttributeError: - raise ImportError("%r not found in %r" % (cls_name, mod_name)) - - -def load_handlers(): - """ Import and instantiate all handlers listed in the given config. """ - for import_path in conf.handlers: - cls = load_class(import_path) - handler = cls() - yield handler - - class BaseHandler(object): """ Abstract base class for event handlers. diff --git a/freshmaker/parsers/brew/__init__.py b/freshmaker/parsers/brew/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/freshmaker/parsers/brew/__init__.py diff --git a/freshmaker/parsers/brew/sign_rpm.py b/freshmaker/parsers/brew/sign_rpm.py new file mode 100644 index 0000000..ceff2ac --- /dev/null +++ b/freshmaker/parsers/brew/sign_rpm.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +from freshmaker.parsers import BaseParser +from freshmaker.events import BrewSignRPMEvent + + +class BrewSignRpmParser(BaseParser): + """Parser parsing message from Brew""" + + name = "BrewSignRpmParser" + topic_suffixes = ["eng.brew.sign.rpm"] + + def can_parse(self, topic, msg): + return any([topic.endswith(s) for s in self.topic_suffixes]) + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + return BrewSignRPMEvent(msg_id, msg_inner_msg['build']['nvr']) diff --git a/freshmaker/utils.py b/freshmaker/utils.py index 431382e..5709531 100644 --- a/freshmaker/utils.py +++ b/freshmaker/utils.py @@ -27,12 +27,35 @@ import getpass import os import shutil import subprocess +import sys import tempfile import time from freshmaker import conf +def load_class(location): + """ Take a string of the form 'fedmsg.consumers.ircbot:IRCBotConsumer' + and return the IRCBotConsumer class. + """ + try: + mod_name, cls_name = location.strip().split(':') + except ValueError: + raise ImportError('Invalid import path.') + + __import__(mod_name) + + try: + return getattr(sys.modules[mod_name], cls_name) + except AttributeError: + raise ImportError("%r not found in %r" % (cls_name, mod_name)) + + +def load_classes(import_paths): + """Load classes from given paths""" + return [load_class(import_path) for import_path in import_paths] + + def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception, logger=None): """A decorator that allows to retry a section of code until success or timeout.""" def wrapper(function): diff --git a/tests/test_consumer.py b/tests/test_consumer.py index 8087d55..9692a40 100644 --- a/tests/test_consumer.py +++ b/tests/test_consumer.py @@ -18,19 +18,25 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import unittest -import mock import fedmsg.config +import mock +import unittest import freshmaker +from freshmaker.events import BrewSignRPMEvent -class ConsumerTest(unittest.TestCase): - def setUp(self): - pass - def tearDown(self): - pass +class ConsumerBaseTest(unittest.TestCase): + + def _create_consumer(self): + hub = mock.MagicMock() + hub.config = fedmsg.config.load_config() + hub.config['freshmakerconsumer'] = True + return freshmaker.consumer.FreshmakerConsumer(hub) + + +class ConsumerTest(ConsumerBaseTest): @mock.patch("freshmaker.handlers.mbs.module_state_change.MBSModuleStateChangeHandler.handle") @mock.patch("freshmaker.consumer.get_global_consumer") @@ -40,10 +46,7 @@ class ConsumerTest(unittest.TestCase): to proper handler and is able to get the further work from the handler. """ - hub = mock.MagicMock() - hub.config = fedmsg.config.load_config() - hub.config['freshmakerconsumer'] = True - consumer = freshmaker.consumer.FreshmakerConsumer(hub) + consumer = self._create_consumer() global_consumer.return_value = consumer msg = {'body': { @@ -68,14 +71,37 @@ class ConsumerTest(unittest.TestCase): """ Tests consumer will try to subscribe specified topics. """ - hub = mock.MagicMock() - hub.config = fedmsg.config.load_config() - consumer = freshmaker.consumer.FreshmakerConsumer(hub) + consumer = self._create_consumer() global_consumer.return_value = consumer topics = freshmaker.events.BaseEvent.get_parsed_topics() callback = consumer._consume_json if consumer.jsonify else consumer.consume for topic in topics: - self.assertIn(mock.call(topic, callback), hub.subscribe.call_args_list) + self.assertIn(mock.call(topic, callback), consumer.hub.subscribe.call_args_list) + + +class ParseBrewSignRPMEventTest(ConsumerBaseTest): + + @mock.patch('freshmaker.events.conf.parsers', + new=['freshmaker.parsers.brew.sign_rpm:BrewSignRpmParser']) + @mock.patch("freshmaker.consumer.get_global_consumer") + def test_get_internal_event_parser(self, get_global_consumer): + consumer = self._create_consumer() + get_global_consumer.return_value = consumer + + msg = { + 'msg_id': 'fake-msg-id', + 'topic': '/topic/VirtualTopic.eng.brew.sign.rpm', + 'msg': { + 'build': { + 'id': 562101, + 'nvr': 'openshift-ansible-3.3.1.32-1.git.0.3b74dea.el7', + } + } + } + msg = consumer.get_abstracted_msg(msg) + self.assertIsInstance(msg, BrewSignRPMEvent) + self.assertEqual('fake-msg-id', msg.msg_id) + self.assertEqual('openshift-ansible-3.3.1.32-1.git.0.3b74dea.el7', msg.nvr) if __name__ == '__main__': diff --git a/tests/test_errata.py b/tests/test_errata.py index 2d1a5dd..d08dfe1 100644 --- a/tests/test_errata.py +++ b/tests/test_errata.py @@ -25,7 +25,7 @@ import unittest from mock import patch from freshmaker.errata import Errata -from freshmaker.events import BrewRPMSignEvent, GitRPMSpecChangeEvent +from freshmaker.events import BrewSignRPMEvent, GitRPMSpecChangeEvent class MockedErrataRESTAPI(object): @@ -81,7 +81,7 @@ class TestErrata(unittest.TestCase): @patch.object(Errata, "_errata_rest_get") def test_advisories_from_event(self, errata_rest_get): MockedErrataRESTAPI(errata_rest_get) - event = BrewRPMSignEvent("msgid", "libntirpc-1.4.3-4.el7rhgs") + event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs") advisories = self.errata.advisories_from_event(event) self.assertEqual(len(advisories), 1) self.assertEqual(advisories[0].errata_id, 28484) @@ -91,7 +91,7 @@ class TestErrata(unittest.TestCase): mocked_errata = MockedErrataRESTAPI(errata_rest_get) del mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"]["all_errata"] - event = BrewRPMSignEvent("msgid", "libntirpc-1.4.3-4.el7rhgs") + event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs") advisories = self.errata.advisories_from_event(event) self.assertEqual(len(advisories), 0)