From fbcc84f20fb2b24adfc78bdaab0dd8705a9abf7f Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jul 30 2019 09:56:26 +0000 Subject: Migrate to Fedora Messaging Signed-off-by: Aurélien Bompard --- diff --git a/MANIFEST.in b/MANIFEST.in index daebeb0..00c43b4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ include LICENSE include README.rst - -graft fedmsg.d +include robosignatory.toml diff --git a/README.rst b/README.rst index 16bead7..67d5d7c 100644 --- a/README.rst +++ b/README.rst @@ -2,14 +2,27 @@ robosignatory ============= -A fedmsg consumer that automatically signs artifacts. +A Fedora Messaging consumer that automatically signs artifacts. RoboSignatory is composed of multiple consumers: -- TagConsumer is a consumer that listens for tags into a specific koji tag, - then signs the build and moves it to a different koji tag. -- AtomicConsumer is a consumer that listens for messages about composed - rpmostree trees and signs those, optionally updating the tag. +- TagSigner listens for tags into a specific koji tag, then signs the build and + moves it to a different koji tag. +- AtomicSigner listens for messages about composed rpmostree trees and signs + those, optionally updating the tag. +- CoreOSSigner listens for requests to sign CoreOS artefacts, downloads them + from AWS S3, signs them, and uploads the signature back to S3. + +The configuration file should be placed in +``/etc/fedora-messaging/robosignatory.toml`` (see the provided +``robosignatory.toml`` file in this repository for an example). +The part specific to RoboSignatory is in the ``[consumer_config]`` section. + +Then the listening service can be activated and started with:: + + # systemctl enable fm-consumer@robosignatory.service + # systemctl start fm-consumer@robosignatory.service + Development ----------- @@ -21,12 +34,8 @@ Run the tests with:: Signing Configuration --------------------- -For an example configuration file, look in config/example-config.py. -This should be placed in /etc/fedmsg.d/robosignatory.py (or some other name in -that directory). - -The only generic part in there is the signing part, for the other options -please check the parts below in this document. +The only generic part in the configuration is the signing part, for the other +options please check the parts below in this document. For signing, the one argument you always provide is "backend". This is the name of a robosignatory.signing.helpers setuptools entry point. @@ -34,25 +43,25 @@ This is the name of a robosignatory.signing.helpers setuptools entry point. Pre-shipped are "echo" and "sigul". The other arguments in this section are passed as keyword arguments to the -helper's __init__ method, so are specific for the module you choose to use. +helper's ``__init__`` method, so are specific for the module you choose to use. -Koji Tag Consumer ------------------ +Koji Tag Signer +--------------- -To enable TagConsumer, set robosignatory.enabled.tagsigner to True. - -Then you will need to add all the koji "instances" your setup should be aware -of under robosignatory.koji_instances. The url is the link to the kojihub main -url of the instance. +You will need to add all the koji "instances" your setup should be aware +of under the ``koji_instances`` section. The url is the link to the kojihub +main url of the instance. Options contains authentication information. There are two authmethods available: -ssl, which takes arguments cert and serverca (both required). -kerberos, which takes arguments principal, keytab and ccache (all optional). + +- ssl, which takes arguments cert and serverca (both required). +- kerberos, which takes arguments principal, keytab and ccache (all optional). In the tags part of the instance configuration is the real configuration for -the TagConsumer. +the TagSigner. + It is a list, with each entry being a dict with the tag that should be watched, a key name (passed to the signing module to indicate which key to use) and keyid (passed to koji to indicate which signatures need to be written out). @@ -75,6 +84,7 @@ This example would watch for any builds tagged into the f26-pending tag. After it sees a build tagged in this tag, it will look up which RPMs need to be signed, and pass their names together with the koji instance name and key name to the signing module. + After the signing module acknowledges that it signed the packages, robosignatory will tell koji to write the signed RPMs out with the keyid. If that is done and the "from" tag is different from the "to" tag, it will @@ -82,11 +92,11 @@ issue a koji moveBuild operation, moving the build from "f26-pending" to "f26". After this, it is done signing the package, and continues to the next step. -Testing Koji Tag Consumer -------------------------- +Testing Koji Tag Signer +----------------------- To test the configuration, you can create the full configuration, and run the -robosignatory-signtagbuild command, providing the name of the koji instance, the +``robosignatory sign-tag`` command, providing the name of the koji instance, the build NVR and the current tag, and whether or not to skip the tag moving. This will follow the exact same procedures as outlined in the previous section, printing a lot of information along the way so you can follow what it's doing diff --git a/fedmsg.d/example-config.py b/fedmsg.d/example-config.py deleted file mode 100644 index 10c540f..0000000 --- a/fedmsg.d/example-config.py +++ /dev/null @@ -1,57 +0,0 @@ -import socket -hostname = socket.gethostname() - -config = { - 'robosignatory.enabled.tagsigner': True, - 'robosignatory.enabled.atomicsigner': True, - 'robosignatory.enabled.coreossigner': True, - - 'robosignatory.signing': { - # This should be the name of an entrypoint plugin that provides - # SigningHelper functionality. - 'backend': 'sigul', - # These are arguments to the __init__ method of that helper. - 'user': 'robosignatory', - 'passphrase_file': 'robosignatory.pass', - 'config_file': '/etc/sigul/client.conf', - }, - - # The keys here need to be the same in the sigul bridge - 'robosignatory.koji_instances': { - 'primary': { - 'url': 'http://koji.fedoraproject.org/kojihub', - 'options': { - # Only ssl and kerberos are supported at the moment - 'authmethod': 'ssl', - 'cert': 'robosignatory.cert', - 'serverca': 'servcer-ca.cert', - }, - 'mbs_user': 'mbs/mbs.example.com', - 'tags': [ - { - 'from': 'rawhide-signcandidate', - 'to': 'rawhide', - 'key': 'fedora26', - 'keyid': 'xxxxxxxx' - }, - { - 'from': 'rawhide-modular-signcandidate', - 'to': 'rawhide-modular', - 'key': 'fedora26', - 'keyid': 'xxxxxxxx', - 'type': 'modular' - }, - ], - }, - }, - - 'robosignatory.ostree_refs': { - 'fedora-atomic/25/x86_64/docker-host': { - 'directory': '/mnt/koji/compose/atomic/25/', - 'key': 'fedora-25' - } - }, - - 'robosignatory.coreos.bucket': 'robosig-dev-fcos-builds', - 'robosignatory.coreos.key': 'coreos', -} diff --git a/fedmsg.d/robosignatory-logging.py b/fedmsg.d/robosignatory-logging.py deleted file mode 100644 index eb0dabd..0000000 --- a/fedmsg.d/robosignatory-logging.py +++ /dev/null @@ -1,16 +0,0 @@ -config = dict( - logging=dict( - loggers=dict( - robosignatory={ - "level": "DEBUG", - "propagate": False, - "handlers": ["console"], - }, - root={ - "level": "INFO", - "propagate": False, - "handlers": ["console"], - }, - ), - ), -) diff --git a/robosignatory.toml b/robosignatory.toml new file mode 100644 index 0000000..e64a6cd --- /dev/null +++ b/robosignatory.toml @@ -0,0 +1,119 @@ +# A sample configuration for fedora-messaging. This file is in the TOML format. +# For complete details on all configuration options, see the documentation. + +amqp_url = "amqp://" + +publish_exchange = "amq.topic" + +callback = "robosignatory.consumer:Consumer" + +# The topic_prefix configuration value will add a prefix to the topics of every sent message. +# This is used for migrating from fedmsg, and should not be used afterwards. +topic_prefix = "org.fedoraproject.dev" + +# Note the double brackets below. +# To add another binding, add another [[bindings]] section. +[[bindings]] +queue = "robosignatory" +exchange = "amq.topic" +routing_keys = [ + "org.fedoraproject.*.pungi.compose.ostree", + "org.fedoraproject.*.coreos.build.request.artifacts-sign", + "org.fedoraproject.*.buildsys.tag", +] + +[tls] +ca_cert = "/etc/pki/tls/certs/ca-bundle.crt" +keyfile = "/my/client/key.pem" +certfile = "/my/client/cert.pem" + +[client_properties] +app = "RoboSignatory" + +[queues.robosignatory] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(name)s %(levelname)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.robosignatory] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "INFO" +handlers = ["console"] + + +# robosignatory consumer configuration +[consumer_config] + + [consumer_config.signing] + # This should be the name of an entrypoint plugin that provides + # SigningHelper functionality. + backend = "sigul" + # These are arguments to the __init__ method of that helper. + user = "robosignatory" + passphrase_file = "robosignatory.pass" + config_file = "/etc/sigul/client.conf" + + [consumer_config.koji_instances] + # The keys here need to be the same in the sigul bridge + [consumer_config.koji_instances.primary] + url = 'http://koji.fedoraproject.org/kojihub' + mbs_user = 'mbs/mbs.example.com' + + [consumer_config.koji_instances.primary.options] + # Only ssl and kerberos are supported at the moment + authmethod = 'ssl' + cert = 'robosignatory.cert' + serverca = 'servcer-ca.cert' + + [[consumer_config.koji_instances.primary.tags]] + from = 'rawhide-signcandidate' + to = 'rawhide' + key = 'fedora26' + keyid = 'xxxxxxxx' + + [[consumer_config.koji_instances.primary.tags]] + from = 'rawhide-modular-signcandidate' + to = 'rawhide-modular' + key = 'fedora26' + keyid = 'xxxxxxxx' + type = 'modular' + + [consumer_config.ostree_refs] + [consumer_config.ostree_refs."fedora-atomic/25/x86_64/docker-host"] + directory = '/mnt/koji/compose/atomic/25/' + key = 'fedora-25' + + [consumer_config.coreos] + bucket = "robosig-dev-fcos-builds" + key = "coreos" + + [consumer_config.coreos.aws] + access_key = "ACCESS_KEY" + access_secret = "SECRET" + region = "us-east-1" diff --git a/robosignatory/atomic.py b/robosignatory/atomic.py new file mode 100644 index 0000000..d9aa368 --- /dev/null +++ b/robosignatory/atomic.py @@ -0,0 +1,47 @@ +from __future__ import unicode_literals, absolute_import + +import robosignatory.utils as utils +import robosignatory.work + +import logging +log = logging.getLogger("robosignatory.atomicconsumer") + + +class AtomicSigner(object): + + def __init__(self, config): + self.config = config + self.signer = utils.get_signing_helper(**self.config["signing"]) + + self.refs = {} + for ref, val in self.config['ostree_refs'].items(): + if 'ref_to' in val: + raise ValueError('ref_to in %s found. This is depricated' % + ref) + self.refs[ref] = val + + log.info('AtomicSigner ready for service') + + def consume(self, msg): + # Example message: + # {u'ref': u'fedora-atomic/25/x86_64/docker-host', + # u'commitid': u'f99114401f....', + # u'arch': u'x86_64', + # u'variant': u'Atomic', + # u'location': u'http://kojipkgs....', + # u'compose_id': u'Fedora-25-20161002.n.0'} + + ref = msg.body['ref'] + commitid = msg.body['commitid'] + + log.info('pungi composed %(ref)s (%(commitid)s, variant %(variant)s, ' + 'arch %(arch)s)' % msg.body) + + if ref not in self.refs: + log.info('Unknown reference %s. Skipping' % ref) + return + + val = self.refs[ref] + + robosignatory.work.process_atomic(self.signer, ref, commitid, + **val) diff --git a/robosignatory/atomicconsumer.py b/robosignatory/atomicconsumer.py deleted file mode 100644 index 8c34e89..0000000 --- a/robosignatory/atomicconsumer.py +++ /dev/null @@ -1,62 +0,0 @@ -import fedmsg.consumers -import robosignatory.utils as utils -import robosignatory.work - -import logging -log = logging.getLogger("robosignatory.atomicconsumer") - - -class AtomicSignerConsumer(fedmsg.consumers.FedmsgConsumer): - config_key = 'robosignatory.enabled.atomicsigner' - - def __init__(self, *args, **kwargs): - super(AtomicSignerConsumer, self).__init__(*args, **kwargs) - - prefix = self.hub.config.get('topic_prefix') - env = self.hub.config.get('environment') - self.topic = [ - '%s.%s.pungi.compose.ostree' % (prefix, env), - ] - - signing_config = self.hub.config['robosignatory.signing'] - self.signer = utils.get_signing_helper(**signing_config) - - self.refs = {} - for ref in self.hub.config['robosignatory.ostree_refs']: - val = self.hub.config['robosignatory.ostree_refs'][ref] - if 'ref_to' in val: - raise ValueError('ref_to in %s found. This is depricated' % - ref) - self.refs[ref] = val - - log.info('AtomicSignerConsumer ready for service') - - def consume(self, msg): - topic = msg['topic'] - if topic not in self.topic: - return - - msg = msg['body']['msg'] - - # Example message: - # {u'ref': u'fedora-atomic/25/x86_64/docker-host', - # u'commitid': u'f99114401f....', - # u'arch': u'x86_64', - # u'variant': u'Atomic', - # u'location': u'http://kojipkgs....', - # u'compose_id': u'Fedora-25-20161002.n.0'} - - ref = msg['ref'] - commitid = msg['commitid'] - - log.info('pungi composed %(ref)s (%(commitid)s, variant %(variant)s, ' - 'arch %(arch)s)' % msg) - - if ref not in self.refs: - log.info('Unknown reference %s. Skipping' % ref) - return - - val = self.refs[ref] - - robosignatory.work.process_atomic(self.signer, ref, commitid, - **val) diff --git a/robosignatory/cli.py b/robosignatory/cli.py index fb10808..53bcc13 100644 --- a/robosignatory/cli.py +++ b/robosignatory/cli.py @@ -1,80 +1,63 @@ -import argparse -import fedmsg.config -import koji -import sys +import os -from robosignatory.tagconsumer import TagSignerConsumer -from robosignatory.coreosconsumer import CoreOSSignerConsumer -import robosignatory.utils as utils -import robosignatory.work - -import logging -logging.basicConfig(level=logging.INFO) - - -def tagsigner(): - if len(sys.argv) not in [4, 5]: - print('Usage: %s [--tag]' % sys.argv[0]) - sys.exit(1) +import click +from fedora_messaging.config import conf - skiptag = True - if len(sys.argv) == 5: - if sys.argv[4] != '--tag': - print('Usage: %s [--tag]' % sys.argv[0]) - sys.exit(1) - skiptag = False - - koji_instance = sys.argv[1] - build_nvr = sys.argv[2] - curtag = sys.argv[3] - - signer = TagSignerConsumer(None) +import robosignatory.work +from robosignatory.tag import TagSigner +from robosignatory.coreos import CoreOSSigner +from robosignatory import utils + + +@click.group() +@click.option( + "-c", "--config", + default="/etc/fedora-messaging/robosignatory.toml", + type=click.Path(exists=True, dir_okay=False), + help="Path to the configuration file" +) +def cli(config): + if not os.path.isfile(config): + raise click.exceptions.BadParameter("{} is not a file".format(config)) + conf.load_config(config_path=config) + conf.setup_logging() + + +@cli.command("sign-tag") +@click.option("--tag", is_flag=True) +@click.argument("koji_instance") +@click.argument("build_nvr") +@click.argument("curtag") +def tag(tag, koji_instance, build_nvr, curtag): + skiptag = not tag + signer = TagSigner(conf["consumer_config"]) signer.dowork(build_nvr, None, curtag, koji_instance, skiptag) -def atomicsigner(): - if len(sys.argv) not in [3, 4]: - print('Usage: %s [--ref-update]' % sys.argv[0]) - sys.exit(1) - - ref = sys.argv[1] - commitid = sys.argv[2] - - doref = False - if len(sys.argv) == 4: - if sys.argv[3] != '--ref-update': - print('Usage: %s [--ref-update]' % sys.argv[0]) - sys.exit(1) - doref = True - - config = fedmsg.config.load_config([], None) - - signing_config = config['robosignatory.signing'] +@cli.command("sign-atomic") +@click.option("--ref-update", is_flag=True) +@click.argument("ref") +@click.argument("commitid") +def atomic(ref_update, ref, commitid): + if ref not in conf["consumer_config"]["ostree_refs"]: + click.fail('Ref %s not found' % ref) + signing_config = conf["consumer_config"]["signing"] signer = utils.get_signing_helper(**signing_config) + val = conf["consumer_config"]["ostree_refs"][ref] + robosignatory.work.process_atomic( + signer, ref, commitid, doref=ref_update, **val) - if ref not in config['robosignatory.ostree_refs']: - print('Ref %s not found' % ref) - sys.exit(1) - - val = config['robosignatory.ostree_refs'][ref] - robosignatory.work.process_atomic(signer, ref, commitid, doref=doref, **val) - - -def coreossigner(): - parser = argparse.ArgumentParser() - parser.add_argument("file") - parser.add_argument("checksum") - args = parser.parse_args() - - config = fedmsg.config.load_config([], None) - signing_config = config['robosignatory.signing'] +@cli.command("sign-coreos") +@click.argument("file_url") +@click.argument("checksum") +def coreos(file_url, checksum): + signing_config = conf["consumer_config"]["signing"] signer = utils.get_signing_helper(**signing_config) - - signer = CoreOSSignerConsumer(None) + signer = CoreOSSigner(conf["consumer_config"]) contents = { "artifacts": [ - {"file": args.file, "checksum": args.checksum} + {"file": file_url, "checksum": checksum} ] } signer.dowork(contents) diff --git a/robosignatory/consumer.py b/robosignatory/consumer.py new file mode 100644 index 0000000..5871448 --- /dev/null +++ b/robosignatory/consumer.py @@ -0,0 +1,54 @@ +from __future__ import unicode_literals, absolute_import + +import logging + +import fedora_messaging + +from .tag import TagSigner +from .atomic import AtomicSigner +from .coreos import CoreOSSigner + + +log = logging.getLogger('robosignatory') + + +class Consumer(object): + """All messages are received by this class's __call__() method.""" + + def __init__(self): + log.info('Initializing Robosignatory consumer') + self.config = fedora_messaging.config.conf["consumer_config"] + self.tag_handler = TagSigner(self.config) + self.atomic_handler = AtomicSigner(self.config) + self.coreos_handler = CoreOSSigner(self.config) + + def __call__(self, msg): + """ + Callback method called by fedora-messaging consume. + + Redirect messages to the correct handler using the + message topic. + + In case of duplicate messages, robosignatory will just try to re-sign + the object, which will have no effect (except wasted cyles but that + should not be a major problem). + + Args: + msg (fedora_messaging.api.Message): The message received from the broker. + """ + log.info('Received message from fedora-messaging with topic: %s', msg.topic) + + try: + if msg.topic.endswith('.buildsys.tag'): + log.debug('Passing message to the Tag handler') + self.tag_handler.consume(msg) + elif msg.topic.endswith('.pungi.compose.ostree'): + log.debug('Passing message to the Atomic handler') + self.atomic_handler.consume(msg) + elif msg.topic.endswith('.coreos.build.request.artifacts-sign'): + log.debug('Passing message to the CoreOS handler') + self.coreos_handler.consume(msg) + except Exception as e: + error_msg = '{e}: Unable to handle message: {msg}'.format(e=e, msg=msg) + log.exception(error_msg) + raise fedora_messaging.exceptions.Nack(error_msg) diff --git a/robosignatory/coreos.py b/robosignatory/coreos.py new file mode 100644 index 0000000..37d8466 --- /dev/null +++ b/robosignatory/coreos.py @@ -0,0 +1,53 @@ +from __future__ import unicode_literals, absolute_import + +import shutil +import tempfile + +import boto3 +import robosignatory.utils as utils +import robosignatory.work + +import logging +log = logging.getLogger("robosignatory.coreosconsumer") + + +class CoreOSSigner(object): + + def __init__(self, config): + self.config = config + + aws_config = self.config['coreos']['aws'] + s3 = boto3.resource( + 's3', + region_name=aws_config['region'], + aws_access_key_id=aws_config['access_key'], + aws_secret_access_key=aws_config['access_secret'], + ) + bucket_name = self.config['coreos']['bucket'] + self.bucket = s3.Bucket(bucket_name) + + signing_config = self.config['signing'] + self.signer = utils.get_signing_helper(**signing_config) + + log.info('CoreOSSigner ready for service') + + def consume(self, msg): + # Message structure: https://github.com/coreos/fedora-coreos-tracker/issues/198#issuecomment-513944390 + log.info( + 'CoreOS wants to sign ' + '%(build_id)s on %(stream)s for %(basearch)s' % msg.body + ) + self.dowork(msg.body) + + def dowork(self, contents): + # This is here and not in __init__ because we may want a stream or + # version-dependant key in the future. + key = self.config["coreos"]["key"] + + tmpdir = tempfile.mkdtemp(prefix="/tmp/robosignatory-") + try: + for artifact in contents["artifacts"]: + robosignatory.work.process_coreos( + self.signer, key, self.bucket, tmpdir, artifact) + finally: + shutil.rmtree(tmpdir) diff --git a/robosignatory/coreosconsumer.py b/robosignatory/coreosconsumer.py deleted file mode 100644 index 4a9b631..0000000 --- a/robosignatory/coreosconsumer.py +++ /dev/null @@ -1,68 +0,0 @@ -import shutil -import tempfile - -import boto3 -import fedmsg.consumers -import robosignatory.utils as utils -import robosignatory.work - -import logging -log = logging.getLogger("robosignatory.coreosconsumer") - - -class CoreOSSignerConsumer(fedmsg.consumers.FedmsgConsumer): - config_key = 'robosignatory.enabled.coreossigner' - - def __init__(self, hub): - if hub: - super(CoreOSSignerConsumer, self).__init__(hub) - self.config = self.hub.config - else: - # No hub, we are in ad-hoc mode - self.config = fedmsg.config.load_config() - - prefix = self.config.get('topic_prefix') - env = self.config.get('environment') - self.topic = [ - "%s.%s.coreos.build.request.artifacts-sign" % (prefix, env) - ] - - aws_config = self.config['robosignatory.coreos.aws'] - s3 = boto3.resource( - 's3', - region_name=aws_config['region'], - aws_access_key_id=aws_config['access_key'], - aws_secret_access_key=aws_config['access_secret'], - ) - bucket_name = self.config['robosignatory.coreos.bucket'] - self.bucket = s3.Bucket(bucket_name) - - signing_config = self.config['robosignatory.signing'] - self.signer = utils.get_signing_helper(**signing_config) - - log.info('CoreOSSignerConsumer ready for service') - - def consume(self, msg): - topic = msg['topic'] - if topic not in self.topic: - return - # Message structure: https://github.com/coreos/fedora-coreos-tracker/issues/198#issuecomment-513944390 - contents = msg['body']['msg'] - log.info( - 'CoreOS wants to sign ' - '%(build_id)s on %(stream)s for %(basearch)s' % contents - ) - self.dowork(contents) - - def dowork(self, contents): - # This is here and not in __init__ because we may want a stream or - # version-dependant key in the future. - key = self.config["robosignatory.coreos.key"] - - tmpdir = tempfile.mkdtemp(prefix="/tmp/robosignatory-") - try: - for artifact in contents["artifacts"]: - robosignatory.work.process_coreos( - self.signer, key, self.bucket, tmpdir, artifact) - finally: - shutil.rmtree(tmpdir) diff --git a/robosignatory/tag.py b/robosignatory/tag.py new file mode 100644 index 0000000..4bd2aa4 --- /dev/null +++ b/robosignatory/tag.py @@ -0,0 +1,176 @@ +from __future__ import unicode_literals, absolute_import + +import koji + +import robosignatory.utils as utils + +import logging +log = logging.getLogger("robosignatory.tagconsumer") + + +KNOWN_TAG_TYPES = ['plain', 'modular'] + + +class TagSigner(object): + + def __init__(self, config): + self.config = config + + signing_config = self.config['signing'] + self.signer = utils.get_signing_helper(**signing_config) + + self.koji_clients = {} + for instance in self.config['koji_instances']: + instance_info = self.config[ + 'koji_instances'][instance] + client = koji.ClientSession(instance_info['url'], + instance_info['options']) + + if instance_info['options']['authmethod'] == 'ssl': + client.ssl_login(instance_info['options']['cert'], + None, + instance_info['options']['serverca']) + elif instance_info['options']['authmethod'] == 'kerberos': + kwargs = {} + for opt in ('principal', 'keytab', 'ccache'): + if opt in instance_info['options']: + kwargs[opt] = instance_info['options'][opt] + client.krb_login(**kwargs) + else: + raise Exception('Only SSL and kerberos authmethods supported') + + instance_obj = {'client': client, + 'tags': {}} + if 'mbs_user' in instance_info: + instance_obj['mbs_user'] = instance_info['mbs_user'] + for tag in instance_info['tags']: + if tag['from'] in instance_obj['tags']: + raise Exception('From detected twice: %s' % tag['from']) + instance_obj['tags'][tag['from']] = {'to': tag['to'], + 'key': tag['key'], + 'keyid': tag['keyid']} + + tag_type = tag.get('type') + if tag_type is None: + tag_type = 'plain' + elif tag_type not in KNOWN_TAG_TYPES: + raise Exception('Invalid tag type detected: %s', tag_type) + instance_obj['tags'][tag['from']]['type'] = tag_type + + self.koji_clients[instance] = instance_obj + + log.info('TagSigner ready for service') + + def consume(self, msg): + # {u'build_id': 799208, + # u'name': u'python-fmn-rules', + # u'tag_id': 374, + # u'instance': u'primary', + # u'tag': u'epel7-infra', + # u'user': u'puiterwijk', + # u'version': u'0.9.1', + # u'owner': u'sayanchowdhury', + # u'release': u'1.el7'}} + + build_nvr = '%(name)s-%(version)s-%(release)s' % msg.body + build_id = msg.body['build_id'] + tag = msg.body['tag'] + koji_instance = msg.body['instance'] + + log.info('Build %s (%s) tagged into %s on %s', + build_nvr, build_id, tag, koji_instance) + + if koji_instance not in self.koji_clients: + log.info('Koji instance not known, skipping') + return + + instance = self.koji_clients[koji_instance] + self.dowork(build_nvr, build_id, tag, koji_instance, + skip_tagging=False) + + def dowork(self, build_nvr, build_id, tag, koji_instance, + skip_tagging=False): + instance = self.koji_clients[koji_instance] + + if not build_id: + build_id = instance['client'].findBuildID(build_nvr) + + if tag not in instance['tags']: + log.info('Tag not autosigned, skipping') + return + tag_info = instance['tags'][tag] + + log.info('Going to sign %s with %s (%s) and move to %s', + build_nvr, tag_info['key'], tag_info['keyid'], + tag_info['to']) + + if tag_info['type'] == 'plain': + self.signwrite_single_build(build_nvr, build_id, tag_info, instance, koji_instance) + elif tag_info['type'] == 'modular': + self.signwrite_module(build_nvr, build_id, tag_info, instance, koji_instance) + else: + raise NotImplementedError('Tag type %s not implemented' % tag_info['type']) + + if skip_tagging: + log.info('Tagging skipped, done') + else: + log.info('Packages correctly signed, moving to %s' % + tag_info['to']) + if tag == tag_info['to']: + log.info('Non-gated, not moving') + else: + instance['client'].tagBuild(tag_info['to'], build_id, False, + tag) + + def signwrite_module(self, build_nvr, build_id, tag_info, instance, koji_instance): + log.info('Signing module build %s', build_nvr) + buildinfo = instance['client'].getBuild(build_id) + if not buildinfo: + raise Exception('No build object found?') + content_koji_tag = buildinfo['extra']['typeinfo']['module']['content_koji_tag'] + log.info('Content tag: %s', content_koji_tag) + log.info('Signing all module content') + for build in instance['client'].listTagged(content_koji_tag): + if build['owner_name'] != instance['mbs_user']: + log.error('Build %(build_id)s has owner %(owner_name)s, which is NOT mbs_user!' % build) + raise Exception('Modular content tag contains invalid owned build') + self.signwrite_single_build(build['nvr'], build['build_id'], tag_info, instance, koji_instance) + + def signwrite_single_build(self, build_nvr, build_id, tag_info, instance, koji_instance): + log.info('Signing and writing build %s', build_nvr) + rpms = utils.get_rpms(instance['client'], + build_nvr=build_nvr, + build_id=build_id, + sigkey=tag_info['keyid']) + log.info('RPMs to sign and move: %s', + ['%s (%s, signed: %s)' % + (key, rpms[key]['id'], rpms[key]['signed']) + for key in rpms.keys()]) + if len(rpms) < 1: + log.info('Build contains no rpms, skipping signing and writing') + + if all([rpms[rpm]['signed'] for rpm in rpms]) or len(rpms) < 1: + log.debug('All RPMs are already signed') + else: + to_sign = [key for key in rpms.keys() if not rpms[key]['signed']] + log.debug('RPMs needing signing: %s' % to_sign) + cmdline = self.signer.build_sign_cmdline(tag_info['key'], + to_sign, + koji_instance) + log.debug('Signing command line: %s' % cmdline) + + ret, stdout, stderr = utils.run_command(cmdline) + if ret != 0: + log.error('Error signing! Signing output: %s, stdout: ' + '%s, stderr: %s', ret, stdout, stderr) + raise Exception('Signing failed') + + if len(rpms) > 1: + log.info('Build was succesfully signed, telling koji to write with key' + ' %s', tag_info['keyid']) + + for rpm in rpms: + instance['client'].writeSignedRPM(rpms[rpm]['id'], + tag_info['keyid']) + + log.info('Signed RPMs written out') diff --git a/robosignatory/tagconsumer.py b/robosignatory/tagconsumer.py deleted file mode 100644 index 361b73e..0000000 --- a/robosignatory/tagconsumer.py +++ /dev/null @@ -1,195 +0,0 @@ -import koji - -import fedmsg -import fedmsg.consumers -import robosignatory.utils as utils - -import logging -log = logging.getLogger("robosignatory.tagconsumer") - - -KNOWN_TAG_TYPES = ['plain', 'modular'] - - -class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): - config_key = 'robosignatory.enabled.tagsigner' - - def __init__(self, hub): - if hub: - super(TagSignerConsumer, self).__init__(hub) - self.config = self.hub.config - else: - # No hub, we are in ad-hoc mode - self.config = fedmsg.config.load_config() - logging.basicConfig(level=logging.DEBUG) - - prefix = self.config.get('topic_prefix') - env = self.config.get('environment') - self.topic = [ - '%s.%s.buildsys.tag' % (prefix, env) - ] - - signing_config = self.config['robosignatory.signing'] - self.signer = utils.get_signing_helper(**signing_config) - - self.koji_clients = {} - for instance in self.config['robosignatory.koji_instances']: - instance_info = self.config[ - 'robosignatory.koji_instances'][instance] - client = koji.ClientSession(instance_info['url'], - instance_info['options']) - - if instance_info['options']['authmethod'] == 'ssl': - client.ssl_login(instance_info['options']['cert'], - None, - instance_info['options']['serverca']) - elif instance_info['options']['authmethod'] == 'kerberos': - kwargs = {} - for opt in ('principal', 'keytab', 'ccache'): - if opt in instance_info['options']: - kwargs[opt] = instance_info['options'][opt] - client.krb_login(**kwargs) - else: - raise Exception('Only SSL and kerberos authmethods supported') - - instance_obj = {'client': client, - 'tags': {}} - if 'mbs_user' in instance_info: - instance_obj['mbs_user'] = instance_info['mbs_user'] - for tag in instance_info['tags']: - if tag['from'] in instance_obj['tags']: - raise Exception('From detected twice: %s' % tag['from']) - instance_obj['tags'][tag['from']] = {'to': tag['to'], - 'key': tag['key'], - 'keyid': tag['keyid']} - - tag_type = tag.get('type') - if tag_type is None: - tag_type = 'plain' - elif tag_type not in KNOWN_TAG_TYPES: - raise Exception('Invalid tag type detected: %s', tag_type) - instance_obj['tags'][tag['from']]['type'] = tag_type - - self.koji_clients[instance] = instance_obj - - log.info('TagSignerConsumer ready for service') - - def consume(self, msg): - topic = msg['topic'] - if topic not in self.topic: - return - - msg = msg['body']['msg'] - - # {u'build_id': 799208, - # u'name': u'python-fmn-rules', - # u'tag_id': 374, - # u'instance': u'primary', - # u'tag': u'epel7-infra', - # u'user': u'puiterwijk', - # u'version': u'0.9.1', - # u'owner': u'sayanchowdhury', - # u'release': u'1.el7'}} - - build_nvr = '%(name)s-%(version)s-%(release)s' % msg - build_id = msg['build_id'] - tag = msg['tag'] - koji_instance = msg['instance'] - - log.info('Build %s (%s) tagged into %s on %s', - build_nvr, build_id, tag, koji_instance) - - if koji_instance not in self.koji_clients: - log.info('Koji instance not known, skipping') - return - - instance = self.koji_clients[koji_instance] - self.dowork(build_nvr, build_id, tag, koji_instance, - skip_tagging=False) - - def dowork(self, build_nvr, build_id, tag, koji_instance, - skip_tagging=False): - instance = self.koji_clients[koji_instance] - - if not build_id: - build_id = instance['client'].findBuildID(build_nvr) - - if tag not in instance['tags']: - log.info('Tag not autosigned, skipping') - return - tag_info = instance['tags'][tag] - - log.info('Going to sign %s with %s (%s) and move to %s', - build_nvr, tag_info['key'], tag_info['keyid'], - tag_info['to']) - - if tag_info['type'] == 'plain': - self.signwrite_single_build(build_nvr, build_id, tag_info, instance, koji_instance) - elif tag_info['type'] == 'modular': - self.signwrite_module(build_nvr, build_id, tag_info, instance, koji_instance) - else: - raise NotImplementedError('Tag type %s not implemented' % tag_info['type']) - - if skip_tagging: - log.info('Tagging skipped, done') - else: - log.info('Packages correctly signed, moving to %s' % - tag_info['to']) - if tag == tag_info['to']: - log.info('Non-gated, not moving') - else: - instance['client'].tagBuild(tag_info['to'], build_id, False, - tag) - - def signwrite_module(self, build_nvr, build_id, tag_info, instance, koji_instance): - log.info('Signing module build %s', build_nvr) - buildinfo = instance['client'].getBuild(build_id) - if not buildinfo: - raise Exception('No build object found?') - content_koji_tag = buildinfo['extra']['typeinfo']['module']['content_koji_tag'] - log.info('Content tag: %s', content_koji_tag) - log.info('Signing all module content') - for build in instance['client'].listTagged(content_koji_tag): - if build['owner_name'] != instance['mbs_user']: - log.error('Build %(build_id)s has owner %(owner_name)s, which is NOT mbs_user!' % build) - raise Exception('Modular content tag contains invalid owned build') - self.signwrite_single_build(build['nvr'], build['build_id'], tag_info, instance, koji_instance) - - def signwrite_single_build(self, build_nvr, build_id, tag_info, instance, koji_instance): - log.info('Signing and writing build %s', build_nvr) - rpms = utils.get_rpms(instance['client'], - build_nvr=build_nvr, - build_id=build_id, - sigkey=tag_info['keyid']) - log.info('RPMs to sign and move: %s', - ['%s (%s, signed: %s)' % - (key, rpms[key]['id'], rpms[key]['signed']) - for key in rpms.keys()]) - if len(rpms) < 1: - log.info('Build contains no rpms, skipping signing and writing') - - if all([rpms[rpm]['signed'] for rpm in rpms]) or len(rpms) < 1: - log.debug('All RPMs are already signed') - else: - to_sign = [key for key in rpms.keys() if not rpms[key]['signed']] - log.debug('RPMs needing signing: %s' % to_sign) - cmdline = self.signer.build_sign_cmdline(tag_info['key'], - to_sign, - koji_instance) - log.debug('Signing command line: %s' % cmdline) - - ret, stdout, stderr = utils.run_command(cmdline) - if ret != 0: - log.error('Error signing! Signing output: %s, stdout: ' - '%s, stderr: %s', ret, stdout, stderr) - raise Exception('Signing failed') - - if len(rpms) > 1: - log.info('Build was succesfully signed, telling koji to write with key' - ' %s', tag_info['keyid']) - - for rpm in rpms: - instance['client'].writeSignedRPM(rpms[rpm]['id'], - tag_info['keyid']) - - log.info('Signed RPMs written out') diff --git a/setup.py b/setup.py index ef97579..5ced7b1 100644 --- a/setup.py +++ b/setup.py @@ -2,36 +2,30 @@ from setuptools import setup setup( name='robosignatory', - version='0.5.0', - description='fedmsg consumer that automatically signs artifacts', + version='0.6.0', + description='fedora-messaging consumer that automatically signs artifacts', author='Patrick Uiterwijk', author_email='puiterwijk@redhat.com', url='https://pagure.io/robosignatory/', license='gplv2+', install_requires=[ - "fedmsg", + "fedora_messaging", "koji", "psutil", - "moksha.hub", "boto3", "six", + "click", ], tests_require=[ "nose", + "mock", ], packages=[ 'robosignatory', ], entry_points=""" - [moksha.consumer] - tagsignerconsumer = robosignatory.tagconsumer:TagSignerConsumer - atomicsignerconsumer = robosignatory.atomicconsumer:AtomicSignerConsumer - coreossignerconsumer = robosignatory.coreosconsumer:CoreOSSignerConsumer - [console_scripts] - robosignatory-signtagbuild = robosignatory.cli:tagsigner - robosignatory-signatomic = robosignatory.cli:atomicsigner - robosignatory-signcoreos = robosignatory.cli:coreossigner + robosignatory = robosignatory.cli:cli [robosignatory.signing.helpers] echo = robosignatory.utils:EchoHelper diff --git a/tests/test_consumers.py b/tests/test_consumers.py new file mode 100644 index 0000000..1ac12f5 --- /dev/null +++ b/tests/test_consumers.py @@ -0,0 +1,81 @@ +import unittest + +from fedora_messaging.api import Message +from fedora_messaging.exceptions import Nack +import mock + +from robosignatory.consumer import Consumer + + +TEST_CONFIG = { + "signing": { + "backend": "echo", + }, + "koji_instances": {}, + "ostree_refs": {}, + "coreos": { + "bucket": "testing", + "key": "testing", + "aws": { + "access_key": "testing", + "access_secret": "testing", + "region": "us-east-1", + } + }, +} + + +@mock.patch( + "robosignatory.consumer.fedora_messaging.config.conf", + {"consumer_config": TEST_CONFIG} +) +class TestConsumers(unittest.TestCase): + + @mock.patch('robosignatory.consumer.TagSigner') + def test_message_tag(self, Handler): + msg = Message( + topic="org.fedoraproject.prod.buildsys.tag", + body={} + ) + consumer = Consumer() + consumer.tag_handler = mock.Mock() + consumer(msg) + consumer.tag_handler.consume.assert_called_once_with(msg) + + @mock.patch('robosignatory.consumer.AtomicSigner') + def test_message_atomic(self, Handler): + msg = Message( + topic="org.fedoraproject.prod.pungi.compose.ostree", + body={} + ) + consumer = Consumer() + consumer.atomic_handler = mock.Mock() + consumer(msg) + consumer.atomic_handler.consume.assert_called_once_with(msg) + + @mock.patch('robosignatory.consumer.CoreOSSigner') + def test_message_coreos(self, Handler): + msg = Message( + topic="org.fedoraproject.prod.coreos.build.request.artifacts-sign", + body={} + ) + consumer = Consumer() + consumer.coreos_handler = mock.Mock() + consumer(msg) + consumer.coreos_handler.consume.assert_called_once_with(msg) + + @mock.patch('robosignatory.consumer.log.exception') + def test_message_exception(self, error): + """Test catching an exception when processing messages.""" + msg = mock.Mock() + # Ensure msg.topic.endswith() throws an exception right away, this way + # we can forgo mocking out handlers or similar complications. + msg.topic = None + + with self.assertRaises(Nack) as exc: + Consumer()(msg) + + msg = ("'NoneType' object has no attribute 'endswith': " + "Unable to handle message: {}".format(msg)) + error.assert_called_once_with(msg) + self.assertEqual(str(exc.exception), msg)