From e80bd4bcfb949fcabd5ae11ea02c71f20ee305b7 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:47 +0000 Subject: [PATCH 1/15] Enqueue a package published by anitya. --- diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py new file mode 100644 index 0000000..45b5c58 --- /dev/null +++ b/kiskadee/plugins/anitya.py @@ -0,0 +1,45 @@ +import kiskadee.plugins +import kiskadee.queue +import requests +import fedmsg +import fedmsg.config +from time import sleep + + +class Plugin(kiskadee.plugins.Plugin): + def __init__(self): + """Loads from /etc/fedmsg.d/endpoints.py the anitya configuration""" + super().__init__() + fedmsg.config.load_config() + + + def watch(self): + """Start the monitoring process for Anitya reports. + + Each package monitored by the plugin will be + queued using the package_enqueuer decorator. + """ + while True: + self._get_messages() + + def get_sources(self, source_data): + return {} + + def _get_messages(self): + """Get messages published on fedmsg""" + + for name, endpoint, topic, msg in fedmsg.tail_messages(): + print(msg.get('msg').get('project')) + self._create_package_dict(msg) + + @kiskadee.queue.package_enqueuer + def _create_package_dict(self, message): + return { + 'name': message.get('msg').get('project').get('name'), + 'version': message.get('msg').get('project').get('version'), + 'plugin': kiskadee.plugins.anitya, + 'meta': { + 'backend': message.get('msg').get('project').get('backend'), + 'homepage': message.get('msg').get('project').get('homepage') + } + } diff --git a/requirements.txt b/requirements.txt index a6b52b6..19cc2e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ sqlalchemy python-debian chardet Sphinx +fedmsg +fedmsg[consumers] diff --git a/util/kiskadee.conf b/util/kiskadee.conf index 83ea34b..1df6b14 100644 --- a/util/kiskadee.conf +++ b/util/kiskadee.conf @@ -24,6 +24,12 @@ description = SAMATE Juliet test suite analyzers = cppcheck active = no +[anitya_plugin] +target = release-monitoring.org +description = A plugin to monitor Anitya packages +analyzers = cppcheck +active = yes + [example_plugin] target = example description = SAMATE Juliet test suite From e7cdb161e90e258218c2f3be1061654fdac81f9b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:47 +0000 Subject: [PATCH 2/15] Initial implementation of a anitya backend. - This commit adds a first attempt to download some github source, published by anitya on fedmsg. We will have to think in a good package architecture to manage the several upstrems that anitya suports. For now we are only dealing with github. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index ccd9729..5cd143a 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -53,9 +53,11 @@ def load_plugins(): """ plugins = [] for plugin in kiskadee_plugins_list: - if config[plugin + '_plugin'].getboolean('active'): - plugins.append(importlib.import_module( - 'kiskadee.plugins.' + plugin)) + #TODO: I will think in a good architecture to anitya backends. + if plugin.find('anitya_') == -1: + if config[plugin + '_plugin'].getboolean('active'): + plugins.append(importlib.import_module('kiskadee.plugins.' + + plugin)) return plugins diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 45b5c58..0d5a899 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -1,9 +1,12 @@ +import tempfile +import importlib +from time import sleep + import kiskadee.plugins import kiskadee.queue import requests import fedmsg import fedmsg.config -from time import sleep class Plugin(kiskadee.plugins.Plugin): @@ -23,7 +26,10 @@ class Plugin(kiskadee.plugins.Plugin): self._get_messages() def get_sources(self, source_data): - return {} + path = tempfile.mkdtemp() + backend_name = source_data.get('meta').get('backend').lower() + backend = self._load_backend(backend_name) + return backend.download_source(source_data, path) def _get_messages(self): """Get messages published on fedmsg""" @@ -43,3 +49,7 @@ class Plugin(kiskadee.plugins.Plugin): 'homepage': message.get('msg').get('project').get('homepage') } } + + def _load_backend(self, backend_name): + return importlib.import_module(''.join(['kiskadee.plugins.anitya_', + backend_name])) diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py new file mode 100644 index 0000000..722c7eb --- /dev/null +++ b/kiskadee/plugins/anitya_github.py @@ -0,0 +1,17 @@ +import urllib.request +import kiskadee.helpers + +def download_source(source_data, path): + pkg_version = ''.join([source_data.get('version'), '.tar.gz']) + with kiskadee.helpers.chdir(path): + url = source_url(pkg_version) + in_file = urllib.request.urlopen(url) + data = in_file.read() + with open(pkg_version, 'wb') as info: + info.write(data) + return ''.join([path, '/', pkg_version]) + +def source_url(version): + github_url = 'https://github.com/firecat53/urlscan/archive/' + source_url = ''.join([github_url, version]) + return source_url From daf4021841eb317a9652e0ed3bcba50987cc6ae9 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:47 +0000 Subject: [PATCH 3/15] Implements compare_versions in anitya. - This commit add packaging.version tool, that we are using to compare versions. It seems better than semver. Anitya monitors different targets, so we need a generic tool to compare project versions. - Cppcheck trigger a error, if the source code is not c/c++. We still cannot predict what the language of the source code that we are analysing, so this was a bug that could make kiskadee breaks a lot. Now we catch the exeption and write a error in the log. --- diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 0d5a899..6de8565 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -1,20 +1,19 @@ import tempfile import importlib from time import sleep - -import kiskadee.plugins -import kiskadee.queue +from packaging import version import requests import fedmsg import fedmsg.config +import kiskadee.plugins +import kiskadee.queue + class Plugin(kiskadee.plugins.Plugin): + def __init__(self): - """Loads from /etc/fedmsg.d/endpoints.py the anitya configuration""" super().__init__() - fedmsg.config.load_config() - def watch(self): """Start the monitoring process for Anitya reports. @@ -22,8 +21,17 @@ class Plugin(kiskadee.plugins.Plugin): Each package monitored by the plugin will be queued using the package_enqueuer decorator. """ - while True: - self._get_messages() + + config = fedmsg.config.load_config([], None) + config["endpoints"] = { "anitya-public-relay": [ + "tcp://release-monitoring.org:9940", + ], + } + config['mute'] = True + config['timeout'] = 0 + + for name, endpoint, topic, msg in fedmsg.tail_messages(**config): + self._create_package_dict(msg) def get_sources(self, source_data): path = tempfile.mkdtemp() @@ -31,12 +39,8 @@ class Plugin(kiskadee.plugins.Plugin): backend = self._load_backend(backend_name) return backend.download_source(source_data, path) - def _get_messages(self): - """Get messages published on fedmsg""" - - for name, endpoint, topic, msg in fedmsg.tail_messages(): - print(msg.get('msg').get('project')) - self._create_package_dict(msg) + def compare_versions(self, new, old): + return version.parse(new) > version.parse(old) @kiskadee.queue.package_enqueuer def _create_package_dict(self, message): diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py index 722c7eb..ab60c6e 100644 --- a/kiskadee/plugins/anitya_github.py +++ b/kiskadee/plugins/anitya_github.py @@ -15,3 +15,4 @@ def source_url(version): github_url = 'https://github.com/firecat53/urlscan/archive/' source_url = ''.join([github_url, version]) return source_url + diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 977b8ca..382ed95 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -75,11 +75,15 @@ def analyze(package): analyzers = plugin.analyzers() for analyzer in analyzers: kiskadee.logger.debug('ANALYSIS: running %s ...' % analyzer) - analysis = kiskadee.analyzers.run(analyzer, path) - firehose_report = kiskadee.converter.to_firehose(analysis, + try: + analysis = kiskadee.analyzers.run(analyzer, path) + firehose_report = kiskadee.converter.to_firehose(analysis, analyzer) - reports.append(str(firehose_report)) - kiskadee.logger.debug('ANALYSIS: DONE running %s' % analyzer) + reports.append(str(firehose_report)) + kiskadee.logger.debug('ANALYSIS: DONE running %s' % analyzer) + except: + kiskadee.logger.debug('ERROR: Could not run + analysis inside container') # TODO: remove compressed/uncompressed files after the analysis return reports else: diff --git a/requirements.txt b/requirements.txt index 19cc2e4..e132351 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ chardet Sphinx fedmsg fedmsg[consumers] +packaging From 665144ecd8c46430c3abbd44d381f0acc5cf3605 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 4/15] Pass project homepage as a parameter to source_url --- diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py index ab60c6e..77e9af3 100644 --- a/kiskadee/plugins/anitya_github.py +++ b/kiskadee/plugins/anitya_github.py @@ -4,15 +4,12 @@ import kiskadee.helpers def download_source(source_data, path): pkg_version = ''.join([source_data.get('version'), '.tar.gz']) with kiskadee.helpers.chdir(path): - url = source_url(pkg_version) + url = source_url(pkg_version, source_data.get('meta').get('homepage')) in_file = urllib.request.urlopen(url) data = in_file.read() with open(pkg_version, 'wb') as info: info.write(data) return ''.join([path, '/', pkg_version]) -def source_url(version): - github_url = 'https://github.com/firecat53/urlscan/archive/' - source_url = ''.join([github_url, version]) - return source_url - +def source_url(version, homepage): + return ''.join([homepage, '/archive/', version]) diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 382ed95..8efad92 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -82,8 +82,8 @@ def analyze(package): reports.append(str(firehose_report)) kiskadee.logger.debug('ANALYSIS: DONE running %s' % analyzer) except: - kiskadee.logger.debug('ERROR: Could not run - analysis inside container') + kiskadee.logger.debug('ERROR: Could not run'\ + 'analysis inside container') # TODO: remove compressed/uncompressed files after the analysis return reports else: From 775b3f892fdc248f5047ce438bb5f0e26ef3622d Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 5/15] Check if a backend exists before load it. --- diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 6de8565..8d680c9 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -9,12 +9,8 @@ import fedmsg.config import kiskadee.plugins import kiskadee.queue - class Plugin(kiskadee.plugins.Plugin): - def __init__(self): - super().__init__() - def watch(self): """Start the monitoring process for Anitya reports. @@ -37,7 +33,10 @@ class Plugin(kiskadee.plugins.Plugin): path = tempfile.mkdtemp() backend_name = source_data.get('meta').get('backend').lower() backend = self._load_backend(backend_name) - return backend.download_source(source_data, path) + if backend: + return backend.download_source(source_data, path) + else: + return {} def compare_versions(self, new, old): return version.parse(new) > version.parse(old) @@ -55,5 +54,10 @@ class Plugin(kiskadee.plugins.Plugin): } def _load_backend(self, backend_name): - return importlib.import_module(''.join(['kiskadee.plugins.anitya_', + try: + return importlib.import_module(''.join(['kiskadee.plugins.anitya_', backend_name])) + except: + kiskadee.logger.info("Backend not "\ + "suported: {}".format(str(backend_name))) + return {} diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py index 77e9af3..e23e6e5 100644 --- a/kiskadee/plugins/anitya_github.py +++ b/kiskadee/plugins/anitya_github.py @@ -2,14 +2,19 @@ import urllib.request import kiskadee.helpers def download_source(source_data, path): - pkg_version = ''.join([source_data.get('version'), '.tar.gz']) - with kiskadee.helpers.chdir(path): - url = source_url(pkg_version, source_data.get('meta').get('homepage')) - in_file = urllib.request.urlopen(url) - data = in_file.read() - with open(pkg_version, 'wb') as info: - info.write(data) - return ''.join([path, '/', pkg_version]) + source_version = ''.join([source_data.get('version'), '.tar.gz']) + homepage = source_data.get('meta').get('homepage') + + if homepage.find("github") != -1: + with kiskadee.helpers.chdir(path): + url = source_url(pkg_version, homepage) + in_file = urllib.request.urlopen(url) + data = in_file.read() + with open(pkg_version, 'wb') as info: + info.write(data) + return ''.join([path, '/', pkg_version]) + else: + return {} def source_url(version, homepage): return ''.join([homepage, '/archive/', version]) From 0252152a2e013ea5111201bdb95fcbf8eb67a884 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 6/15] Fix logs. - Now we use only debug level for logs. If in the future we really need other log level, we must create a new logger in kiskadee/__init__.py. - Closes #10 --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 5cd143a..b32f1ee 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -92,35 +92,16 @@ else: # log _read[0] loaded pass - log_file = config['DEFAULT']['log_file'] logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - ' + - '%(levelname)s - %(message)s') -if log_file != 'stdout': - _warning = logging.FileHandler(log_file, mode='w+') - _info = logging.FileHandler(log_file, mode='w+') - _debug = logging.FileHandler(log_file, mode='w+') - _debug.setFormatter(formatter) - _warning.setFormatter(formatter) - _info.setFormatter(formatter) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +if log_file != 'stdout': + _debug = logging.FileHandler(log_file, mode='w+') else: _debug = logging.StreamHandler(sys.stdout) - _warning = logging.StreamHandler(sys.stdout) - _info = logging.StreamHandler(sys.stdout) - - _debug.setFormatter(formatter) - _warning.setFormatter(formatter) - _info.setFormatter(formatter) - - -_debug.setLevel(logging.DEBUG) -_warning.setLevel(logging.WARNING) -_info.setLevel(logging.INFO) +_debug.setFormatter(formatter) +logger.setLevel(logging.DEBUG) logger.addHandler(_debug) -logger.addHandler(_warning) -logger.addHandler(_info) diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index cde8272..b867e33 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -3,10 +3,10 @@ kiskadee monitors repositories checking for new package versions to be analyzed. This module provides such capabilities. """ - import threading from multiprocessing import Process import time + import kiskadee.database import kiskadee.runner import kiskadee.queue @@ -14,7 +14,6 @@ from kiskadee.model import Package, Plugin, Version, Base RUNNING = True - class Monitor: """Provide kiskadee monitoring objects.""" @@ -22,7 +21,6 @@ class Monitor: """Return a non initialized Monitor.""" self.engine = None self.session = None - self.logger = kiskadee.logger def initialize(self): """Start all threads related to the monitoring process. @@ -68,7 +66,7 @@ class Monitor: if not kiskadee.queue.packages_queue.empty(): pkg = kiskadee.queue.dequeue_package() kiskadee.queue.package_done() - self.logger.info("Dequed Package: {}".format(str(pkg))) + kiskadee.logger.debug("Dequed Package: {}".format(str(pkg))) time.sleep(1) return pkg return {} @@ -91,9 +89,9 @@ class Monitor: has_analysis=False) _package.versions.append(_version) self.session.add(_package) - self.logger.info("Saving package in db: {}".format(str(pkg))) + kiskadee.logger.debug("Saving package in db: {}".format(str(pkg))) self.session.commit() - self.logger.info("Enqueue package {}_{} " + kiskadee.logger.debug("Enqueue package {}_{} " " for analysis".format(pkg['name'], pkg['version'])) kiskadee.queue.enqueue_analysis(pkg) @@ -110,14 +108,14 @@ class Monitor: _pkg.versions.append(_new_version) self.session.add(_pkg) self.session.commit() - self.logger.info("Enqueue package {}_{}" + kiskadee.logger.debug("Enqueue package {}_{}" "for analysis".format(pkg['name'], pkg['version'])) kiskadee.queue.enqueue_analysis(pkg) else: return {} except ValueError: - self.logger.info("Could not compare versions") + kiskadee.logger.debug("Could not compare versions") def _plugin_name(self, plugin): return plugin.__name__.split('.')[len(plugin.__name__.split('.')) - 1] @@ -125,7 +123,7 @@ class Monitor: def _save_plugin(self, plugin): name = self._plugin_name(plugin) plugin = plugin.Plugin() - self.logger.info("Saving {} plugin in database".format(name)) + kiskadee.logger.debug("Saving {} plugin in database".format(name)) if not self.session.query(Plugin).filter(Plugin.name == name).first(): _plugin = Plugin(name=name, target=plugin.config['target'], diff --git a/kiskadee/plugins/__init__.py b/kiskadee/plugins/__init__.py index 06da9fa..b9e9100 100644 --- a/kiskadee/plugins/__init__.py +++ b/kiskadee/plugins/__init__.py @@ -7,6 +7,7 @@ Plugin class here defined. import abc import kiskadee import inspect +import logging class Plugin(): @@ -26,7 +27,6 @@ class Plugin(): self.name = full_name.split('.')[-1] config_section = self.name + '_plugin' self.config = kiskadee.config[config_section] - self.logger = kiskadee.logger @abc.abstractmethod def get_sources(package): diff --git a/kiskadee/plugins/debian.py b/kiskadee/plugins/debian.py index aa6f564..dfd99bd 100644 --- a/kiskadee/plugins/debian.py +++ b/kiskadee/plugins/debian.py @@ -32,7 +32,7 @@ class Plugin(kiskadee.plugins.Plugin): Each package monitored by the plugin will be queued using the package_enqueuer decorator. """ - self.logger.info("Starting Debian plugin") + kiskadee.logger.debug("Starting Debian plugin") while RUNNING: url = self._sources_gz_url() try: @@ -42,7 +42,7 @@ class Plugin(kiskadee.plugins.Plugin): sleep(float(self.config['schedule']) * 60) shutil.rmtree(sources_gz_dir) except urllib.error.URLError: - self.logger.debug("Cannot reach debian mirror") + kiskadee.logger.debug("Cannot reach debian mirror") def get_sources(self, source_data): """Download packages from some debian mirror.""" @@ -53,7 +53,7 @@ class Plugin(kiskadee.plugins.Plugin): subprocess.check_output(['dget', url]) return ''.join([path, '/', self._source_path(path)]) except: - self.logger.debug('Cannot download {} source'. + kiskadee.logger.debug('Cannot download {} source'. format(source_data['name'])) return None diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 8efad92..28a8096 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -61,7 +61,7 @@ def analyze(package): return: list with firehose reports """ plugin = package['plugin'].Plugin() - kiskadee.logger.info('ANALYSIS: Downloading {} ' + kiskadee.logger.debug('ANALYSIS: Downloading {} ' 'source...'.format(package['name'])) compressed_source = plugin.get_sources(package) if compressed_source: From 964e98f724c9fdf35fb583e630e6dc04975490aa Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 7/15] Integrates anitya plugin with fedmsg-hub events. - This commit is the core of anitya plugin. With fedmg-hub we are able to monitors fedmsg bus events, and be notified when a new package was updated. As fedmsg-hub and kiskadee runs in different processes, we are using 0mq to integrates then. - Minor fixes in anitya plugin. --- diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 8d680c9..cf669dd 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -1,10 +1,12 @@ import tempfile -import importlib +import zmq +import yaml from time import sleep from packaging import version -import requests -import fedmsg -import fedmsg.config +import fedmsg.consumers +import time +import urllib.request +import kiskadee.helpers import kiskadee.plugins import kiskadee.queue @@ -16,48 +18,94 @@ class Plugin(kiskadee.plugins.Plugin): Each package monitored by the plugin will be queued using the package_enqueuer decorator. + + The plugin will use zmq as messaging protocol to receive + the fedmsg-hub events. Kiskadee and fedmsg-hub runs in different + processes, so we need something to enable the comunication between then. + When a message come to fedmsg-hub, the AnityaConsumer instance, will + publish this event to zmq server, and kiskadee will consume this message. """ - config = fedmsg.config.load_config([], None) - config["endpoints"] = { "anitya-public-relay": [ - "tcp://release-monitoring.org:9940", - ], - } - config['mute'] = True - config['timeout'] = 0 + context = zmq.Context() + socket = context.socket(zmq.SUB) + kiskadee.logger.debug("Starting anitya plugin") + socket.connect("tcp://localhost:5556") + socket.setsockopt(zmq.SUBSCRIBE, b"anitya") + kiskadee.logger.debug("Connecting to 0mq server") + + while True: + msg = socket.recv_string() + msg = msg[msg.find(" ")+1::] + msg_dict = yaml.load(msg) + self._create_package_dict(msg_dict) - for name, endpoint, topic, msg in fedmsg.tail_messages(**config): - self._create_package_dict(msg) + return {} def get_sources(self, source_data): path = tempfile.mkdtemp() backend_name = source_data.get('meta').get('backend').lower() - backend = self._load_backend(backend_name) - if backend: - return backend.download_source(source_data, path) + run_backend = self._load_backend(backend_name) + if run_backend: + return run_backend(source_data, path) else: return {} def compare_versions(self, new, old): return version.parse(new) > version.parse(old) + def _load_backend(self, backend_name): + try: + backend = Backends() + return getattr(backend, backend_name) + except: + kiskadee.logger.debug("Backend not "\ + "suported: {}".format(str(backend_name))) + return {} + @kiskadee.queue.package_enqueuer - def _create_package_dict(self, message): + def _create_package_dict(self, msg): return { - 'name': message.get('msg').get('project').get('name'), - 'version': message.get('msg').get('project').get('version'), + 'name': msg.get('body').get('msg').get('project').get('name'), + 'version': msg.get('body').get('msg').get('project').get('version'), 'plugin': kiskadee.plugins.anitya, 'meta': { - 'backend': message.get('msg').get('project').get('backend'), - 'homepage': message.get('msg').get('project').get('homepage') + 'backend': msg.get('body').get('msg').get('project').get('backend'), + 'homepage': msg.get('body').get('msg').get('project').get('homepage') } } - def _load_backend(self, backend_name): - try: - return importlib.import_module(''.join(['kiskadee.plugins.anitya_', - backend_name])) - except: - kiskadee.logger.info("Backend not "\ - "suported: {}".format(str(backend_name))) + +class Backends(): + + @staticmethod + def github(source_data, path): + source_version = ''.join([source_data.get('version'), '.tar.gz']) + homepage = source_data.get('meta').get('homepage') + + if homepage.find("github") != -1: + with kiskadee.helpers.chdir(path): + url = ''.join([homepage, '/archive/', source_version]) + in_file = urllib.request.urlopen(url) + data = in_file.read() + with open(source_version, 'wb') as info: + info.write(data) + return ''.join([path, '/', source_version]) + else: return {} + + +class AnityaConsumer(fedmsg.consumers.FedmsgConsumer): + + topic = 'org.release-monitoring.prod.anitya.project.version.update' + config_key = 'anityaconsumer' + validate_signatures = False + + def __init__(self, *args, **kw): + super().__init__(*args, *kw) + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + self.socket.bind("tcp://*:5556") + + def consume(self, msg): + self.socket.send_string("%s %s" % ("anitya", str(msg))) + time.sleep(1) diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py index e23e6e5..e69de29 100644 --- a/kiskadee/plugins/anitya_github.py +++ b/kiskadee/plugins/anitya_github.py @@ -1,20 +0,0 @@ -import urllib.request -import kiskadee.helpers - -def download_source(source_data, path): - source_version = ''.join([source_data.get('version'), '.tar.gz']) - homepage = source_data.get('meta').get('homepage') - - if homepage.find("github") != -1: - with kiskadee.helpers.chdir(path): - url = source_url(pkg_version, homepage) - in_file = urllib.request.urlopen(url) - data = in_file.read() - with open(pkg_version, 'wb') as info: - info.write(data) - return ''.join([path, '/', pkg_version]) - else: - return {} - -def source_url(version, homepage): - return ''.join([homepage, '/archive/', version]) diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 28a8096..76e1905 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -19,6 +19,7 @@ def runner(): :func:`analyze` method, passing the dequeued package. After the analysis, updates the status of this package on the database. """ + kiskadee.logger.debug("Starting runner component") database = kiskadee.database.Database() engine = database.engine session = database.session diff --git a/requirements.txt b/requirements.txt index e132351..cdb08e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ python-debian chardet Sphinx fedmsg -fedmsg[consumers] packaging +pyyaml diff --git a/setup.py b/setup.py index d7c8d3f..eb0963b 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,9 @@ setup( packages=find_packages(), include_package_data=False, entry_points={'console_scripts': [ - 'kiskadee = kiskadee.monitor:daemon']}, + 'kiskadee = kiskadee.monitor:daemon',], + 'moksha.consumer': ( + 'anityaconsumer = kiskadee.plugins.anitya:AnityaConsumer')}, install_requires=get_requirements(), test_suite='nose.collector', tests_require=[ From 25a59c26e1c55c09efcaad4023642bf2a3bdeab9 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 8/15] Anitya tests. - Closes #11 - Fix flake8 and pydocstyle warnings. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index b32f1ee..4ba6b32 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -53,11 +53,11 @@ def load_plugins(): """ plugins = [] for plugin in kiskadee_plugins_list: - #TODO: I will think in a good architecture to anitya backends. + # TODO: I will think in a good architecture to anitya backends. if plugin.find('anitya_') == -1: if config[plugin + '_plugin'].getboolean('active'): - plugins.append(importlib.import_module('kiskadee.plugins.' - + plugin)) + plugins.append(importlib.import_module( + 'kiskadee.plugins.' + plugin)) return plugins @@ -95,7 +95,8 @@ else: log_file = config['DEFAULT']['log_file'] logger = logging.getLogger(__name__) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') if log_file != 'stdout': _debug = logging.FileHandler(log_file, mode='w+') diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index b867e33..09ed83e 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -14,6 +14,7 @@ from kiskadee.model import Package, Plugin, Version, Base RUNNING = True + class Monitor: """Provide kiskadee monitoring objects.""" @@ -91,8 +92,10 @@ class Monitor: self.session.add(_package) kiskadee.logger.debug("Saving package in db: {}".format(str(pkg))) self.session.commit() - kiskadee.logger.debug("Enqueue package {}_{} " - " for analysis".format(pkg['name'], pkg['version'])) + kiskadee.logger.debug( + "Enqueue package {}_{} " + " for analysis".format(pkg['name'], pkg['version']) + ) kiskadee.queue.enqueue_analysis(pkg) def _update_pkg_version(self, pkg): @@ -108,9 +111,10 @@ class Monitor: _pkg.versions.append(_new_version) self.session.add(_pkg) self.session.commit() - kiskadee.logger.debug("Enqueue package {}_{}" - "for analysis".format(pkg['name'], - pkg['version'])) + kiskadee.logger.debug( + "Enqueue package {}_{}" + "for analysis".format(pkg['name'], pkg['version']) + ) kiskadee.queue.enqueue_analysis(pkg) else: return {} diff --git a/kiskadee/plugins/__init__.py b/kiskadee/plugins/__init__.py index b9e9100..4844a09 100644 --- a/kiskadee/plugins/__init__.py +++ b/kiskadee/plugins/__init__.py @@ -7,7 +7,6 @@ Plugin class here defined. import abc import kiskadee import inspect -import logging class Plugin(): diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index cf669dd..487a317 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -1,17 +1,20 @@ +"""Plugin to monitor Anitya events related to packages updates.""" + import tempfile import zmq import yaml -from time import sleep from packaging import version import fedmsg.consumers import time import urllib.request -import kiskadee.helpers +import kiskadee.helpers import kiskadee.plugins import kiskadee.queue + class Plugin(kiskadee.plugins.Plugin): + """Plugin to monitor Anitya (https://release-monitoring.org) Events.""" def watch(self): """Start the monitoring process for Anitya reports. @@ -21,27 +24,25 @@ class Plugin(kiskadee.plugins.Plugin): The plugin will use zmq as messaging protocol to receive the fedmsg-hub events. Kiskadee and fedmsg-hub runs in different - processes, so we need something to enable the comunication between then. - When a message come to fedmsg-hub, the AnityaConsumer instance, will - publish this event to zmq server, and kiskadee will consume this message. - """ + processes, so we need something to enable the + comunication between then. When a message come to fedmsg-hub, + the AnityaConsumer instance, will publish this event to zmq server, + and kiskadee will consume this message. - context = zmq.Context() - socket = context.socket(zmq.SUB) + """ kiskadee.logger.debug("Starting anitya plugin") - socket.connect("tcp://localhost:5556") - socket.setsockopt(zmq.SUBSCRIBE, b"anitya") - kiskadee.logger.debug("Connecting to 0mq server") - - while True: - msg = socket.recv_string() - msg = msg[msg.find(" ")+1::] - msg_dict = yaml.load(msg) - self._create_package_dict(msg_dict) + socket = self._connect_to_zmq( + self.config["zmq_port"], + self.config["zmq_topic"]) + if socket: + while True: + msg = socket.recv_string() + self._create_package_dict(msg) return {} def get_sources(self, source_data): + """Download packages from some Anitya Backend.""" path = tempfile.mkdtemp() backend_name = source_data.get('meta').get('backend').lower() run_backend = self._load_backend(backend_name) @@ -51,6 +52,7 @@ class Plugin(kiskadee.plugins.Plugin): return {} def compare_versions(self, new, old): + """Compare anitya source versions. If new > old, returns true.""" return version.parse(new) > version.parse(old) def _load_backend(self, backend_name): @@ -58,27 +60,54 @@ class Plugin(kiskadee.plugins.Plugin): backend = Backends() return getattr(backend, backend_name) except: - kiskadee.logger.debug("Backend not "\ - "suported: {}".format(str(backend_name))) + kiskadee.logger.debug( + "Backend not suported: {}".format(str(backend_name)) + ) return {} + def _connect_to_zmq(self, port, topic): + try: + context = zmq.Context() + socket = context.socket(zmq.SUB) + socket.connect("tcp://localhost:{}".format(port)) + socket.setsockopt_string( + zmq.SUBSCRIBE, topic) + kiskadee.logger.debug("Connecting to 0mq server") + return socket + except Exception as err: + kiskadee.logger.debug("Could not connect to zmq server") + kiskadee.logger.debug(err) + return False + @kiskadee.queue.package_enqueuer - def _create_package_dict(self, msg): - return { - 'name': msg.get('body').get('msg').get('project').get('name'), - 'version': msg.get('body').get('msg').get('project').get('version'), - 'plugin': kiskadee.plugins.anitya, - 'meta': { - 'backend': msg.get('body').get('msg').get('project').get('backend'), - 'homepage': msg.get('body').get('msg').get('project').get('homepage') + def _create_package_dict(self, msg_as_string): + msg_as_string = msg_as_string[msg_as_string.find(" ")+1::] + msg = yaml.load(msg_as_string) + _msg = msg.get('body').get('msg').get('project') + source_dict = {} + if _msg: + source_dict = { + 'name': _msg.get('name'), + 'version': _msg.get('version'), + 'plugin': kiskadee.plugins.anitya, + 'meta': { + 'backend': _msg.get('backend'), + 'homepage': _msg.get('homepage') } - } + } + return source_dict class Backends(): + """Class to implement Anitya Backends. + + Each method implemented in this class, should returns a absolute path + to the downloaded source, or a empty dict if the download could + not be made. + """ - @staticmethod - def github(source_data, path): + def github(self, source_data, path): + """Backend implementation to download github sources.""" source_version = ''.join([source_data.get('version'), '.tar.gz']) homepage = source_data.get('meta').get('homepage') @@ -95,17 +124,20 @@ class Backends(): class AnityaConsumer(fedmsg.consumers.FedmsgConsumer): + """Consumer used by fedmsg-hub to subscribe to fedmsg bus.""" topic = 'org.release-monitoring.prod.anitya.project.version.update' config_key = 'anityaconsumer' validate_signatures = False def __init__(self, *args, **kw): + """Anityaconsumer constructor.""" super().__init__(*args, *kw) context = zmq.Context() self.socket = context.socket(zmq.PUB) self.socket.bind("tcp://*:5556") def consume(self, msg): + """Consume events from fedmsg-hub.""" self.socket.send_string("%s %s" % ("anitya", str(msg))) time.sleep(1) diff --git a/kiskadee/plugins/anitya_github.py b/kiskadee/plugins/anitya_github.py deleted file mode 100644 index e69de29..0000000 --- a/kiskadee/plugins/anitya_github.py +++ /dev/null diff --git a/kiskadee/plugins/debian.py b/kiskadee/plugins/debian.py index dfd99bd..a3141a4 100644 --- a/kiskadee/plugins/debian.py +++ b/kiskadee/plugins/debian.py @@ -1,10 +1,3 @@ -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. """Plugin to monitor Debian Project Packages.""" import os @@ -53,8 +46,9 @@ class Plugin(kiskadee.plugins.Plugin): subprocess.check_output(['dget', url]) return ''.join([path, '/', self._source_path(path)]) except: - kiskadee.logger.debug('Cannot download {} source'. - format(source_data['name'])) + kiskadee.logger.debug( + 'Cannot download {} source' + .format(source_data['name'])) return None def compare_versions(self, new, old): diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 76e1905..aad3cd9 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -62,8 +62,10 @@ def analyze(package): return: list with firehose reports """ plugin = package['plugin'].Plugin() - kiskadee.logger.debug('ANALYSIS: Downloading {} ' - 'source...'.format(package['name'])) + kiskadee.logger.debug( + 'ANALYSIS: Downloading {} ' + 'source...'.format(package['name']) + ) compressed_source = plugin.get_sources(package) if compressed_source: kiskadee.logger.debug('ANALYSIS: Downloaded!') @@ -78,13 +80,17 @@ def analyze(package): kiskadee.logger.debug('ANALYSIS: running %s ...' % analyzer) try: analysis = kiskadee.analyzers.run(analyzer, path) - firehose_report = kiskadee.converter.to_firehose(analysis, - analyzer) + firehose_report = kiskadee.converter.to_firehose( + analysis, analyzer + ) reports.append(str(firehose_report)) - kiskadee.logger.debug('ANALYSIS: DONE running %s' % analyzer) + kiskadee.logger.debug( + 'ANALYSIS: DONE running %s' % analyzer + ) except: - kiskadee.logger.debug('ERROR: Could not run'\ - 'analysis inside container') + kiskadee.logger.debug( + 'ERROR: Could not run analysis inside container' + ) # TODO: remove compressed/uncompressed files after the analysis return reports else: diff --git a/kiskadee/tests/test_plugins.py b/kiskadee/tests/test_plugins.py index 908150a..b595c9b 100644 --- a/kiskadee/tests/test_plugins.py +++ b/kiskadee/tests/test_plugins.py @@ -2,6 +2,10 @@ from unittest import TestCase import os import shutil import tempfile +import zmq +import threading +import time + import kiskadee.queue @@ -89,3 +93,99 @@ class TestDebianPlugin(TestCase): old = '3.3.3-0' result = self.debian_plugin.compare_versions(new, old) self.assertFalse(result) + + +class TestAnityaPlugin(TestCase): + + def setUp(self): + import kiskadee.plugins.anitya + self.anitya_plugin = kiskadee.plugins.anitya.Plugin() + + self.msg = "anitya {'body':{'msg':{'project':{name: 'urlscan',"\ + "'version':'0.8.5','backend':'GitHub',"\ + "'homepage':'https://github.com/firecat53/urlscan'}}}}" + + self.msg1 = "{'body':{'msg':{'project':{name: 'urlscan',"\ + "'version':'0.8.5','backend':'GitHub',"\ + "'homepage':'https://github.com/firecat53/urlscan'}}}}" + + def test_connect_to_zmq(self): + + def zmq_server(): + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:7776") + + zmq_server() + socket = self.anitya_plugin._connect_to_zmq("7776", "anitya") + self.assertIsNotNone(socket) + + def test_receive_msg_from_zmq(self): + """definitely this is not a unit test, but is important to kiskadee + be able to interact correctly with ZeroMQ. + We need to define other test levels to kiskadee asap. + When we do that, we can move integration tests + to a proper place. For now we will maintain this test here""" + + def zmq_server(): + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:5556") + time.sleep(1) + socket.send_string("%s" % (self.msg)) + time.sleep(1) + + def receive_msg_from_server(): + client_socket = self.anitya_plugin._connect_to_zmq( + "5556", "anitya") + if client_socket: + response = client_socket.recv_string() + results[0] = response[response.find(" ")+1::] + else: + results[0] = "invalid" + + results = [None] + + client_as_thread = threading.Thread(target=receive_msg_from_server) + server_as_thread = threading.Thread( + target=zmq_server) + + server_as_thread.start() + client_as_thread.start() + server_as_thread.join() + self.assertEqual(self.msg1, results[0]) + + def test_compare_versions(self): + is_greater = self.anitya_plugin.compare_versions('0.8.5-2', '0.8.5-1') + self.assertTrue(is_greater) + + def test_load_backend(self): + backend = self.anitya_plugin._load_backend('github') + self.assertIsNotNone(backend) + + def test_not_load_backend(self): + backend = self.anitya_plugin._load_backend('foo') + self.assertEqual(backend, {}) + + def test_get_sources(self): + + def mock_github(self, source_data, path): + return 'kiskadee/tests/test_source/Sources.gz' + + kiskadee.plugins.anitya.Backends.github = mock_github + source_data = {'meta': {'backend': 'GitHub'}} + source_path = self.anitya_plugin.get_sources(source_data) + self.assertEqual(source_path, mock_github("self", "foo", "bla")) + + def test_create_package_dict(self): + + self.anitya_plugin._create_package_dict(self.msg) + _dict = kiskadee.queue.dequeue_package() + self.assertEqual(_dict['name'], 'urlscan') + self.assertEqual(_dict['version'], '0.8.5') + self.assertEqual(_dict['meta']['backend'], 'GitHub') + self.assertEqual( + _dict['meta']['homepage'], + 'https://github.com/firecat53/urlscan' + ) + self.assertEqual(_dict['plugin'], kiskadee.plugins.anitya) diff --git a/requirements.txt b/requirements.txt index cdb08e8..52a437b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,5 @@ Sphinx fedmsg packaging pyyaml +flake8 +pydocstyle diff --git a/util/kiskadee.conf b/util/kiskadee.conf index 1df6b14..323b284 100644 --- a/util/kiskadee.conf +++ b/util/kiskadee.conf @@ -29,6 +29,8 @@ target = release-monitoring.org description = A plugin to monitor Anitya packages analyzers = cppcheck active = yes +zmq_port = 5556 +zmq_topic = anitya [example_plugin] target = example From 9c9052837e0865a37954b9d298e0056e5d53a899 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 9/15] Add a brief description of each kiskadee plugin. --- diff --git a/doc/plugins.rst b/doc/plugins.rst index 3d600dc..d904f8c 100644 --- a/doc/plugins.rst +++ b/doc/plugins.rst @@ -45,3 +45,33 @@ A simple example of a kiskadee plugin This method does not matter here, let's just pass """ return 0 + +List of kiskadee plugins +---------------------------- + +Inside tha package `plugins` you can check which plugins kiskadee have, +and which targets are monitored with this plugins. This section is a brief +overview of this plugins. + + - *debian.py*: A plugin to monitor the Debian ftp repository. This plugin + will every hour downloads the *Sources.gz* file of the repository and + loads it in memory. This file is a representation of all the packages + present in the repository. After kiskadee loads it in memory, all the + packages are compared with the database, and if a new package is + identified, it source code is downloaded, and a analysis is made. + + - *anitya.py*: A plugin to monitor fedmsg events, published on the + Anitya project. The Anitya project monitor upstream releases and + broadcast them on fedmsg. The plugin will consume this events, and + trigger analysis when it's possible. + + + - *juliet.py*: Juliet is a static analysis test suite provided by + NIST's SAMATE team. It contains injected, known CWE's in specific + points and similar code snippets with the injected flaws fixed. This + plugin downloads the source code of this test suite, and run static + analyzers on it. + + - *example.py*: A simple example of a kiskadee plugin. Can be used as + start point for new plugins. + From 15bd17fdec193990fae47996713719fe101e0fbe Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:50:48 +0000 Subject: [PATCH 10/15] Updates README - Adds anityaconsumer.py --- diff --git a/README.md b/README.md index a618aff..4a61046 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,27 @@ permissions so the container can access external files. kiskadee looks for its configuration file under `/etc/kiskadee.conf`. +### Anitya Plugin +If you intend to run the anitya plugin, you will have to install fedmsg-hub, +in order to kiskadee be able to consume the fedmsg events. +To install fedmsg-hub follow this steps inside the kiskadee root path: + + sudo yum install fedmsg-hub + virtualenv -p /usr/bin/python3 . + source bin/activate + sudo cp util/anityaconsumer.py /etc/fedmsg.d/anityaconsumer.py + pip install -e . + PYTHONPATH=$(pwd) fedmsg-hub + +With this steps, fedmsg-hub will instanciate `AnityaConsumer` and publish +the monitored events using ZeroMQ. When kiskadee starts it will consume +the messages published by the consumer, and will run the analysis. + +### Debian Plugin +If you intend to use the debian plugin, you will have to install the +`devscripts` package, in order use the necessary debian tools to run the +plugin. + ## Development kiskadee development is hosted at [pagure](https://pagure.io/kiskadee). Feel diff --git a/util/anityaconsumer.py b/util/anityaconsumer.py new file mode 100644 index 0000000..f5c5e0c --- /dev/null +++ b/util/anityaconsumer.py @@ -0,0 +1 @@ +config = dict(anityaconsumer=True) From 485b04348fc057c0bf2024d53f45410f3e2208b4 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:51:20 +0000 Subject: [PATCH 11/15] Avoid kiskadee to crash when downloading source. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f7c96..d092b6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - flawfinder analyzer. - scan-build (clang) analyzer. - frama-c analyzer. +- New anitya plugin. ## [0.1] - 2017-06-12 ### Added diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 487a317..9e21d6d 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -112,15 +112,23 @@ class Backends(): homepage = source_data.get('meta').get('homepage') if homepage.find("github") != -1: - with kiskadee.helpers.chdir(path): - url = ''.join([homepage, '/archive/', source_version]) - in_file = urllib.request.urlopen(url) - data = in_file.read() - with open(source_version, 'wb') as info: - info.write(data) - return ''.join([path, '/', source_version]) - else: - return {} + try: + with kiskadee.helpers.chdir(path): + url = ''.join([homepage, '/archive/', source_version]) + in_file = urllib.request.urlopen(url) + data = in_file.read() + with open(source_version, 'wb') as info: + info.write(data) + return ''.join([path, '/', source_version]) + except Exception as err: + kiskadee.logger.debug( + "Cannot download {} " + "source code".format(source_data["name"]) + ) + kiskadee.logger.debug(err) + return {} + + return {} class AnityaConsumer(fedmsg.consumers.FedmsgConsumer): From 325a7c4dc8da859967137c33aea608ad2f16a62a Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:51:32 +0000 Subject: [PATCH 12/15] Minor fixes in anitya plugin. - Create download method inside kiskadee.helpers - Improve code style. - Remove unnecessary code - Related to #18 --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 4ba6b32..63a5493 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -53,11 +53,9 @@ def load_plugins(): """ plugins = [] for plugin in kiskadee_plugins_list: - # TODO: I will think in a good architecture to anitya backends. - if plugin.find('anitya_') == -1: - if config[plugin + '_plugin'].getboolean('active'): - plugins.append(importlib.import_module( - 'kiskadee.plugins.' + plugin)) + if config[plugin + '_plugin'].getboolean('active'): + plugins.append(importlib.import_module( + 'kiskadee.plugins.' + plugin)) return plugins diff --git a/kiskadee/helpers.py b/kiskadee/helpers.py index b6bdb80..14d680b 100644 --- a/kiskadee/helpers.py +++ b/kiskadee/helpers.py @@ -8,8 +8,11 @@ """Helper functions needing a new home.""" import os +import urllib.request from contextlib import contextmanager +import kiskadee + @contextmanager def chdir(path): @@ -18,3 +21,27 @@ def chdir(path): os.chdir(path) yield os.chdir(initial_dir) + + +def download(path, url, file_name): + """Download something from the internet. + + :path: The path where the file will be placed when downloaded. + :url: Url of the file. + :file_name: The name of the file that will be saved on the disc. + :return: The absolute path to the downloaded file. + """ + try: + with chdir(path): + in_file = urllib.request.urlopen(url) + data = in_file.read() + with open(file_name, 'wb') as info: + info.write(data) + return ''.join([path, '/', file_name]) + except Exception as err: + kiskadee.logger.debug( + "Cannot download {} " + "source code".format(file_name) + ) + kiskadee.logger.debug(err) + return {} diff --git a/kiskadee/plugins/anitya.py b/kiskadee/plugins/anitya.py index 9e21d6d..2dd80a3 100644 --- a/kiskadee/plugins/anitya.py +++ b/kiskadee/plugins/anitya.py @@ -6,7 +6,6 @@ import yaml from packaging import version import fedmsg.consumers import time -import urllib.request import kiskadee.helpers import kiskadee.plugins @@ -80,23 +79,26 @@ class Plugin(kiskadee.plugins.Plugin): return False @kiskadee.queue.package_enqueuer - def _create_package_dict(self, msg_as_string): - msg_as_string = msg_as_string[msg_as_string.find(" ")+1::] - msg = yaml.load(msg_as_string) - _msg = msg.get('body').get('msg').get('project') + def _create_package_dict(self, fedmsg_event): + event = self._event_to_dict(fedmsg_event) + project = event.get('body').get('msg').get('project') source_dict = {} - if _msg: + if project: source_dict = { - 'name': _msg.get('name'), - 'version': _msg.get('version'), + 'name': project.get('name'), + 'version': project.get('version'), 'plugin': kiskadee.plugins.anitya, 'meta': { - 'backend': _msg.get('backend'), - 'homepage': _msg.get('homepage') + 'backend': project.get('backend'), + 'homepage': project.get('homepage') } } return source_dict + def _event_to_dict(self, msg): + msg = msg[msg.find(" ")+1::] + return yaml.load(msg) + class Backends(): """Class to implement Anitya Backends. @@ -110,25 +112,8 @@ class Backends(): """Backend implementation to download github sources.""" source_version = ''.join([source_data.get('version'), '.tar.gz']) homepage = source_data.get('meta').get('homepage') - - if homepage.find("github") != -1: - try: - with kiskadee.helpers.chdir(path): - url = ''.join([homepage, '/archive/', source_version]) - in_file = urllib.request.urlopen(url) - data = in_file.read() - with open(source_version, 'wb') as info: - info.write(data) - return ''.join([path, '/', source_version]) - except Exception as err: - kiskadee.logger.debug( - "Cannot download {} " - "source code".format(source_data["name"]) - ) - kiskadee.logger.debug(err) - return {} - - return {} + url = ''.join([homepage, '/archive/', source_version]) + return kiskadee.helpers.download(path, url, source_version) class AnityaConsumer(fedmsg.consumers.FedmsgConsumer): diff --git a/kiskadee/tests/test_plugins.py b/kiskadee/tests/test_plugins.py index b595c9b..de2b8dc 100644 --- a/kiskadee/tests/test_plugins.py +++ b/kiskadee/tests/test_plugins.py @@ -130,14 +130,14 @@ class TestAnityaPlugin(TestCase): def zmq_server(): context = zmq.Context() socket = context.socket(zmq.PUB) - socket.bind("tcp://*:5556") + socket.bind("tcp://*:7776") time.sleep(1) socket.send_string("%s" % (self.msg)) time.sleep(1) def receive_msg_from_server(): client_socket = self.anitya_plugin._connect_to_zmq( - "5556", "anitya") + "7776", "anitya") if client_socket: response = client_socket.recv_string() results[0] = response[response.find(" ")+1::] From da3ba2a0bf9f22f62db094884eb50417cf6eda96 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 18:51:36 +0000 Subject: [PATCH 13/15] Updates README - Add fedmsg-hub configuration files. --- diff --git a/README.md b/README.md index 4a61046..f6a19ee 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,11 @@ If you intend to run the anitya plugin, you will have to install fedmsg-hub, in order to kiskadee be able to consume the fedmsg events. To install fedmsg-hub follow this steps inside the kiskadee root path: - sudo yum install fedmsg-hub - virtualenv -p /usr/bin/python3 . - source bin/activate - sudo cp util/anityaconsumer.py /etc/fedmsg.d/anityaconsumer.py - pip install -e . - PYTHONPATH=$(pwd) fedmsg-hub + sudo mkdir -p /etc/fedmsg.d/ + sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ + sudo cp util/anityaconsumer.py /etc/fedmsg.d/ + pip install -e . + PYTHONPATH=`pwd` fedmsg-hub With this steps, fedmsg-hub will instanciate `AnityaConsumer` and publish the monitored events using ZeroMQ. When kiskadee starts it will consume diff --git a/util/base.py b/util/base.py new file mode 100644 index 0000000..c7586ef --- /dev/null +++ b/util/base.py @@ -0,0 +1,83 @@ +# This file is part of fedmsg. +# Copyright (C) 2012 Red Hat, Inc. +# +# fedmsg is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# fedmsg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with fedmsg; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: Ralph Bean + +config = dict( + # Prefix for the topic of each message sent. + topic_prefix="org.fedoraproject", + + # Set this to dev if you're hacking on fedmsg or an app. + # Set to stg or prod if running in the Fedora Infrastructure + environment="dev", + + # Default is 0 + high_water_mark=0, + io_threads=1, + + # For the fedmsg-hub and fedmsg-relay. ## + + # This is a status dir to keep a record of the last processed message + # status_directory=os.getcwd() + "/status", + # status_directory='/var/run/fedmsg/status', + + # This is the URL of a datagrepper instance that we can query for backlog. + # datagrepper_url="https://apps.fedoraproject.org/datagrepper/raw", + + # We almost always want the fedmsg-hub to be sending messages with zmq as + # opposed to amqp or stomp. You can send with only *one* of the messaging + # backends: zeromq or amqp or stomp. You cannot send with two or more at + # the same time. Here, zmq is either enabled, or it is not. If it is not, + # see the options below for how to configure stomp or amqp. + zmq_enabled=True, + + # On the other hand, if you wanted to use STOMP *instead* of zeromq, you + # could do the following... + # zmq_enabled=False, + # stomp_uri='localhost:59597,localhost:59598', + # stomp_user='username', + # stomp_pass='password', + # stomp_ssl_crt='/path/to/an/optional.crt', + # stomp_ssl_key='/path/to/an/optional.key', + + # When subscribing to messages, we want to allow splats ('*') so we tell + # the hub to not be strict when comparing messages topics to subscription + # topics. + zmq_strict=False, + + # Number of seconds to sleep after initializing waiting for sockets to sync + post_init_sleep=0.5, + + # Wait a whole second to kill all the last io threads for messages to + # exit our outgoing queue (if we have any). This is in milliseconds. + zmq_linger=1000, + + # See the following + # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html + # - http://api.zeromq.org/3-2:zmq-setsockopt + zmq_tcp_keepalive=1, + zmq_tcp_keepalive_cnt=3, + zmq_tcp_keepalive_idle=60, + zmq_tcp_keepalive_intvl=5, + + # Number of miliseconds that zeromq will wait to reconnect until it gets + # a connection if an endpoint is unavailable. + zmq_reconnect_ivl=100, + # Max delay that you can reconfigure to reduce reconnect storm spam. This + # is in miliseconds. + zmq_reconnect_ivl_max=1000, +) diff --git a/util/endpoints.py b/util/endpoints.py new file mode 100644 index 0000000..640e203 --- /dev/null +++ b/util/endpoints.py @@ -0,0 +1,29 @@ +# This file is part of fedmsg. +# Copyright (C) 2012 Red Hat, Inc. +# +# fedmsg is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# fedmsg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with fedmsg; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: Ralph Bean +# +config = dict( + # This is a dict of possible addresses from which fedmsg can send + # messages. fedmsg.init(...) requires that a 'name' argument be passed + # to it which corresponds with one of the keys in this dict. + endpoints={ + "anitya-public-relay": [ + "tcp://release-monitoring.org:9940", + ], + }, +) From 1a5d8c916a5d2c551d0b89abbed05f0f79b869bf Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 19:34:10 +0000 Subject: [PATCH 14/15] Add psutil as dependency. --- diff --git a/requirements.txt b/requirements.txt index 52a437b..13187a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +psutil docker psycopg2 firehose>=0.5 From 7addb6239a615f0502d5c12872d08fee419a6d9f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 02 2017 19:37:31 +0000 Subject: [PATCH 15/15] Add step to install fedmsg[consumers] package. --- diff --git a/README.md b/README.md index f6a19ee..19439c7 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,12 @@ If you intend to run the anitya plugin, you will have to install fedmsg-hub, in order to kiskadee be able to consume the fedmsg events. To install fedmsg-hub follow this steps inside the kiskadee root path: + # Run this inside the kiskadee's virtualenv sudo mkdir -p /etc/fedmsg.d/ sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ sudo cp util/anityaconsumer.py /etc/fedmsg.d/ pip install -e . + pip install "fedmsg[consumers]" PYTHONPATH=`pwd` fedmsg-hub With this steps, fedmsg-hub will instanciate `AnityaConsumer` and publish