From 43a147d9ad94cf828460f25dc9e240f194d1c959 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 23 2017 10:00:07 +0000 Subject: [PATCH 1/7] Move creating the SRPM into mock instead of basic rpmbuild Fixes https://pagure.io/fedora-ci/simple-koji-ci/issue/1 Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index 4830a03..282c429 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -141,23 +141,49 @@ class Koji(object): new_sources = spec_sources(specfile, tmp) _log.info("Sources %r" % new_sources) + # Initialize the mock env + cmd = ['mock', '--init'] + _log.info("Calling %r" % ' '.join(cmd)) output = sp.check_output( - [ - 'rpmbuild', '-D', '_sourcedir .', '-D', '_topdir .', - '-bs', specfile], + cmd, cwd=tmp, stderr=sp.STDOUT, ) - - srpm = os.path.join(tmp, output.strip().split()[-1]) + _log.debug(output) + + # Create the SRPM + cmd = [ + 'mock', '--buildsrpm', '--define', + '"%_disable_source_fetch 0"', '--spec', specfile, + '--sources', tmp, '--resultdir', tmp + ] + _log.info("Calling %r" % ' '.join(cmd)) + output = sp.check_output( + cmd, + cwd=tmp, + stderr=sp.STDOUT, + ) + _log.debug(output) + + # Retrieve the path to the srpm + srpm = None + for f in os.listdir(tmp): + if f.endswith('.src.rpm'): + srpm = os.path.join(tmp, f) + break _log.debug("Got srpm %r" % srpm) + # Kick off scratch build session = self.session_maker() _log.debug("Kicking koji build for: %r (%r)" % (package, srpm)) task_id = self.scratch_build(session, package, srpm) _log.debug("Task ID: %r" % task_id) return task_id + except sp.CalledProcessError as err: + _log.debug(err.cmd) + _log.debug(err.returncode) + _log.debug(err.output) finally: _log.debug("Removing %r" % tmp) shutil.rmtree(tmp, ignore_errors=True) From db1aba72a033fe02e22b84f36065b5826f7c0e17 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:03:40 +0000 Subject: [PATCH 2/7] Fix building the srpm in mock, thanks @clime! Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index 282c429..01e2501 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -154,8 +154,9 @@ class Koji(object): # Create the SRPM cmd = [ 'mock', '--buildsrpm', '--define', - '"%_disable_source_fetch 0"', '--spec', specfile, - '--sources', tmp, '--resultdir', tmp + '%_disable_source_fetch 0', '--spec', specfile, + '--sources', tmp, '--resultdir', tmp, + '--old-chroot', ] _log.info("Calling %r" % ' '.join(cmd)) output = sp.check_output( From 857811b1c66f59c9f9a87d99d84318c08f47a777 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:04:03 +0000 Subject: [PATCH 3/7] Run fedpkg source before trying to create the srpm This should help in the cases when the sources did not change. Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index 01e2501..5e4b5e6 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -135,6 +135,8 @@ class Koji(object): url = self.git_url.format(package=package) _log.info("Cloning %r to %r" % (url, tmp)) sp.check_output(['git', 'clone', url, tmp], stderr=sp.STDOUT) + sp.check_output( + ['fedpkg', 'sources'], cwd=tmp, stderr=sp.STDOUT) specfile = tmp + '/' + package + '.spec' From ef662312d9de35b258f27aa59ac331ff5e0ebfaf Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:05:27 +0000 Subject: [PATCH 4/7] Do not download the sources manually Since mock does it for us now. Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index 5e4b5e6..e694a65 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -140,8 +140,6 @@ class Koji(object): specfile = tmp + '/' + package + '.spec' - new_sources = spec_sources(specfile, tmp) - _log.info("Sources %r" % new_sources) # Initialize the mock env cmd = ['mock', '--init'] From c2c1dd42676c4538ae9a78c06981e443cfce8d73 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:05:47 +0000 Subject: [PATCH 5/7] Checkout the pull-request's head so we actually test the PR Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index e694a65..dc54102 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -137,6 +137,12 @@ class Koji(object): sp.check_output(['git', 'clone', url, tmp], stderr=sp.STDOUT) sp.check_output( ['fedpkg', 'sources'], cwd=tmp, stderr=sp.STDOUT) + sp.check_output([ + 'git', 'fetch', 'origin', 'pull/%s/head:pr_%s' % (prid, prid) + ], cwd=tmp, stderr=sp.STDOUT) + sp.check_output( + ['git', 'checkout', 'pr_%s' % prid], + cwd=tmp, stderr=sp.STDOUT) specfile = tmp + '/' + package + '.spec' From 4bc7296de620c1a9c0eb4494e749169d4a026f92 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:06:12 +0000 Subject: [PATCH 6/7] Drop requirement on sphinx Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev-requirements.txt b/dev-requirements.txt index 535234c..932a895 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1 @@ mock -sphinx From 54317c8ac99d85408996402c9c25c42fabc7bc8e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 30 2017 11:15:02 +0000 Subject: [PATCH 7/7] Project wide flake8 cleaning Signed-off-by: Pierre-Yves Chibon --- diff --git a/simple_koji_ci/buildsys.py b/simple_koji_ci/buildsys.py index dc54102..6818d79 100644 --- a/simple_koji_ci/buildsys.py +++ b/simple_koji_ci/buildsys.py @@ -21,7 +21,6 @@ from __future__ import absolute_import, print_function from warnings import warn -import hashlib import logging import os import random @@ -32,11 +31,8 @@ import tempfile import threading import time -from six.moves.urllib.parse import urlparse import koji -from simple_koji_ci import exceptions - _log = logging.getLogger(__name__) @@ -61,9 +57,13 @@ class Koji(object): try: self.email_user = config['user_email'] except KeyError: - msg = 'userstring is deprecated, please use the "email_user" tuple' + msg = 'userstring is deprecated, please use the "email_user" '\ + 'tuple' warn(msg, DeprecationWarning) - self.email_user = [p.strip() for p in config['userstring'].rsplit('<', 1)] + self.email_user = [ + p.strip() + for p in config['userstring'].rsplit('<', 1) + ] self.email_user[1] = '<' + self.email_user[1] self.opts = config['opts'] self.priority = config['priority'] @@ -72,7 +72,8 @@ class Koji(object): def session_maker(self): _log.info('Creating a new Koji session to %s', self.server) with _koji_session_lock: - koji_session = koji.ClientSession(self.server, self.krb_sessionopts) + koji_session = koji.ClientSession( + self.server, self.krb_sessionopts) result = koji_session.krb_login( principal=self.krb_principal, keytab=self.krb_keytab, @@ -146,7 +147,6 @@ class Koji(object): specfile = tmp + '/' + package + '.spec' - # Initialize the mock env cmd = ['mock', '--init'] _log.info("Calling %r" % ' '.join(cmd)) @@ -194,91 +194,3 @@ class Koji(object): finally: _log.debug("Removing %r" % tmp) shutil.rmtree(tmp, ignore_errors=True) - - -def _validate_spec_urls(specfile_path): - """ - Validate a specfile's Source URLs. - - Args: - specfile_path (str): The path to the specfile to parse and validate. - - Raises: - exceptions.SpecUrlException: If the specfile contains Source URLs that - are invalid. - """ - # The output of spectool -l is in the format: - # Source0: some-string-we-want-to-be-a-url.tar.gz - # Source1: some-string-we-want-to-be-a-url.tar.gz - # ... - # Patch0: patch-we-expect-to-be-in-dist-git.patch - # ... - output = sp.check_output(['spectool', '-l', specfile_path]) - for line in output.splitlines(): - if line.startswith('Source'): - # Parse to make sure it's a url - url = line.split(':', 1)[1].strip() - parsed_url = urlparse(url) - if not parsed_url.scheme or not parsed_url.netloc: - msg = ("One or more of the specfile's Sources is not a valid URL " - "so we cannot automatically build the new version for you. " - "Please use a URL in your Source declarations if possible.") - raise exceptions.SpecUrlException(msg) - - -def spec_sources(specfile_path, target_dir): - """ - Retrieve a specfile's sources and store them in the given target directory. - - Example: - >>> spec_sources('/path/to/specfile', '/tmp/dir') - ['/tmp/dir/source0.tar.gz', '/tmp/dir/source1.tar.gz'] - - Args: - specfile_path (str): The filesystem path to the specfile - target_dir (str): The directory is where the file(s) will be saved. - - Returns: - list: A list of absolute paths to source files downloaded - - Raises: - exceptions.SpecUrlException: If the specfile contains Source URLs that - are invalid. - exceptions.DownloadException: If a networking-related error occurs while - downloading the specfile sources. This includes hostname resolution, - non-200 HTTP status codes, SSL errors, etc. - """ - _validate_spec_urls(specfile_path) - files = [] - try: - output = sp.check_output(['spectool', '-g', specfile_path], cwd=target_dir) - for line in output.splitlines(): - if line.startswith('Getting'): - files.append(os.path.realpath(os.path.join(target_dir, line.split()[-1]))) - except sp.CalledProcessError as e: - # spectool passes the cURL exit codes back so see its manpage for the full list - if e.returncode == 1: - # Unknown protocol (e.g. not ftp, http, or https) - msg = ('The specfile contains a Source URL with an unknown protocol; it should' - 'be "https", "http", or "ftp".') - raise exceptions.SpecUrlException(msg) - elif e.returncode in (5, 6): - msg = "Unable to resolve the hostname for one of the package's Source URLs" - elif e.returncode == 7: - # Failed to connect to the host - msg = "Unable to connect to the host for one of the package's Source URLs" - elif e.returncode == 22: - # cURL uses 22 for 400+ HTTP errors; the final line contains the specific code - msg = ("An HTTP error occurred downloading the package's new Source URLs: " + - e.output.splitlines()[-1]) - elif e.returncode == 60: - msg = ("Unable to validate the TLS certificate for one of the package's" - "Source URLs") - else: - msg = (u'An unexpected error occurred while downloading the new package sources; ' - u'please report this as a bug on the-new-hotness issue tracker.') - _log.error('{cmd} failed (exit {code}): {msg}'.format( - cmd=e.cmd, code=e.returncode, msg=e.output)) - raise exceptions.DownloadException(msg) - - return files diff --git a/simple_koji_ci/consumers.py b/simple_koji_ci/consumers.py index e33630f..b4f8e8e 100644 --- a/simple_koji_ci/consumers.py +++ b/simple_koji_ci/consumers.py @@ -22,15 +22,13 @@ Authors: Ralph Bean """ import logging -import socket -import subprocess from requests.packages.urllib3.util import retry import fedmsg import fedmsg.consumers import requests -from simple_koji_ci import exceptions, __version__ +from simple_koji_ci import __version__ import simple_koji_ci.buildsys import simple_koji_ci.cache @@ -120,7 +118,8 @@ class SimpleKojiCi(fedmsg.consumers.FedmsgConsumer): _log.info("Configuring cache.") with simple_koji_ci.cache.cache_lock: if not hasattr(simple_koji_ci.cache.cache, 'backend'): - simple_koji_ci.cache.cache.configure(**self.config['simple-koji-ci.cache']) + simple_koji_ci.cache.cache.configure( + **self.config['simple-koji-ci.cache']) # Retrieve the requests configuration; by default requests time out # after 15 seconds and are retried up to 3 times. @@ -130,14 +129,16 @@ class SimpleKojiCi(fedmsg.consumers.FedmsgConsumer): self.config.get('simple-koji-ci.read_timeout', 15), ) retries = self.config.get('simple-koji-ci.requests_retries', 3) - retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1) + retry_conf = retry.Retry( + total=retries, connect=retries, read=retries, backoff_factor=1) retry_conf.BACKOFF_MAX = 5 self.requests_session.mount( 'http://', requests.adapters.HTTPAdapter(max_retries=retry_conf)) self.requests_session.mount( 'https://', requests.adapters.HTTPAdapter(max_retries=retry_conf)) _log.info('Requests timeouts are {}s (connect) and {}s (read)' - ' with {} retries'.format(self.timeout[0], self.timeout[1], retries)) + ' with {} retries'.format( + self.timeout[0], self.timeout[1], retries)) # Build a little store where we'll keep track of what koji scratch # builds we have kicked off. We'll look later for messages indicating @@ -155,8 +156,6 @@ class SimpleKojiCi(fedmsg.consumers.FedmsgConsumer): if topic.endswith('pagure.pull-request.new'): self.handle_pagure_new_pr(msg) - #elif topic.endswith('anitya.project.map.new'): - #self.handle_anitya_map_new(msg) elif topic.endswith('buildsys.task.state.change'): self.handle_buildsys_scratch(msg) else: @@ -246,7 +245,7 @@ class SimpleKojiCi(fedmsg.consumers.FedmsgConsumer): headers = { "Authorization": "token " - + self.config["simple-koji-ci.pagure_token"], + + self.config["simple-koji-ci.pagure_token"], 'User-Agent': 'simple-koji-ci %s' % __version__, } data = {