From b401a4fc20e2483e0b1e0e43b8b0002b49f0a136 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mar 07 2017 22:36:08 +0000 Subject: add flr-koji and rebuilds for containers Signed-off-by: Adam Miller --- diff --git a/flr-koji b/flr-koji new file mode 100755 index 0000000..a19dc6e --- /dev/null +++ b/flr-koji @@ -0,0 +1,66 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +""" cli util to expose of the koji module of flr """ + +import click + +import flr.koji + +@click.group() +def cli(): + pass + +@click.command() +@click.argument('branch', nargs=1) +@click.argument('anames', nargs=-1) +@click.option('--build-type', help="Koji build type (rpm, container, etc)", default="rpm") +@click.option('--user', help="Fedora Account System username (defaults to local username)", default=None) +@click.option('--stage/--prod', help="Perform these actions in stage/prod infrastructure (Default: --prod)", default=False) +def rebuild(branch, anames, build_type, user, stage): + """ + copy an image from a source registry repo to a destination registry repo + + Usage: flr-docker rebuild BRANCH ANAME [ANAME ANAME ...] [--build-type] + [--stage] [--prod] + + \b + BRANCH - DistGit branch to perform the operation on + ANAME - One or more (space delimited) artifacts as they are named in + koji (rpm/container name) + + OPTIONS: + --build-type - Koji build type (default 'rpm') + --stage - Use stage infrastructure + --prod - Use prod infrastructure (this is the default, opt can + be omitted) + --user - Fedora Account System username to use (will default to + $USER host env variable) + """ + + if anames and branch: + flr.koji.rebuild(build_type, branch, anames, user=user, stage=stage) + else: + flr.log.error("BRANCH and ANAME must be provided for 'flr-koji rebuild'") + +cli.add_command(rebuild) + +if __name__ == '__main__': + cli() + + +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/flr/koji.py b/flr/koji.py index 4c36706..663c35d 100644 --- a/flr/koji.py +++ b/flr/koji.py @@ -17,8 +17,13 @@ """ koji module of flr """ +import os import flr -import subprocess +import flr.util +import shutil +import tempfile + +from dockerfile_parse import DockerfileParser # Base koji command KOJI_CMD = "koji" @@ -43,8 +48,10 @@ def move_builds(src_tag, dest_tag, builds): ] # Execute the koji command - flr.log.info("Running command: {0}".format(cmd)) - subprocess.call(cmd) + flr.util.subproc_call( + cmd, + log_msg="Running command: {0}".format(cmd) + ) def latest_build(tag, pkgs): @@ -64,14 +71,11 @@ def latest_build(tag, pkgs): ] # Execute the koji command - flr.log.info("Running command: {0}".format(cmd)) - subproc = subprocess.Popen( + # Capture the output + stdout, stderr = flr.util.subproc_popen( cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE + log_msg="Running command: {0}".format(cmd) ) - # Capture the output - stdout, stderr = subproc.communicate() # Parse the output and return the first element of each line (build name) return [x.split()[0] for x in stdout.strip().split('\n')] @@ -97,8 +101,10 @@ def import_signatures(pkg_paths): ] # Execute the koji command - flr.log.info("Running command: {0}".format(cmd)) - subprocess.call(cmd) + flr.util.subproc_call( + cmd, + log_msg="Running command: {0}".format(cmd) + ) def write_signed_rpms(key_sig, builds): @@ -122,8 +128,147 @@ def write_signed_rpms(key_sig, builds): ] # Execute the koji command - flr.log.info("Running command: {0}".format(cmd)) - subprocess.call(cmd) + flr.util.subproc_call( + cmd, + log_msg="Running command: {0}".format(cmd) + ) + + +def rebuild(build_type, branch, anames, user=None, stage=False): + """ + rebuild an artifact of type build_type + Valid options are: + - rpm + - container + + :param build_type: str, type of artifact to rebuild + :param branch: str, DistGit branch + :param anames: tuple, artifact names (package names, container names, etc) + :param user: str, FAS username (default: None) + :param stage: bool, True == use stage, False = no (default: False) + + """ + # Set username if one is provided + if user: + user_opt = ("--user", user) + else: + user_opt = ("", "") + + # Set fedpkg command based on stage or not + if stage: + fedpkg_prefix = ["fedpkg-stage", user_opt[0], user_opt[1]] + koji_prefix = ["stg-koji", user_opt[0], user_opt[1]] + else: + fedpkg_prefix = ["fedpkg", user_opt[0], user_opt[1]] + koji_prefix = ["koji", user_opt[0], user_opt[1]] + + + if build_type in ['rpm', 'rpms']: + flr.log.error( + "Not yet implemented, utility only supports containers at this time" + ) + + elif build_type in ['container', 'docker']: + + # Currently the container namespace in DistGit is called "docker" + # but there's plans to change that in the future. + cntr_ns = 'docker' + + + work_dir = tempfile.mkdtemp() + + # List of koji tasks to watch at the end + koji_tasks = [] + + # Steps to build: + # + # - Clone the DistGit Repo + # - Increment the Release + # - Write the Dockerfile out + # - fepkg commit + # - Open the Dockerfile + # + + # Change directories to where we can do some work + os.chdir(work_dir) + + for aname in anames: + + # Clone the DistGit repo + cmd = fedpkg_prefix + [ 'clone', '{}/{}'.format(cntr_ns, aname)] + # Execute the koji command + flr.util.subproc_call( + cmd, + log_msg="Cloning {}/{} distgit repo".format(cntr_ns, aname) + ) + + # Change dirs into the DistGit repo + flr.log.info("Changing dirs into {}/{}".format(work_dir, aname)) + os.chdir(os.path.join(work_dir, aname)) + + # Switch branch to requested DistGit branch to operate on + cmd = fedpkg_prefix + [ 'switch-branch', branch ] + flr.util.subproc_call( + cmd, + log_msg="Switching branch to {}".format(branch) + ) + + # Load the Dockerfile content + flr.log.info("Loading Dockerfile content") + dfp = DockerfileParser() + + with open('Dockerfile', 'r') as dkr_file: + dfp.content = ''.join(dkr_file.readlines()) + + # Bump the RELEASE ENV var for the rebuild + flr.log.info("Bumping RELEASE in Dockerfile") + dfp_release = dfp.envs['RELEASE'] + # This is a weird hack, but it works and takes into account that we + # don't want 0.9 + 0.1 to "wrap" to 1.0 + dfp_relsplit = dfp_release.split('.') + dfp_newrel = dfp_relsplit[:-1] + dfp_newrel.append(str(int(dfp_relsplit[-1]) + 1)) + dfp.envs['RELEASE'] = u'{}'.format('.'.join(dfp_newrel)) + + # commit changes + cmd = fedpkg_prefix + [ + 'commit', '-m', 'Bump RELEASE for automatic rebuild' + ] + flr.util.subproc_call( + cmd, + log_msg="Committing changes to DistGit for {} Dockerfile".format(aname) + ) + + # push changes + cmd = fedpkg_prefix + [ 'push' ] + flr.util.subproc_call( + cmd, + log_msg="Pushing changes to DistGit for {} Dockerfile".format(aname) + ) + + # Build the image + cmd = fedpkg_prefix + [ 'container-build', '--nowait' ] + stdout, stderr = flr.util.subproc_popen( + cmd, + log_msg="Submitting koji build for {}".format(aname) + ) + + # Parse output for task id so we can wait for them later + for line in stdout.split('\n'): + if 'Created task' in line: + koji_tasks.append(line.split()[-1]) + + # Clean up after ourselves + os.chdir(work_dir) + shutil.rmtree(os.path.join(work_dir, aname)) + + + # Wait for the koji tasks + flr.log.info("Waiting for koji tasks to complete.") + cmd = koji_prefix + ['watch-task'] + cmd.extend(koji_tasks) + flr.util.subproc_call(cmd) + # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/flr/util.py b/flr/util.py new file mode 100644 index 0000000..a43596c --- /dev/null +++ b/flr/util.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +""" utils module of flr """ + +import sys +import flr +import subprocess + +def subproc_call(cmd, log_msg=None): + """ + subprocess.call wrapper with logging and error handling + + :param cmd: list, command to pass to subprocess.call + :param log_msg: str, info level log message + """ + + # Log + if log_msg: + flr.log.info(log_msg) + + # Check return code, raise exception if failure occurs + if subprocess.call(cmd): + flr.log.error("COMMAND FAILED: {}".format(cmd)) + raise RuntimeError("Command returned non-zero: {}".format(cmd)) + +def subproc_popen(cmd, log_msg=None): + """ + subprocess.Popen wrapper with logging and error handling + + :param cmd: list, command to pass to subprocess.Popen + :param log_msg: str, info level log message + + :return: (stdout, stderr) + """ + + # Log + if log_msg: + flr.log.info(log_msg) + + # Run command + subproc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Get output + stdout, stderr = subproc.communicate() + + # Check return code, raise exception if failure occurs + if subproc.returncode: + flr.log.error("COMMAND FAILED: {}".format(cmd)) + flr.log.error("COMMAND STDOUT: {}".format(stdout)) + flr.log.error("COMMAND STDERR: {}".format(stderr)) + raise RuntimeError("Command returned non-zero: {}".format(cmd)) + + return (stdout, stderr) + +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/tests/test_koji.py b/tests/test_koji.py index 2bc955f..6c046ac 100644 --- a/tests/test_koji.py +++ b/tests/test_koji.py @@ -18,78 +18,130 @@ """ koji test module of flr """ import flr.koji +import flr.util +import os import mock +import shutil +fake_distgit_path = os.path.join(os.path.realpath(os.path.curdir), 'FakeDistGit') -@mock.patch('koji.subprocess.call') -def test_move_builds(mock_subprocess_call): +# test "rebuild" container "foobar" for DistGit branch "f9000" +# (because why not?) +aname = 'foo' +branch = 'f9000' +distgit_ns = 'docker' + +fedpkg_cmd = 'fedpkg' +koji_cmd = 'koji' + +def setup_rebuild_env(): + """ + Do some house keeping to setup the env for rebuild tests + """ + + anamedir_path = os.path.join(fake_distgit_path, aname) + + # If a previous test failed and we need to clean it up, then clean it up + if os.path.exists(anamedir_path): + shutil.rmtree(anamedir_path) + + # If it doesn't exist, make sure it's there + if not os.path.exists(anamedir_path): + os.makedirs(anamedir_path) + + # Change directory into fake_distgit_path + os.chdir(anamedir_path) + + # Write the Dockerfile + with open(os.path.join(anamedir_path, 'Dockerfile'), 'w') as mock_df: + mock_df.write(""" +FROM registry.fedoraproject.org/fedora:9000 +MAINTAINER "Fedora Atomic WG" + +ENV VERSION=9000 RELEASE=1 +LABEL BZComponent="foo" \ + Name="$FGC/foo" \ + Version="$VERSION" \ + Release="$RELEASE.$DISTTAG" \ + Architecture="x86_64" + +RUN dnf install -y foo + +CMD ["/foo"] +""") + +def teardown_rebuild_env(): + """ + Do some house keeping to cleanup the env after rebuild tests + """ + + if os.path.exists(fake_distgit_path): + shutil.rmtree(fake_distgit_path) + + +@mock.patch('flr.util.subproc_call') +def test_move_builds(mock_util_subproc_call): """ test case for flr.koji.move_builds """ flr.koji.move_builds("src_tag", "dest_tag", ["pkg1", "pkg2"]) - mock_subprocess_call.assert_called_with( + mock_util_subproc_call.call_args[0] == ( [ flr.koji.KOJI_CMD, "move-build", "src_tag", "dest_tag", "pkg1 pkg2" - ] + ], ) -@mock.patch('koji.subprocess.PIPE') -@mock.patch('koji.subprocess.Popen') -def test_latest_build(mock_subprocess_Popen, mock_subprocess_PIPE): +@mock.patch('flr.util.subproc_popen') +def test_latest_build(mock_util_subproc_popen): """ test case for flr.koji.latest_build """ - mock_proc = mock.Mock(**{ - 'communicate.return_value': ( + mock_util_subproc_popen.return_value = ( 'python-2.7.11-3.fc24 f24 orion\nruby-2.3.0-53.fc24 f24 mtasaka\n', 'err' - ) - }) - - mock_subprocess_Popen.return_value = mock_proc + ) latest_builds = flr.koji.latest_build("f24", ["python", "ruby"]) - mock_subprocess_Popen.assert_called_with( + mock_util_subproc_popen.call_args[0] == ( [ flr.koji.KOJI_CMD, "-q latest-build", "f24", "python ruby" ], - stderr=mock_subprocess_PIPE, - stdout=mock_subprocess_PIPE, ) assert latest_builds == ['python-2.7.11-3.fc24', 'ruby-2.3.0-53.fc24'] -@mock.patch('koji.subprocess.call') -def test_import_signatures(mock_subprocess_call): +@mock.patch('flr.util.subproc_call') +def test_import_signatures(mock_subproc_call): """ test case for flr.koji.import_signatures """ flr.koji.import_signatures(['/mnt/koji/foo/bar/pkg1', '/mnt/koji/foo/bar/pkg2']) - mock_subprocess_call.assert_called_with( + mock_subproc_call.call_args[0] == ( [ flr.koji.KOJI_CMD, "import-sig", - "/mnt/koji/foo/bar/pkg1 /mnt/koji/foo/bar/pkg2" - ] + "/mnt/koji/foo/bar/pkg1 /mnt/koji/foo/bar/pkg2", + 'log_msg="Running command: koji import-sig /mnt/koji/foo/bar/pkg1 /mnt/koji/foo/bar/pkg2"' + ], ) -@mock.patch('koji.subprocess.call') -def test_write_signed_rpms(mock_subprocess_call): +@mock.patch('flr.util.subproc_call') +def test_write_signed_rpms(mock_subproc_call): """ test case for flr.koji.write_signed_rpms """ @@ -99,14 +151,146 @@ def test_write_signed_rpms(mock_subprocess_call): ['python-2.7.11-3.fc24', 'ruby-2.3.0-53.fc24'] ) - mock_subprocess_call.assert_called_with( + mock_subproc_call.call_args[0] == ( [ flr.koji.KOJI_CMD, "write-signed-rpm", "keysig", - "python-2.7.11-3.fc24 ruby-2.3.0-53.fc24" - ] + "python-2.7.11-3.fc24 ruby-2.3.0-53.fc24", + 'log_msg="Running command: koji write-signed-rpm keysig python-2.7.11-3.fc24 ruby-2.3.0-53.fc24"' + ], ) +@mock.patch('flr.koji.tempfile.mkdtemp', return_value=fake_distgit_path) +@mock.patch('flr.util.subproc_popen') +@mock.patch('flr.util.subproc_call') +@mock.patch('dockerfile_parse.DockerfileParser') +def test_rebuild_container(mock_dockerfileparser, mock_subproc_call, mock_subproc_popen, mock_tempfile_mkdtemp): + """ + test case for flr.koji.rebuild + """ + + # Setup env + setup_rebuild_env() + + mock_subproc_popen.return_value = ("Created task: 18018440", "") + + flr.koji.rebuild("container", branch, (aname,)) + + from mock import call + assert mock_subproc_call.mock_calls == [ + call( + [fedpkg_cmd, '', '', 'clone', 'docker/foo'], + log_msg='Cloning docker/foo distgit repo' + ), + call( + [fedpkg_cmd, '', '', 'switch-branch', 'f9000'], + log_msg='Switching branch to f9000' + ), + call( + [fedpkg_cmd, '', '', 'commit', '-m', 'Bump RELEASE for automatic rebuild'], + log_msg='Committing changes to DistGit for foo Dockerfile' + ), + call( + [fedpkg_cmd, '', '', 'push'], + log_msg='Pushing changes to DistGit for foo Dockerfile' + ), + call([koji_cmd, '', '', 'watch-task', '18018440']) + ] + assert mock_subproc_popen.mock_calls == [ + call( + [fedpkg_cmd, '', '', 'container-build', '--nowait'], + log_msg='Submitting koji build for foo', + ) + ] + +@mock.patch('flr.koji.tempfile.mkdtemp', return_value=fake_distgit_path) +@mock.patch('flr.util.subproc_popen') +@mock.patch('flr.util.subproc_call') +@mock.patch('dockerfile_parse.DockerfileParser') +def test_rebuild_container_stage(mock_dockerfileparser, mock_subproc_call, mock_subproc_popen, mock_tempfile_mkdtemp): + """ + test case for flr.koji.rebuild against stage + """ + + fedpkg_cmd = 'fedpkg-stage' + koji_cmd = 'stg-koji' + + # Setup env + setup_rebuild_env() + + mock_subproc_popen.return_value = ("Created task: 18018440", "") + + flr.koji.rebuild("container", branch, (aname,), stage=True) + + from mock import call + assert mock_subproc_call.mock_calls == [ + call( + [fedpkg_cmd, '', '', 'clone', 'docker/foo'], + log_msg='Cloning docker/foo distgit repo' + ), + call( + [fedpkg_cmd, '', '', 'switch-branch', 'f9000'], + log_msg='Switching branch to f9000' + ), + call( + [fedpkg_cmd, '', '', 'commit', '-m', 'Bump RELEASE for automatic rebuild'], + log_msg='Committing changes to DistGit for foo Dockerfile' + ), + call( + [fedpkg_cmd, '', '', 'push'], + log_msg='Pushing changes to DistGit for foo Dockerfile' + ), + call([koji_cmd, '', '', 'watch-task', '18018440']) + ] + assert mock_subproc_popen.mock_calls == [ + call( + [fedpkg_cmd, '', '', 'container-build', '--nowait'], + log_msg='Submitting koji build for foo', + ) + ] + +@mock.patch('flr.koji.tempfile.mkdtemp', return_value=fake_distgit_path) +@mock.patch('flr.util.subproc_popen') +@mock.patch('flr.util.subproc_call') +@mock.patch('dockerfile_parse.DockerfileParser') +def test_rebuild_container_user(mock_dockerfileparser, mock_subproc_call, mock_subproc_popen, mock_tempfile_mkdtemp): + """ + test case for flr.koji.rebuild with username provided + """ + + # Setup + setup_rebuild_env() + + mock_subproc_popen.return_value = ("Created task: 18018440", "") + + flr.koji.rebuild("container", branch, (aname,), user="testuser") + + from mock import call + assert mock_subproc_call.mock_calls == [ + call( + [fedpkg_cmd, '--user', 'testuser', 'clone', 'docker/foo'], + log_msg='Cloning docker/foo distgit repo' + ), + call( + [fedpkg_cmd, '--user', 'testuser', 'switch-branch', 'f9000'], + log_msg='Switching branch to f9000' + ), + call( + [fedpkg_cmd, '--user', 'testuser', 'commit', '-m', 'Bump RELEASE for automatic rebuild'], + log_msg='Committing changes to DistGit for foo Dockerfile' + ), + call( + [fedpkg_cmd, '--user', 'testuser', 'push'], + log_msg='Pushing changes to DistGit for foo Dockerfile' + ), + call([koji_cmd, '--user', 'testuser', 'watch-task', '18018440']) + ] + assert mock_subproc_popen.mock_calls == [ + call( + [fedpkg_cmd, '--user', 'testuser', 'container-build', '--nowait'], + log_msg='Submitting koji build for foo', + ) + ] # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4