From af984b6803434fc4ae1d3c04ed386df87417ed61 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Aug 25 2017 09:02:16 +0000 Subject: [PATCH 1/30] WIP: STI compatibility --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 6a7118a..6dfd86a 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -99,17 +99,19 @@ class Executor(object): :param str ipaddr: IP address of the machine the task will be run on ''' - taskdir = os.path.dirname(self.arg_data['task']) - taskfile = os.path.basename(self.arg_data['task']) + if os.path.isfile(os.path.join(self.arg_data['taskdir'], 'inventory')): + sti_inventory = os.path.join(config.get_config().client_taskdir, "inventory") + else: + sti_inventory = "/usr/share/ansible/inventory" cmd = [ 'ansible-playbook', 'runner.yml', '--inventory=%s,' % ipaddr, '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'subjects=%s' % self.arg_data['item'], - '-e', 'taskdir=%s' % taskdir, - '-e', 'taskfile=%s' % taskfile, + '-e', 'taskdir=%s' % self.arg_data['taskdir'], '-e', 'client_taskdir=%s' % config.get_config().client_taskdir, + '-e', 'sti_inventory=%s' % sti_inventory, ] if self.run_remotely: diff --git a/libtaskotron/main.py b/libtaskotron/main.py index f3d46c9..0906cf1 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -65,7 +65,7 @@ def get_argparser(): pass parser = argparse.ArgumentParser(epilog=ITEM_TYPE_DOCS, formatter_class=CustomFormatter) - parser.add_argument("task", help="task playbook to run") + parser.add_argument("taskdir", help="taskdir with STI playbook to run") parser.add_argument("-a", "--arch", choices=["i386", "x86_64", "armhfp", "noarch"], default='noarch', help="architecture specifying the item to be checked. 'noarch' value " @@ -186,6 +186,10 @@ def main(): executor = Executor(arg_data) executor.execute() + # report results + # test_rc = read_test_rc() + # resultsdb.report(subject, rc, hurr, durr) + # finalize log.info('Execution finished at: %s. Task artifacts were saved in: %s', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'), diff --git a/runner.yml b/runner.yml index 8aaf7ab..2b6ee44 100644 --- a/runner.yml +++ b/runner.yml @@ -29,12 +29,23 @@ with_items: - ansible - libselinux-python + - standard-test-roles + - python2-dnf when: not local + #- name: Download subjects (rpms atm) + # our_custom_module: "{{}}" # probably wrapped koji + # export: subjects + - name: Run the task become: yes become_user: root - command: ansible-playbook "{{ client_taskdir }}/{{ taskfile }}" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ subjects }}" + # FIXME add context tags + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ subjects }}" + environment: + TEST_SUBJECTS: "{{ subjects }}" + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes register: output - name: Dump task output @@ -45,9 +56,16 @@ dest: "{{ artifacts }}/ansible.log" content: "{{ output.stdout }}" + - name: RC of test + shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + - name: Collect logs synchronize: mode: pull src: "{{ artifacts }}/*" dest: "{{ artifacts }}" when: not local + + #- name: generate resultsdb result file + # resultsdb: + # rc: {{ ... }} \ No newline at end of file From c9d42b4b00b2bbf2c3c4603a3be7bf7cfad1c2cb Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Aug 29 2017 12:21:02 +0000 Subject: [PATCH 2/30] Merge branch 'develop' into feature/ansiblize --- diff --git a/.arcconfig b/.arcconfig deleted file mode 100644 index e44f980..0000000 --- a/.arcconfig +++ /dev/null @@ -1,7 +0,0 @@ -{ - "project_id" : "libtaskotron", - "conduit_uri" : "https://phab.qa.fedoraproject.org", - "arc.land.onto.default" : "develop", - "arc.feature.start.default" : "develop", - "unit.engine" : "PytestTestEngine" -} diff --git a/.arclint b/.arclint deleted file mode 100644 index 290b9d2..0000000 --- a/.arclint +++ /dev/null @@ -1,12 +0,0 @@ -{ - "linters": { - "flake8": { - "type": "flake8", - "include": "(\\.py$)", - "severity.rules": { - "(^E)": "warning", - "(^F)": "error" - } - } - } -} diff --git a/README.rst b/README.rst index 5fab1a2..001a9da 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,8 @@ On your Fedora system, install the necessary packages:: rpm-build \ rpm-python +On Fedora 27+, install ``python2-koji`` instead of ``koji``. + If you have not yet cloned the repository, do it now:: git clone https://pagure.io/taskotron/libtaskotron.git @@ -109,7 +111,7 @@ There is a tmpfiles.d template prepared for you, look into ``conf/tmpfiles.d``. Running a Task ============== -A relatively simple example task is `rpmlint `_. +A relatively simple example task is `rpmlint `_. The task requires the rpmlint tool to be installed, so be sure to run:: @@ -117,7 +119,7 @@ The task requires the rpmlint tool to be installed, so be sure to run:: To run that task against a koji build with NVR ````, do the following:: - git clone https://bitbucket.org/fedoraqa/task-rpmlint.git + git clone https://pagure.io/taskotron/task-rpmlint.git runtask -i -t koji_build task-rpmlint/runtask.yml This will download the ```` from koji into a temp directory under diff --git a/arclint.readme b/arclint.readme deleted file mode 100644 index 094473c..0000000 --- a/arclint.readme +++ /dev/null @@ -1,24 +0,0 @@ -The linter's behaviour is configured in the .arclint file. - -As the .arclint file parser does not support YAML comments, some documentation -can be put here if needed. Generic arcanist linter documentation is here: -https://secure.phabricator.com/book/phabricator/article/arcanist_lint/ - -Flake8 -====== -All PEP8 error codes (E* codes) are considered warnings in Phabricator, so that -only changes to the modified lines are displayed by default (unlike errors, -which are always displayed regardless of which lines were modified). - -All PyFlakes error codes (F* codes) are considered errors, because they are -often introduced when modifying other lines than the one in question, and we -want to be notified of those. - -Additional Flake8 configuration is stored in tox.ini - it contains configuration -which is not possible to do in .arclint file. - -If you want to ignore a specific source code line, use '# noqa' comment. If you -want to ignore the whole file, add '# flake8: noqa' comment. Read more -documentation about flake8 at: -https://flake8.readthedocs.org/ - diff --git a/conf/namespaces.yaml.example b/conf/namespaces.yaml.example index 0df0d81..e82f212 100644 --- a/conf/namespaces.yaml.example +++ b/conf/namespaces.yaml.example @@ -5,8 +5,17 @@ namespaces_safe: # A list of git repos that are allowed to post a result into a particular namespace namespaces_whitelist: dist: - - git@bitbucket.org:fedoraqa/task-rpmlint.git - - git@bitbucket.org:fedoraqa/task-depcheck.git - - git@bitbucket.org:fedoraqa/task-upgradepath.git + - https://pagure.io/taskotron/task-rpmlint.git + - https://pagure.io/taskotron/task-upgradepath.git + - https://pagure.io/task-abicheck.git + - https://pagure.io/taskotron/task-rpmgrill.git + - https://github.com/fedora-python/taskotron-python-versions.git + - https://github.com/fedora-modularity/check_modulemd.git + - https://pagure.io/taskotron/task-rpmdeplint.git + - https://pagure.io/taskotron/task-upstream-atomic.git + - https://pagure.io/taskotron/task-fedora-cloud-tests.git + - https://pagure.io/taskotron/task-modularity-testing-framework.git pkg: - - git://pkgs.fedoraproject.org/rpms-checks/ + - git://pkgs.fedoraproject.org/test-rpms/ + - git://pkgs.fedoraproject.org/test-modules/ + - git://pkgs.fedoraproject.org/test-docker/ diff --git a/conf/yumrepoinfo.conf.example b/conf/yumrepoinfo.conf.example index aa4795b..3631ca9 100644 --- a/conf/yumrepoinfo.conf.example +++ b/conf/yumrepoinfo.conf.example @@ -36,15 +36,31 @@ release_status = [rawhide] url = %(rawhideurl)s path = development/rawhide -tag = f27 +tag = f28 release_status = rawhide -# Fedora 26 -[f26] +# Fedora 27 +[f27] url = %(rawhideurl)s -path = development/26 +path = development/27 release_status = branched +[f27-updates] +url = %(updatesurl)s +path = 27 +parent = f27 + +[f27-updates-testing] +url = %(updatesurl)s +path = testing/27 +parent = f27-updates + +# Fedora 26 +[f26] +url = %(goldurl)s +path = 26 +release_status = stable + [f26-updates] url = %(updatesurl)s path = 26 @@ -77,24 +93,3 @@ parent = f25-updates primary_arches = armhfp, i386, x86_64 alternate_arches = aarch64, ppc64, ppc64le, s390x -# Fedora 24 -[f24] -url = %(goldurl)s -path = 24 -release_status = stable -primary_arches = armhfp, i386, x86_64 -alternate_arches = aarch64, ppc64, ppc64le, s390x - -[f24-updates] -url = %(updatesurl)s -path = 24 -parent = f24 -primary_arches = armhfp, i386, x86_64 -alternate_arches = aarch64, ppc64, ppc64le, s390x - -[f24-updates-testing] -url = %(updatesurl)s -path = testing/24 -parent = f24-updates -primary_arches = armhfp, i386, x86_64 -alternate_arches = aarch64, ppc64, ppc64le, s390x diff --git a/docs/source/devguide.rst b/docs/source/devguide.rst index 2a31d9b..e214e11 100644 --- a/docs/source/devguide.rst +++ b/docs/source/devguide.rst @@ -11,23 +11,18 @@ General information Libtaskotron is written mostly in `Python `_. -Source code ------------ - -The source code for libtaskotron is available at: -https://pagure.io/taskotron/libtaskotron - -If you submit patches, please use the process of :ref:`submitting-code`. .. _taskotron-bugs: -Bugs, issues and tasks ----------------------- +Source code, bugs, tasks +------------------------ + +The source code for libtaskotron is available at: +https://pagure.io/taskotron/libtaskotron -We use `Phabricator `_ to track issues and facilitate -code reviews for several projects related to libtaskotron and Fedora QA. +This project is also used to track bugs and tasks. -Our phabricator instance can be found at https://phab.qa.fedoraproject.org/ +If you submit patches, please use the process of :ref:`submitting-code`. .. _running-tests: @@ -36,7 +31,7 @@ Running unit and functional tests --------------------------------- We place a high value on having decent test coverage for the libtaskotron code. -In general, tests are written using `pytest ` and are broken +In general, tests are written using `pytest `_ and are broken up into two types: * **Unit Tests** test the core logic of code. They do not touch the filesystem @@ -47,13 +42,9 @@ up into two types: tests are often much slower than unit tests but they offer coverage which is not present in the unit tests. -To run the unit tests:: - - py.test testing/ +To run the unit tests, execute:: -To run the functional and unit tests:: - - py.test -F testing/ + py.test Continuous integration @@ -82,52 +73,6 @@ In the development environment, it's also useful to have taskotron-generated files automatically cleaned up, so that they don't occupy disk space in vain. There is a tmpfiles.d template prepared for you, look into ``conf/tmpfiles.d``. -Support tools -------------- - -There are several tools that, while not required, make the process of developing -for libtaskotron significantly easier. - - -.. _installing-arcanist: - -Arcanist -^^^^^^^^ - -`Arcanist `_ -is a command line interface to Phabricator which can be used to submit code -reviews, download/apply code under review among other useful functions. - -As Arcanist is an interface to Phabricator, we **strongly recommend** that you -install it using our packages instead of from the upstream git repos (as -described in upstream documentation). That way, there is no question that the -Arcanist version used locally is compatible with our Phabricator instance. - -To add our dnf repository containing Phabricator related packages, run:: - - sudo curl https://repos.fedorapeople.org/repos/tflink/phabricator/fedora-phabricator.repo \ - -o /etc/yum.repos.d/fedora-phabricator.repo - -Once the repository has been configured, install Arcanist using:: - - sudo dnf -y install arcanist - -Arcanist is written in PHP and installing it will pull in several PHP packages -as dependencies. - - -.. _installing-gitflow: - -gitflow -^^^^^^^ - -The `gitflow plugin for git `_ is another -useful, but not required tool that is available in the Fedora repositories. - -To install the gitflow plugin, run:: - - sudo dnf -y install gitflow - .. _submitting-code: @@ -138,9 +83,8 @@ Libtaskotron follows the `gitflow `_ -is recommended as it makes the process significantly easier. See -:ref:`installing-gitflow` for instructions on installing gitflow on Fedora. +If you want to use the `gitflow plugin for git `_ +to make this process more user-friendly, simply install the ``gitflow`` package. Start a new feature @@ -148,9 +92,13 @@ Start a new feature To start work on a new feature, use:: - git flow feature start TXXX-short-description + git checkout -b feature/XXX-short-description develop -Where ``TXXX`` is the issue number in Phabricator and ``short-description`` is a +or if you want to use gitflow, use:: + + git flow feature start XXX-short-description + +where ``XXX`` is the issue number in Pagure and ``short-description`` is a short, human understandable description of the change contained in this branch. In general, short reviews are better than long reviews. If you can, please break @@ -160,36 +108,10 @@ up features into chunks that are smaller and easier to manage. Submitting code for review ^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. note:: - - Make sure to run all unit and functional tests before submitting code for - review. Any code that causes test failure will receive requests to fix the - offending code or will be rejected. See :ref:`running-tests` for information - on running unit and functional tests. - -Code reviews are done through Phabricator, not pull requests. While it is possible -to submit code for review through the web interface, :ref:`installing-arcanist` -is recommended. - -You do not need to push code anywhere public before submitting a review. Unless -there is a good reason to do so (and there are very few), pushing a feature -branch to origin is frowned on as it makes repository maintenance more difficult -for no real benefit. - -To submit code for review, make sure that your code has been updated with respect -to ``origin/develop`` and run the following from your checked-out feature branch:: - - arc diff develop - -The first time that you use Arcanist, it will ask for an API key which can be -retrieved from a link contained in that prompt. - -Arcanist will create a new code review on our Phabricator instance and prompt -you for information about the testing which has been done, a description of the -code under review and people to whom the review should be assigned. If you're -not clear on who should review your code, leave the ``reviewers`` section blank -and someone will either review your code or assign the review task to someone -who will. +Make sure to run all unit and functional tests before submitting code for +review. Any code that causes test failure will receive requests to fix the +offending code or will be rejected. See :ref:`running-tests` for information +on running unit and functional tests. Updating code reviews @@ -199,13 +121,6 @@ There will often be requests for changes to the code submitted for review. Once the requested changes have been made in your feature branch, commit them and make sure that your branch is still up to date with respect to ``origin/develop``. -To update the existing review, use:: - - arc diff develop --update DXXX - -Where ``DXXX`` is the Differential ID assigned to your review when it was -originally created. - Pushing code ^^^^^^^^^^^^ @@ -226,46 +141,27 @@ Make sure that your local ``develop`` branch is up-to-date with ``origin/develop before starting the merge process, else messy commits and merges may ensue. Once ``develop`` is up-to-date, the basic workflow to use is:: - git checkout feature/TXXX-some-feature + git checkout feature/XXX-some-feature git rebase develop To merge the code into develop, use one of two commands. If the feature can be reasonably expressed in one commit (most features), use:: - git flow feature finish --squash TXXX-some-feature + git flow feature finish --squash XXX-some-feature Else, if the Feature is larger and should cover multiple commits (less common), use:: - git flow feature finish TXXX-some-feature + git flow feature finish XXX-some-feature -After merging the code, please inspect log messages in case they need to be -shortened (Phabricator likes to make long commit messages). Groups of commits -should at least have a short description of their content and a link to the -revision in differential. Once the feature is ready, push to origin:: +After merging the code, please inspect git commit description and make it +prettier if needed. Groups of commits should at least have a short description +of their content and a link to the issue in Pagure. Once the feature is ready, +push to origin:: git push origin develop - -Reviewing code --------------- - -To review code, use `Phabricator's web interface `_ -to submit comments, request changes or accept reviews. - -If you want to look at the code under review locally to run tests or test -suggestions prior to posting them, use Arcanist to apply the review code. - -Make sure that your local repo is at the same base revision as the code under -review (usually origin/develop) and run the following command:: - - arc patch DXXX - -Where ``DXXX`` is the review id that you're interested in. Arcanist will grab the -code under review and apply it to a local branch named ``arcpatch-DXXX``. You can -then look at the code locally or make modifications. - Writing directives ================== @@ -308,11 +204,9 @@ extension can easily render into HTML. Sphinx has several built-in info fields which should be used to document function/method arguments and return data. - The following is an excerpt from `the Sphinx documentation `_ - Inside Python object description directives, reST field lists with these fields are recognized and formatted nicely: diff --git a/docs/source/formula.rst b/docs/source/formula.rst index 7a13476..7a270a9 100644 --- a/docs/source/formula.rst +++ b/docs/source/formula.rst @@ -12,8 +12,8 @@ The documentation here is a description of how things are implemented but are a little light on the practical creation of tasks. :doc:`writingtasks` and some existing tasks are also good references: -* `rpmlint `_ -* `examplebodhi `_ +* `rpmlint `_ +* `task examples `_ Task description ================ @@ -73,8 +73,8 @@ Dependencies ------------ A task may also require the presence of other code to support execution. Those -dependencies are specified as part of the environment description. Anything that -``dnf install`` supports as an argument on the command line is supported. +dependencies are specified as part of the environment description. It is +recommended to only use package names or file paths. .. code-block:: yaml @@ -82,6 +82,13 @@ dependencies are specified as part of the environment description. Anything that rpm: - python-solv - python-librepo + - /usr/bin/xz + +.. note:: + + You might also use advanced syntax like package version comparisons or group + names (as long as it's supported by ``dnf install``), but such tasks might + not work properly when executed under a non-root user. .. note:: diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 908f98f..c9eb2f4 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -103,7 +103,7 @@ task-rpmlint ^^^^^^^^^^^^ A very simple task from which you can learn all the basics. See its -`source code `_. +`source code `_. Run the task like this (replace the example NVR with the build you want to test): @@ -120,11 +120,10 @@ Some more real-world tasks include: * `task-abicheck `_ * `task-python-versions `_ -* `task-rpmgrill `_ +* `task-rpmgrill `_ You can also check our -`example tasks `_ +`task examples `_ (warning, some of them might be out of date) or all the rest of the tasks on -`bitbucket `_ and `pagure `_ (projects starting with *task-*). diff --git a/docs/source/runningtasks.rst b/docs/source/runningtasks.rst index 92f7ba6..715cf68 100644 --- a/docs/source/runningtasks.rst +++ b/docs/source/runningtasks.rst @@ -46,7 +46,7 @@ Running tasks Refer to the :ref:`quick start ` guide for the basics. -Using `task-rpmlint `_ as an +Using `task-rpmlint `_ as an example, the task repository contains the following important files:: task-rpmlint/ diff --git a/docs/source/writingtasks.rst b/docs/source/writingtasks.rst index 434b723..2cd70ca 100644 --- a/docs/source/writingtasks.rst +++ b/docs/source/writingtasks.rst @@ -15,12 +15,12 @@ aren't running from git. Also, see :doc:`quickstart` if you haven't already. Examples ======== -examplebodhi ------------- +bodhi example +------------- -`examplebodhi task git repository `_ +`bodhi example files `_ -The examplebodhi task is a trivial example which takes an update and determines +The bodhi example task is a trivial example which takes an update and determines a PASS/FAIL results in a pseudo-random fashion. This task wasn't designed to be used in production but is useful as a starting point and for development purposes. @@ -28,7 +28,7 @@ purposes. rpmlint ------- -`rpmlint task git repository `_ +`rpmlint task git repository `_ rpmlint is the simplest of the production tasks. It takes a koji build and runs rpmlint on it, reporting results as result-YAML. diff --git a/libtaskotron.spec b/libtaskotron.spec index 0674414..1cd6bb8 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -1,6 +1,6 @@ Name: libtaskotron # NOTE: if you update version, *make sure* to also update `libtaskotron/__init__.py` -Version: 0.5.0 +Version: 0.4.99.1 Release: 1%{?dist} Summary: Taskotron Support Library @@ -62,14 +62,22 @@ Summary: Fedora specific module for libtaskotron Requires: createrepo Requires: dnf >= 0.6.4 +%if 0%{?fedora} >= 27 +Requires: python2-koji >= 1.10.0 +%else Requires: koji >= 1.10.0 +%endif Requires: libtaskotron-core = %{version}-%{release} Requires: mash Requires: python-fedora >= 0.8.0 Requires: python-hawkey >= 0.4.13-1 Requires: python-munch >= 2.0.2 Requires: rpm-python +%if 0%{?fedora} >= 27 +BuildRequires: python2-koji >= 1.10.0 +%else BuildRequires: koji >= 1.10.0 +%endif BuildRequires: mash BuildRequires: python-fedora >= 0.8.0 BuildRequires: python-hawkey >= 0.4.13-1 @@ -168,9 +176,27 @@ install -d %{buildroot}/%{_sharedstatedir}/taskotron/images %{python2_sitelib}/libtaskotron/ext/disposable/* %changelog -* Wed May 10 2017 Martin Krizek - 0.5.0-1 +* Tue Aug 29 2017 Martin Krizek - 0.4.99.1-1 - Support Ansible style tasks (D1195) +* Fri Jul 14 2017 Kamil Páral - 0.4.24-1 +- do not use --cacheonly for dnf operations + +* Wed Jul 12 2017 Kamil Páral - 0.4.23-1 +- fix python2-koji dep on F27+ +- fix broken test suite + +* Wed Jul 12 2017 Kamil Páral - 0.4.22-1 +- mark Fedora 26 as stable in yumrepoinfo +- remove check for installed packages because it was problematic + +* Fri Jun 30 2017 Kamil Páral - 0.4.21-1 +- documentation improvements +- DNF_REPO item type removed +- default task artifact now points to artifacts root dir instead of task log +- fix rpm deps handling via dnf on Fedora 26 (but only support package names + and filepaths as deps in task formulas) + * Tue Apr 4 2017 Martin Krizek - 0.4.20-1 - Add module_build item type (D1184) - taskformula: replace vars in dictionary keys (D1176) @@ -179,9 +205,28 @@ install -d %{buildroot}/%{_sharedstatedir}/taskotron/images - argparse: change --arch to be a single value instead of a string (D1171) - yumrepoinfo: specify all primary and alternate arches (D1172) +* Fri Mar 24 2017 Tim Flink - 0.4.19-4 +- bumping revision to test package-specific testing again + +* Fri Mar 17 2017 Tim Flink - 0.4.19-3 +- bumping revision to test package-specific testing again + +* Fri Mar 17 2017 Tim Flink - 0.4.19-2 +- bumping revision to test package-specific testing + +* Fri Mar 17 2017 Tim Flink - 0.4.19-1 +- updating yumrepoinfo for F26 +- improved support for secondary architectures + +* Thu Mar 16 2017 Tim Flink - 0.4.18-3 +- bumping revision to test package-specific testing + * Fri Feb 17 2017 Kamil Páral - 0.4.18-4 - require koji >= 1.10.0 because of T910 +* Fri Feb 10 2017 Fedora Release Engineering - 0.4.18-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild + * Fri Feb 10 2017 Kamil Páral - 0.4.18-3 - add python-pytest-cov builddep because the test suite now needs it, and python-rpmfluff because we're missing it diff --git a/libtaskotron/__init__.py b/libtaskotron/__init__.py index e9b2971..38c0587 100644 --- a/libtaskotron/__init__.py +++ b/libtaskotron/__init__.py @@ -4,4 +4,4 @@ # See the LICENSE file for more details on Licensing from __future__ import absolute_import -__version__ = '0.5.0' +__version__ = '0.4.99.1' diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index d3266d1..12dc9df 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -5,6 +5,18 @@ from __future__ import absolute_import +import os +import configparser +import pprint + +from libtaskotron.directives import BaseDirective +from libtaskotron import check +from libtaskotron import config +from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError +from libtaskotron.logger import log +from libtaskotron.ext.fedora import rpm_utils +import resultsdb_api + DOCUMENTATION = """ module: resultsdb_directive short_description: send task results to ResultsDB or check ResultYAML format @@ -96,21 +108,9 @@ summary will be printed out into the log, like this:: 'summary': 'RPMLINT PASSED for xchat-tcl-2.8.8-21.fc20.x86_64.rpm'}> """ -import os -import configparser - -from libtaskotron.directives import BaseDirective - -from libtaskotron import check -from libtaskotron import config -from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError -from libtaskotron.logger import log -from libtaskotron.ext.fedora import rpm_utils - -import resultsdb_api - directive_class = 'ResultsdbDirective' + class ResultsdbDirective(BaseDirective): def __init__(self, resultsdb = None): @@ -274,7 +274,7 @@ class ResultsdbDirective(BaseDirective): conf = config.get_config() if not conf.report_to_resultsdb: log.info("Reporting to ResultsDB is disabled. Once enabled, the " - "following would get reported:\n%s" % params['results']) + "following would get reported:\n%s", params['results']) return check.export_YAML(check_details) checkname = '%s.%s' % (arg_data['namespace'], arg_data['checkname']) @@ -311,6 +311,7 @@ class ResultsdbDirective(BaseDirective): item=detail.item, type=detail.report_type, **detail.keyvals) + log.debug('Result saved in ResultsDB:\n%s', pprint.pformat(result)) detail._internal['resultsdb_result_id'] = result['id'] except resultsdb_api.ResultsDBapiException, e: diff --git a/libtaskotron/ext/fedora/rpm_utils.py b/libtaskotron/ext/fedora/rpm_utils.py index 7a03504..971d33f 100644 --- a/libtaskotron/ext/fedora/rpm_utils.py +++ b/libtaskotron/ext/fedora/rpm_utils.py @@ -108,10 +108,6 @@ def cmpNEVR(nevr1, nevr2): def install(pkgs): '''Install packages from system repositories using DNF. Either root or sudo access required. - Note: This is first tested with ``--cacheonly`` to avoid downloading metadata and speed up the - process. If that command fails for whatever reason, we run it again, this time without - ``--cacheonly``. - :param pkgs: packages to be installed, e.g. ``['pidgin']``, or any other argument supported by ``dnf install`` command :type pkgs: list of str @@ -131,111 +127,21 @@ def install(pkgs): cmd = ['dnf', '--assumeyes', 'install'] cmd.extend(pkgs) - if dnf_cache_available(): - cmd.insert(1, '--cacheonly') - else: - log.warn('No DNF cache available, metadata will be need to be downloaded. If your DNF ' - "cache doesn't persist across task executions (e.g. on a disposable VM), " - 'consider creating the cache in your base system image to speed up execution.') - if not os_utils.is_root(): # we must have sudo at this point, don't test again needlessly cmd = ['sudo', '--non-interactive'] + cmd - while True: # we need to call the command twice, if the first run with --cacheonly fails - log.debug('Running: %s', ' '.join([pipes.quote(c) for c in cmd])) - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - if '--cacheonly' in cmd: # maybe just outdated cache - log.debug(u'✘ Package installation failed, DNF output was:\n%s', e.output.rstrip()) - log.debug("Trying again with forced metadata refresh...") - cmd = list(cmd) # make a copy to allow unit tests to intercept both calls easily - cmd.remove('--cacheonly') - cmd.append('--refresh') - continue - - log.error(u'✘ Package installation failed. We tried to install following packages:\n%s' - '\nDNF returned exit code %d and output:\n%s', pkglist, e.returncode, - e.output.rstrip()) - raise exc.TaskotronError("Unable to install packages: %s" % pkglist) - else: - log.debug(u'✔ Package installation completed successfully. DNF output was:\n%s', - output.rstrip()) - return - - # safeguard against infinite loop, we should never reach this code - log.critical("Infinite loop detected in 'install()'") - assert False, "Infinite loop detected in 'install()'" - return - - -def is_installed(pkgs): - '''Check if packages are installed using DNF. No elevated permissions needed. - - :param pkgs: packages to be checked whether they are installed, e.g. ``['pidgin']`` or any - other argument supported by ``dnf install`` command - :type pkgs: list of str - :return: True if all specified packages are installed, False otherwise - :rtype: bool - ''' - if not pkgs: - return True - - log.info('Checking installed state of %d packages...', len(pkgs)) - cmd = ['dnf', '--assumeno', '--disableplugin=noroot', 'install'] - cmd.extend(pkgs) - - if dnf_cache_available(): - cmd.insert(1, '--cacheonly') - else: - log.warn('No DNF cache available, metadata will be need to be downloaded. If your DNF ' - "cache doesn't persist across task executions (e.g. on a disposable VM), " - 'consider creating the cache in your base system image to speed up execution.') - - if not os_utils.is_root() and os_utils.has_sudo(): - cmd = ['sudo', '--non-interactive'] + cmd - log.debug('Running: %s', ' '.join([pipes.quote(c) for c in cmd])) try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - log.debug(u'✘ Some of specified packages are not installed. DNF returned exit code %d and ' - 'output:\n%s', e.returncode, e.output.rstrip()) - return False + log.error(u'✘ Package installation failed. We tried to install following packages:\n%s' + '\nDNF returned exit code %d and output:\n%s', pkglist, e.returncode, + e.output.rstrip()) + raise exc.TaskotronError("Unable to install packages: %s" % pkglist) else: - log.debug(u'✔ All specified packages are installed') - return True - - -def dnf_cache_available(): - '''Determine whether DNF cache is available. - - If it is, we can run read-only actions solely from the cache, and try to use it even for - ``install`` action (provided the repos haven't changed much). This way we avoid contacting - DNF repos and downloading new metadata if they changed. - - This tests the system cache (if running as root or with sudo access) or the user cache - (otherwise). Since DNF has no direct way of testing this, we simply run an action that should - always pass (``dnf repolist``) with ``--cacheonly`` option and look at the exit code. - - :return: bool whether DNF cache is currently available and can be used (system or user cache, - depending on your current admin privileges) - ''' - cmd = ['dnf', '--cacheonly', 'repolist'] - - if not os_utils.is_root() and os_utils.has_sudo(): - cmd = ['sudo', '--non-interactive'] + cmd - - log.debug('Deciding whether DNF cache is available. Running: %s', ' '.join(cmd)) - try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - log.debug(u'✘ DNF cache is not available. Received exit code %d and output:\n%s', - e.returncode, e.output.rstrip()) - return False - else: - log.debug(u'✔ DNF cache is available.') - return True + log.debug(u'✔ Package installation completed successfully. DNF output was:\n%s', + output.rstrip()) + return def get_dist_tag(rpmstr): diff --git a/libtaskotron/taskotron_result.py b/libtaskotron/taskotron_result.py index 275a1b5..e7ba63d 100644 --- a/libtaskotron/taskotron_result.py +++ b/libtaskotron/taskotron_result.py @@ -21,7 +21,8 @@ def get_argparser(): parser.add_argument("-k", "--keyval", action="append", default=[], metavar='KEY=VALUE', help="all key-value pairs in this dictionary are stored " "in ResultsDB as 'extra data'") - parser.add_argument("-c", "--checkname", help="name of the check", default=None) + parser.add_argument("-c", "--checkname", + help="name of the check (don't include namespace here)", default=None) parser.add_argument("-a", "--artifact", help="file or directory placed in the artifacts dir", default=None) diff --git a/testing/test_rpm_utils.py b/testing/test_rpm_utils.py index 67731c9..edbaee3 100644 --- a/testing/test_rpm_utils.py +++ b/testing/test_rpm_utils.py @@ -8,9 +8,8 @@ import pytest import subprocess -from libtaskotron.ext.fedora.rpm_utils import (cmpNEVR, dnf_cache_available, get_dist_tag, install, - is_installed, rpmformat) -from libtaskotron.ext.fedora import rpm_utils +from libtaskotron.ext.fedora.rpm_utils import (cmpNEVR, get_dist_tag, install, + rpmformat) from libtaskotron import exceptions as exc from libtaskotron import os_utils @@ -142,8 +141,6 @@ class TestInstall(object): monkeypatch.setattr(os_utils, 'is_root', self.mock_is_root) self.mock_has_sudo = mock.Mock(return_value=True) monkeypatch.setattr(os_utils, 'has_sudo', self.mock_has_sudo) - self.mock_dnf_cache = mock.Mock(return_value=True) - monkeypatch.setattr(rpm_utils, 'dnf_cache_available', self.mock_dnf_cache) self.mock_check_output = mock.Mock(return_value='') monkeypatch.setattr(subprocess, 'check_output', self.mock_check_output) @@ -189,144 +186,6 @@ class TestInstall(object): call_args = self.mock_check_output.call_args[0][0] assert all([pkg in call_args for pkg in pkgs]) - def test_run_once(self): - '''The command should be executed only once if it passes''' - install(['foo']) - assert self.mock_check_output.call_count == 1 - - def test_run_once_fail(self): - '''If there's no cache and install fails, it should not try again''' - self.mock_dnf_cache.return_value = False - self.mock_check_output.side_effect = self.err - - with pytest.raises(exc.TaskotronError) as excinfo: - install(['foo']) - - assert self.mock_check_output.call_count == 1 - # direct comparison here, because isinstance() would also accept subclasses we throw for - # different issues (missing permissions) - assert type(excinfo.value) is exc.TaskotronError - - def test_run_twice_on_cache_fail(self): - '''If cache is used and install fails, it should run again without cache''' - self.mock_check_output.side_effect = self.err - with pytest.raises(exc.TaskotronError): - install(['foo']) - - assert self.mock_check_output.call_count == 2 - assert '--cacheonly' in self.mock_check_output.call_args_list[0][0][0] - assert '--cacheonly' not in self.mock_check_output.call_args_list[1][0][0] - - def test_run_twice_second_ok(self): - '''If installation is tried twice and second one succeeds, the whole operation should - succeed.''' - self.mock_check_output.side_effect = (self.err, mock.DEFAULT) - install(['foo']) - assert self.mock_check_output.call_count == 2 - - -@pytest.mark.usefixtures('setup') -class TestIsInstalled(object): - '''Test rpm_utils.is_installed()''' - - @pytest.fixture - def setup(self, monkeypatch): - self.mock_is_root = mock.Mock(return_value=True) - monkeypatch.setattr(os_utils, 'is_root', self.mock_is_root) - self.mock_has_sudo = mock.Mock(return_value=True) - monkeypatch.setattr(os_utils, 'has_sudo', self.mock_has_sudo) - self.mock_dnf_cache = mock.Mock(return_value=True) - monkeypatch.setattr(rpm_utils, 'dnf_cache_available', self.mock_dnf_cache) - - self.mock_check_output = mock.Mock(return_value='') - monkeypatch.setattr(subprocess, 'check_output', self.mock_check_output) - self.err = subprocess.CalledProcessError(1, 'cmd', output='') - - def test_installed(self): - assert is_installed(['foo']) is True - - def test_not_installed(self): - self.mock_check_output.side_effect = self.err - assert is_installed(['foo']) is False - - def test_no_permissions_is_ok(self): - self.mock_is_root.return_value = False - self.mock_has_sudo.return_value = False - assert is_installed(['foo']) is True - - def test_add_sudo(self): - self.mock_is_root.return_value = False - is_installed(['foo']) - assert self.mock_check_output.call_args[0][0].index('sudo') == 0 - - def test_dont_add_sudo(self): - is_installed(['foo']) - assert 'sudo' not in self.mock_check_output.call_args[0][0] - - def test_no_pkgs(self): - assert is_installed([]) is True - assert self.mock_check_output.call_count == 0 - - def test_special_args(self): - '''Make sure args like 'rpmlint > 1.0' are passed in correctly''' - pkgs = ['foo', 'bar >= 1.0', '@group'] - is_installed(pkgs) - - call_args = self.mock_check_output.call_args[0][0] - assert all([pkg in call_args for pkg in pkgs]) - - def test_cache_exists(self): - is_installed(['foo']) - assert '--cacheonly' in self.mock_check_output.call_args_list[0][0][0] - - def test_cache_doesnt_exist(self): - self.mock_dnf_cache.return_value = False - is_installed(['foo']) - assert '--cacheonly' not in self.mock_check_output.call_args_list[0][0][0] - - -@pytest.mark.usefixtures('setup') -class TestDnfCacheAvailable(object): - '''Test rpm_utils.dnf_cache_available()''' - - @pytest.fixture - def setup(self, monkeypatch): - self.mock_is_root = mock.Mock(return_value=True) - monkeypatch.setattr(os_utils, 'is_root', self.mock_is_root) - self.mock_has_sudo = mock.Mock(return_value=True) - monkeypatch.setattr(os_utils, 'has_sudo', self.mock_has_sudo) - - self.mock_check_output = mock.Mock(return_value='') - monkeypatch.setattr(subprocess, 'check_output', self.mock_check_output) - self.err = subprocess.CalledProcessError(1, 'cmd', output='') - - def test_cache_exists(self): - assert dnf_cache_available() is True - - def test_cache_exists_user(self): - self.mock_is_root.return_value = False - self.mock_has_sudo.return_value = False - assert dnf_cache_available() is True - - def test_cache_doesnt_exist(self): - self.mock_check_output.side_effect = self.err - assert dnf_cache_available() is False - - def test_cache_doesnt_exist_user(self): - self.mock_is_root.return_value = False - self.mock_has_sudo.return_value = False - self.mock_check_output.side_effect = self.err - assert dnf_cache_available() is False - - def test_add_sudo(self): - self.mock_is_root.return_value = False - dnf_cache_available() - assert self.mock_check_output.call_args[0][0].index('sudo') == 0 - - def test_dont_add_sudo(self): - dnf_cache_available() - assert 'sudo' not in self.mock_check_output.call_args[0][0] - class TestGetDistTag(object): def test_nvrs(self): diff --git a/tox.ini b/tox.ini index 6f5aa9e..05f2c97 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,10 @@ # This is a common file where different test suites/linters can be configured. -# Phabricator uses this file when running `arc unit` or `arc lint`. [flake8] +# If you want to ignore a specific source code line, use '# noqa' comment. If +# you want to ignore the whole file, add '# flake8: noqa' comment. Read more +# documentation about flake8 at: +# https://flake8.readthedocs.org/ max-line-length=99 [pep8] From 1cf5e6265a5db2506c1ee6567efc71c9c746ed95 Mon Sep 17 00:00:00 2001 From: Lili Nie Date: Aug 29 2017 13:05:07 +0000 Subject: [PATCH 3/30] don't fail when ssh key is not provided Fix a bug that caused connection to fail when --ssh-privkey option was not used. Instead, the default ssh keys should be used. Merges #394. --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 6dfd86a..7e05561 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -115,7 +115,8 @@ class Executor(object): ] if self.run_remotely: - cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) + if self.arg_data['ssh_privkey']: + cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) else: cmd.extend(['--ask-become-pass', '--connection=local', '-e', 'local=true']) From 2b05db04ce86b7661d5805c7fdf20618874b5f7c Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Aug 29 2017 19:11:49 +0000 Subject: [PATCH 4/30] WIP: moar STI compatibility --- diff --git a/library/koji.py b/library/koji.py new file mode 100644 index 0000000..1d1b472 --- /dev/null +++ b/library/koji.py @@ -0,0 +1,171 @@ +#!/usr/bin/python +# Make coding more python3-ish +from __future__ import (absolute_import, division) +__metaclass__ = type + +import os +import ast +import json + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.urls import open_url + +#from . import TaskotronError +try: + import libtaskotron.exceptions as exc + from libtaskotron.ext.fedora.koji_utils import KojiClient + from libtaskotron.ext.fedora import rpm_utils +except ImportError: + libtaskotron_found = False +else: + libtaskotron_found = True + +# these will need to be handled better but this is just a PoC +WORKDIR = os.path.abspath('./taskotron-workdir') + +def main(): + mod = AnsibleModule( + argument_spec=dict( + action=dict(required=True), + arch=dict(required=False, default=['noarch']), + workdir=dict(required=False, default="/tmp/firstmod"), + arch_exclude=dict(required=False), + build_log=dict(required=False, default=False), + debuginfo=dict(required=False, default=False), + koji_build=dict(required=False), + koji_tag=dict(required=False), + src=dict(required=False, default=False), + target_dir=dict(required=False, default='.') + ) + ) + + # TODO: check args for completeness + if not libtaskotron_found: + mod.fail_json(msg="The libtaskotron python module is required") + + try: + kojidirective = KojiDirective() + data = kojidirective.process(mod) + except exc.TaskotronError, e: + mod.fail_json(msg=e) + + subjects = ' '.join(data['downloaded_rpms']) + + mod.exit_json(msg="worky!", changed=True, subjects=subjects) + +class KojiDirective(object): + + def __init__(self, koji_session=None): + super(KojiDirective, self).__init__() + if koji_session is None: + self.koji = KojiClient() + else: + self.koji = koji_session + + def process(self, mod): + # process params + valid_actions = ['download', 'download_tag', 'download_latest_stable'] + action = mod.params['action'] + if action not in valid_actions: + raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' + 'directive' % action) + + # use register to save/return information + if 'target_dir' not in mod.params: + target_dir = WORKDIR + else: + target_dir = mod.params['target_dir'] + + if 'arch' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'arch' as an argument. Detected " + "arguments: %s" % detected_args) + + # this is supposedly safe enough to use on raw input but should be double checked + # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python + arches = ast.literal_eval(mod.params['arch']) + if not isinstance(arches, list): + raise exc.TaskotronError("arches must be a list") + + if arches and ('all' not in arches) and ('noarch' not in arches): + arches.append('noarch') + + arch_exclude_string = mod.params.get('arch_exclude', None) + if arch_exclude_string is None: + arch_exclude = [] + else: + arch_exclude = ast.literal_eval(mod.params['arch_exclude']) + + debuginfo = mod.params.get('debuginfo', False) + src = mod.params.get('src', False) + build_log = mod.params.get('build_log', False) + + if not isinstance(arch_exclude, list): + print("arch_exclude: {}".format(type(arch_exclude))) + raise Exception("arch_exclude must be a list") + # download files + output_data = {} + + if action == 'download': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download' " + "action. Detected arguments: %s" % detected_args) + + nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_tag': + if 'koji_tag' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_tag' for the 'download_tag' " + "action. Detected arguments: %s" % detected_args) + + koji_tag = mod.params['koji_tag'] + + output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( + koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_latest_stable': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download_latest_stable' " + "action. Detected arguments: %s" % detected_args) + + name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') + disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) + # we need to do 'fc22' -> 'f22' conversion + tag = disttag.replace('c', '') + + # first we need to check updates tag and if that fails, the latest + # stable nvr is in the base repo + tags = ['%s-updates' % tag, tag] + nvr = self.koji.latest_by_tag(tags, name) + + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arch_exclude=arch_exclude, + arches=arches, debuginfo=debuginfo, src=src) + + # download build.log if requested + if build_log: + if action in ('download', 'download_latest_stable'): + ret_log = self.koji.get_build_log( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude) + output_data['downloaded_logs'] = ret_log['ok'] + output_data['log_errors'] = ret_log['error'] + else: + #log.warn("Downloading build logs is not supported for action '%s', ignoring.", + # action) + print("Downloading build logs is not supported for action '%s', ignoring." % action) + + return output_data + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 7e05561..bdcd0bd 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -108,7 +108,7 @@ class Executor(object): 'ansible-playbook', 'runner.yml', '--inventory=%s,' % ipaddr, '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], - '-e', 'subjects=%s' % self.arg_data['item'], + '-e', 'taskotron_item=%s' % self.arg_data['item'], '-e', 'taskdir=%s' % self.arg_data['taskdir'], '-e', 'client_taskdir=%s' % config.get_config().client_taskdir, '-e', 'sti_inventory=%s' % sti_inventory, diff --git a/runner.yml b/runner.yml index 2b6ee44..e3accec 100644 --- a/runner.yml +++ b/runner.yml @@ -3,17 +3,17 @@ vars: local: false tasks: - - name: Clean taskdir + - name: Clean client taskdir file: path: "{{ client_taskdir }}" state: absent - - name: Create taskdir + - name: Create client taskdir file: path: "{{ client_taskdir }}" state: directory - - name: Upload taskdir + - name: Upload taskdir to client taskdir synchronize: src: "{{ taskdir }}/" dest: "{{ client_taskdir }}" @@ -33,17 +33,25 @@ - python2-dnf when: not local - #- name: Download subjects (rpms atm) - # our_custom_module: "{{}}" # probably wrapped koji - # export: subjects + # item is specified on command line (e.g. koji_build), this module should download + # artifacsts related to the item and return subject(s) (path strings) for STI tests.yml + - name: Download subjects (only rpms atm) + koji: + action: "download" + koji_build: "{{ taskotron_item }}" + arch: ['x86_64'] #FIXME + target_dir: "{{ client_taskdir }}" #FIXME? + register: koji_output + + - debug: var=koji_output - name: Run the task become: yes become_user: root # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ subjects }}" + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" environment: - TEST_SUBJECTS: "{{ subjects }}" + TEST_SUBJECTS: "{{ koji_output['subjects'] }}" TEST_ARTIFACTS: "{{ artifacts }}" ignore_errors: yes register: output From afa906b532285b62da90855376e9c605d456672b Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Aug 30 2017 14:54:17 +0000 Subject: [PATCH 5/30] WIP: generating results for resultsdb --- diff --git a/libtaskotron/main.py b/libtaskotron/main.py index 0906cf1..4cbb2bc 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -187,8 +187,9 @@ def main(): executor.execute() # report results - # test_rc = read_test_rc() - # resultsdb.report(subject, rc, hurr, durr) + with open(os.path.join(arg_data['artifactsdir'], 'results.yml')) as results_file: + rusults_yaml = results_file.read() + log.info('WIP, this will be reported to resultsDB:\n%s\n' % rusults_yaml) # finalize log.info('Execution finished at: %s. Task artifacts were saved in: %s', diff --git a/runner.yml b/runner.yml index e3accec..705a3cf 100644 --- a/runner.yml +++ b/runner.yml @@ -34,7 +34,7 @@ when: not local # item is specified on command line (e.g. koji_build), this module should download - # artifacsts related to the item and return subject(s) (path strings) for STI tests.yml + # artifacts related to the item and return subject(s) (path strings) for STI tests.yml - name: Download subjects (only rpms atm) koji: action: "download" @@ -67,13 +67,23 @@ - name: RC of test shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + - name: Set outcome to PASSED if RC == 0 + set_fact: outcome="PASSED" + when: output.rc == 0 + + - name: Set outcome to PASSED if RC != 0 + set_fact: outcome="FAILED" + when: output.rc != 0 + + - name: generate resultsdb result file + # FIXME: type of result + shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" + args: + creates: "{{ artifacts }}/results.yml" + - name: Collect logs synchronize: mode: pull src: "{{ artifacts }}/*" dest: "{{ artifacts }}" when: not local - - #- name: generate resultsdb result file - # resultsdb: - # rc: {{ ... }} \ No newline at end of file From 49a52c6f80cfbf9fc19dcb2c925c5bfca577e9ab Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 06 2017 12:30:15 +0000 Subject: [PATCH 6/30] ansiblizing cleanup --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index bdcd0bd..4ef0494 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -104,8 +104,12 @@ class Executor(object): else: sti_inventory = "/usr/share/ansible/inventory" + #TODO add logic to decide what playbook to use + #runner_playbook = 'runner_si.yml' + runner_playbook = 'runner_generic.yml' + cmd = [ - 'ansible-playbook', 'runner.yml', + 'ansible-playbook', runner_playbook, '--inventory=%s,' % ipaddr, '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'taskotron_item=%s' % self.arg_data['item'], diff --git a/libtaskotron/main.py b/libtaskotron/main.py index 4cbb2bc..73b5979 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -187,7 +187,7 @@ def main(): executor.execute() # report results - with open(os.path.join(arg_data['artifactsdir'], 'results.yml')) as results_file: + with open(os.path.join(arg_data['artifactsdir'], 'taskotron_results.yml')) as results_file: rusults_yaml = results_file.read() log.info('WIP, this will be reported to resultsDB:\n%s\n' % rusults_yaml) diff --git a/runner.yml b/runner.yml deleted file mode 100644 index 705a3cf..0000000 --- a/runner.yml +++ /dev/null @@ -1,89 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - # item is specified on command line (e.g. koji_build), this module should download - # artifacts related to the item and return subject(s) (path strings) for STI tests.yml - - name: Download subjects (only rpms atm) - koji: - action: "download" - koji_build: "{{ taskotron_item }}" - arch: ['x86_64'] #FIXME - target_dir: "{{ client_taskdir }}" #FIXME? - register: koji_output - - - debug: var=koji_output - - - name: Run the task - become: yes - become_user: root - # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" - environment: - TEST_SUBJECTS: "{{ koji_output['subjects'] }}" - TEST_ARTIFACTS: "{{ artifacts }}" - ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" - - - name: RC of test - shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" - - - name: Set outcome to PASSED if RC == 0 - set_fact: outcome="PASSED" - when: output.rc == 0 - - - name: Set outcome to PASSED if RC != 0 - set_fact: outcome="FAILED" - when: output.rc != 0 - - - name: generate resultsdb result file - # FIXME: type of result - shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" - args: - creates: "{{ artifacts }}/results.yml" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local diff --git a/runner_generic.yml b/runner_generic.yml new file mode 100644 index 0000000..a7312bd --- /dev/null +++ b/runner_generic.yml @@ -0,0 +1,58 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + - name: Run the task + become: yes + become_user: root + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} + environment: + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local diff --git a/runner_si.yml b/runner_si.yml new file mode 100644 index 0000000..f427994 --- /dev/null +++ b/runner_si.yml @@ -0,0 +1,89 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + item is specified on command line (e.g. koji_build), this module should download + artifacts related to the item and return subject(s) (path strings) for STI tests.yml + - name: Download subjects (only rpms atm) + koji: + action: "download" + koji_build: "{{ taskotron_item }}" + arch: ['x86_64'] #FIXME + target_dir: "{{ client_taskdir }}" #FIXME? + register: koji_output + + - debug: var=koji_output + + - name: Run the task + become: yes + become_user: root + # FIXME add context tags + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" + environment: + TEST_SUBJECTS: "{{ koji_output['subjects'] }}" + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: RC of test + shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + + - name: Set outcome to PASSED if RC == 0 + set_fact: outcome="PASSED" + when: output.rc == 0 + + - name: Set outcome to PASSED if RC != 0 + set_fact: outcome="FAILED" + when: output.rc != 0 + + - name: generate resultsdb result file + # FIXME: type of result + shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" + args: + creates: "{{ artifacts }}/results.yml" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local From 7e08f16fd951c819a98452be95a8377a8a44cc9f Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 07 2017 09:27:04 +0000 Subject: [PATCH 7/30] move all ansible files into a subdirectory --- diff --git a/ansible/library/koji.py b/ansible/library/koji.py new file mode 100644 index 0000000..1d1b472 --- /dev/null +++ b/ansible/library/koji.py @@ -0,0 +1,171 @@ +#!/usr/bin/python +# Make coding more python3-ish +from __future__ import (absolute_import, division) +__metaclass__ = type + +import os +import ast +import json + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.urls import open_url + +#from . import TaskotronError +try: + import libtaskotron.exceptions as exc + from libtaskotron.ext.fedora.koji_utils import KojiClient + from libtaskotron.ext.fedora import rpm_utils +except ImportError: + libtaskotron_found = False +else: + libtaskotron_found = True + +# these will need to be handled better but this is just a PoC +WORKDIR = os.path.abspath('./taskotron-workdir') + +def main(): + mod = AnsibleModule( + argument_spec=dict( + action=dict(required=True), + arch=dict(required=False, default=['noarch']), + workdir=dict(required=False, default="/tmp/firstmod"), + arch_exclude=dict(required=False), + build_log=dict(required=False, default=False), + debuginfo=dict(required=False, default=False), + koji_build=dict(required=False), + koji_tag=dict(required=False), + src=dict(required=False, default=False), + target_dir=dict(required=False, default='.') + ) + ) + + # TODO: check args for completeness + if not libtaskotron_found: + mod.fail_json(msg="The libtaskotron python module is required") + + try: + kojidirective = KojiDirective() + data = kojidirective.process(mod) + except exc.TaskotronError, e: + mod.fail_json(msg=e) + + subjects = ' '.join(data['downloaded_rpms']) + + mod.exit_json(msg="worky!", changed=True, subjects=subjects) + +class KojiDirective(object): + + def __init__(self, koji_session=None): + super(KojiDirective, self).__init__() + if koji_session is None: + self.koji = KojiClient() + else: + self.koji = koji_session + + def process(self, mod): + # process params + valid_actions = ['download', 'download_tag', 'download_latest_stable'] + action = mod.params['action'] + if action not in valid_actions: + raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' + 'directive' % action) + + # use register to save/return information + if 'target_dir' not in mod.params: + target_dir = WORKDIR + else: + target_dir = mod.params['target_dir'] + + if 'arch' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'arch' as an argument. Detected " + "arguments: %s" % detected_args) + + # this is supposedly safe enough to use on raw input but should be double checked + # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python + arches = ast.literal_eval(mod.params['arch']) + if not isinstance(arches, list): + raise exc.TaskotronError("arches must be a list") + + if arches and ('all' not in arches) and ('noarch' not in arches): + arches.append('noarch') + + arch_exclude_string = mod.params.get('arch_exclude', None) + if arch_exclude_string is None: + arch_exclude = [] + else: + arch_exclude = ast.literal_eval(mod.params['arch_exclude']) + + debuginfo = mod.params.get('debuginfo', False) + src = mod.params.get('src', False) + build_log = mod.params.get('build_log', False) + + if not isinstance(arch_exclude, list): + print("arch_exclude: {}".format(type(arch_exclude))) + raise Exception("arch_exclude must be a list") + # download files + output_data = {} + + if action == 'download': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download' " + "action. Detected arguments: %s" % detected_args) + + nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_tag': + if 'koji_tag' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_tag' for the 'download_tag' " + "action. Detected arguments: %s" % detected_args) + + koji_tag = mod.params['koji_tag'] + + output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( + koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_latest_stable': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download_latest_stable' " + "action. Detected arguments: %s" % detected_args) + + name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') + disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) + # we need to do 'fc22' -> 'f22' conversion + tag = disttag.replace('c', '') + + # first we need to check updates tag and if that fails, the latest + # stable nvr is in the base repo + tags = ['%s-updates' % tag, tag] + nvr = self.koji.latest_by_tag(tags, name) + + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arch_exclude=arch_exclude, + arches=arches, debuginfo=debuginfo, src=src) + + # download build.log if requested + if build_log: + if action in ('download', 'download_latest_stable'): + ret_log = self.koji.get_build_log( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude) + output_data['downloaded_logs'] = ret_log['ok'] + output_data['log_errors'] = ret_log['error'] + else: + #log.warn("Downloading build logs is not supported for action '%s', ignoring.", + # action) + print("Downloading build logs is not supported for action '%s', ignoring." % action) + + return output_data + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ansible/runner_generic.yml b/ansible/runner_generic.yml new file mode 100644 index 0000000..a7312bd --- /dev/null +++ b/ansible/runner_generic.yml @@ -0,0 +1,58 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + - name: Run the task + become: yes + become_user: root + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} + environment: + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local diff --git a/ansible/runner_si.yml b/ansible/runner_si.yml new file mode 100644 index 0000000..f427994 --- /dev/null +++ b/ansible/runner_si.yml @@ -0,0 +1,89 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + item is specified on command line (e.g. koji_build), this module should download + artifacts related to the item and return subject(s) (path strings) for STI tests.yml + - name: Download subjects (only rpms atm) + koji: + action: "download" + koji_build: "{{ taskotron_item }}" + arch: ['x86_64'] #FIXME + target_dir: "{{ client_taskdir }}" #FIXME? + register: koji_output + + - debug: var=koji_output + + - name: Run the task + become: yes + become_user: root + # FIXME add context tags + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" + environment: + TEST_SUBJECTS: "{{ koji_output['subjects'] }}" + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: RC of test + shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + + - name: Set outcome to PASSED if RC == 0 + set_fact: outcome="PASSED" + when: output.rc == 0 + + - name: Set outcome to PASSED if RC != 0 + set_fact: outcome="FAILED" + when: output.rc != 0 + + - name: generate resultsdb result file + # FIXME: type of result + shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" + args: + creates: "{{ artifacts }}/results.yml" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local diff --git a/library/koji.py b/library/koji.py deleted file mode 100644 index 1d1b472..0000000 --- a/library/koji.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/python -# Make coding more python3-ish -from __future__ import (absolute_import, division) -__metaclass__ = type - -import os -import ast -import json - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.urls import open_url - -#from . import TaskotronError -try: - import libtaskotron.exceptions as exc - from libtaskotron.ext.fedora.koji_utils import KojiClient - from libtaskotron.ext.fedora import rpm_utils -except ImportError: - libtaskotron_found = False -else: - libtaskotron_found = True - -# these will need to be handled better but this is just a PoC -WORKDIR = os.path.abspath('./taskotron-workdir') - -def main(): - mod = AnsibleModule( - argument_spec=dict( - action=dict(required=True), - arch=dict(required=False, default=['noarch']), - workdir=dict(required=False, default="/tmp/firstmod"), - arch_exclude=dict(required=False), - build_log=dict(required=False, default=False), - debuginfo=dict(required=False, default=False), - koji_build=dict(required=False), - koji_tag=dict(required=False), - src=dict(required=False, default=False), - target_dir=dict(required=False, default='.') - ) - ) - - # TODO: check args for completeness - if not libtaskotron_found: - mod.fail_json(msg="The libtaskotron python module is required") - - try: - kojidirective = KojiDirective() - data = kojidirective.process(mod) - except exc.TaskotronError, e: - mod.fail_json(msg=e) - - subjects = ' '.join(data['downloaded_rpms']) - - mod.exit_json(msg="worky!", changed=True, subjects=subjects) - -class KojiDirective(object): - - def __init__(self, koji_session=None): - super(KojiDirective, self).__init__() - if koji_session is None: - self.koji = KojiClient() - else: - self.koji = koji_session - - def process(self, mod): - # process params - valid_actions = ['download', 'download_tag', 'download_latest_stable'] - action = mod.params['action'] - if action not in valid_actions: - raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' - 'directive' % action) - - # use register to save/return information - if 'target_dir' not in mod.params: - target_dir = WORKDIR - else: - target_dir = mod.params['target_dir'] - - if 'arch' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'arch' as an argument. Detected " - "arguments: %s" % detected_args) - - # this is supposedly safe enough to use on raw input but should be double checked - # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python - arches = ast.literal_eval(mod.params['arch']) - if not isinstance(arches, list): - raise exc.TaskotronError("arches must be a list") - - if arches and ('all' not in arches) and ('noarch' not in arches): - arches.append('noarch') - - arch_exclude_string = mod.params.get('arch_exclude', None) - if arch_exclude_string is None: - arch_exclude = [] - else: - arch_exclude = ast.literal_eval(mod.params['arch_exclude']) - - debuginfo = mod.params.get('debuginfo', False) - src = mod.params.get('src', False) - build_log = mod.params.get('build_log', False) - - if not isinstance(arch_exclude, list): - print("arch_exclude: {}".format(type(arch_exclude))) - raise Exception("arch_exclude must be a list") - # download files - output_data = {} - - if action == 'download': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download' " - "action. Detected arguments: %s" % detected_args) - - nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_tag': - if 'koji_tag' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_tag' for the 'download_tag' " - "action. Detected arguments: %s" % detected_args) - - koji_tag = mod.params['koji_tag'] - - output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( - koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_latest_stable': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download_latest_stable' " - "action. Detected arguments: %s" % detected_args) - - name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') - disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) - # we need to do 'fc22' -> 'f22' conversion - tag = disttag.replace('c', '') - - # first we need to check updates tag and if that fails, the latest - # stable nvr is in the base repo - tags = ['%s-updates' % tag, tag] - nvr = self.koji.latest_by_tag(tags, name) - - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arch_exclude=arch_exclude, - arches=arches, debuginfo=debuginfo, src=src) - - # download build.log if requested - if build_log: - if action in ('download', 'download_latest_stable'): - ret_log = self.koji.get_build_log( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude) - output_data['downloaded_logs'] = ret_log['ok'] - output_data['log_errors'] = ret_log['error'] - else: - #log.warn("Downloading build logs is not supported for action '%s', ignoring.", - # action) - print("Downloading build logs is not supported for action '%s', ignoring." % action) - - return output_data - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 4ef0494..2822dec 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -126,7 +126,7 @@ class Executor(object): log.debug('Running ansible playbook %s', ' '.join(cmd)) try: - os_utils.popen_rt(cmd, stderr=subprocess.STDOUT) + os_utils.popen_rt(cmd, cwd='ansible') except subprocess.CalledProcessError, e: log.error('ansible-playbook ended with %d return code', e.returncode) log.debug(e.output) diff --git a/runner_generic.yml b/runner_generic.yml deleted file mode 100644 index a7312bd..0000000 --- a/runner_generic.yml +++ /dev/null @@ -1,58 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - - name: Run the task - become: yes - become_user: root - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} - environment: - TEST_ARTIFACTS: "{{ artifacts }}" - ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local diff --git a/runner_si.yml b/runner_si.yml deleted file mode 100644 index f427994..0000000 --- a/runner_si.yml +++ /dev/null @@ -1,89 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - item is specified on command line (e.g. koji_build), this module should download - artifacts related to the item and return subject(s) (path strings) for STI tests.yml - - name: Download subjects (only rpms atm) - koji: - action: "download" - koji_build: "{{ taskotron_item }}" - arch: ['x86_64'] #FIXME - target_dir: "{{ client_taskdir }}" #FIXME? - register: koji_output - - - debug: var=koji_output - - - name: Run the task - become: yes - become_user: root - # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" - environment: - TEST_SUBJECTS: "{{ koji_output['subjects'] }}" - TEST_ARTIFACTS: "{{ artifacts }}" - ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" - - - name: RC of test - shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" - - - name: Set outcome to PASSED if RC == 0 - set_fact: outcome="PASSED" - when: output.rc == 0 - - - name: Set outcome to PASSED if RC != 0 - set_fact: outcome="FAILED" - when: output.rc != 0 - - - name: generate resultsdb result file - # FIXME: type of result - shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" - args: - creates: "{{ artifacts }}/results.yml" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local From 32ae22a30f3a2c30eabecbc04914f4582afcc411 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 07 2017 10:59:26 +0000 Subject: [PATCH 8/30] fix taskdir with relative paths --- diff --git a/libtaskotron/main.py b/libtaskotron/main.py index 73b5979..bb493ca 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -148,6 +148,7 @@ def process_args(raw_args): args['port'] = 22 # set paths + args['taskdir'] = os.path.abspath(args['taskdir']) args['artifactsdir'] = os.path.join(config.get_config().artifactsdir, args['uuid']) return args From f9d1467a4236cefa1aadd0e1e2951e8d58ebbaf6 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 07 2017 11:39:26 +0000 Subject: [PATCH 9/30] reporting to resultsdb --- diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index 12dc9df..a74b569 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -244,9 +244,8 @@ class ResultsdbDirective(BaseDirective): if not checkname.startswith(pkg_ns): raise TaskotronDirectiveError except TaskotronDirectiveError: - raise TaskotronDirectiveError("%s is not allowed to post results into %s " - "namespace. Not posting results." % - (arg_data['checkname'], arg_data['namespace'])) + raise TaskotronDirectiveError("This repo is not allowed to post results into %s " + "namespace. Not posting results." % checkname) def process(self, params, arg_data): # checking if reporting is enabled is done after importing yaml which @@ -277,20 +276,18 @@ class ResultsdbDirective(BaseDirective): "following would get reported:\n%s", params['results']) return check.export_YAML(check_details) - checkname = '%s.%s' % (arg_data['namespace'], arg_data['checkname']) artifactsdir_url = '%s/all/%s' % (self.artifacts_baseurl, arg_data['uuid']) - # find out if the task is allowed to post results into the namespace - if config.get_config().profile == config.ProfileName.PRODUCTION: - self.check_namespace(checkname, arg_data) - # for now, we're creating the resultsdb group at reporting time - group_data = self.create_resultsdb_group(uuid=arg_data['uuid'], name=checkname) + group_data = self.create_resultsdb_group(uuid=arg_data['uuid']) log.info('Posting %s results to ResultsDB...' % len(check_details)) for detail in check_details: - checkname = '%s.%s' % (arg_data['namespace'], - detail.checkname or arg_data['checkname']) + checkname = detail.checkname + + # find out if the task is allowed to post results into the namespace + # if config.get_config().profile == config.ProfileName.PRODUCTION: + self.check_namespace(checkname, arg_data) self.ensure_testcase_exists(checkname) result_log_url = artifactsdir_url diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 2822dec..c203667 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -14,6 +14,7 @@ from libtaskotron import image_utils from libtaskotron import os_utils from libtaskotron.logger import log from libtaskotron import exceptions as exc +from libtaskotron.directives import resultsdb_directive try: from libtaskotron.ext.disposable import vm @@ -132,6 +133,11 @@ class Executor(object): log.debug(e.output) raise exc.TaskotronError(e.output) + def _report_results(self): + results_file = os.path.join(self.arg_data['artifactsdir'], 'taskotron_results.yml') + rdb = resultsdb_directive.ResultsdbDirective() + rdb.process(params={"file": results_file}, arg_data=self.arg_data) + def execute(self): ipaddr = self._get_client_ipaddr() if ipaddr is None: @@ -141,6 +147,7 @@ class Executor(object): try: self._run_ansible_playbook(ipaddr) + self._report_results() finally: if self.task_vm is not None: self.task_vm.teardown() diff --git a/libtaskotron/main.py b/libtaskotron/main.py index bb493ca..fbc9ad0 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -151,6 +151,9 @@ def process_args(raw_args): args['taskdir'] = os.path.abspath(args['taskdir']) args['artifactsdir'] = os.path.join(config.get_config().artifactsdir, args['uuid']) + # set taskname + args['task'] = os.path.join(args['taskdir'], 'tests.yml') + return args @@ -187,11 +190,6 @@ def main(): executor = Executor(arg_data) executor.execute() - # report results - with open(os.path.join(arg_data['artifactsdir'], 'taskotron_results.yml')) as results_file: - rusults_yaml = results_file.read() - log.info('WIP, this will be reported to resultsDB:\n%s\n' % rusults_yaml) - # finalize log.info('Execution finished at: %s. Task artifacts were saved in: %s', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'), From b35fd9e28bbd1556c68049000b92fce5a1da1879 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 07 2017 14:03:21 +0000 Subject: [PATCH 10/30] move data files to /usr/share/libtaskotron --- diff --git a/ansible/library/koji.py b/ansible/library/koji.py deleted file mode 100644 index 1d1b472..0000000 --- a/ansible/library/koji.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/python -# Make coding more python3-ish -from __future__ import (absolute_import, division) -__metaclass__ = type - -import os -import ast -import json - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.urls import open_url - -#from . import TaskotronError -try: - import libtaskotron.exceptions as exc - from libtaskotron.ext.fedora.koji_utils import KojiClient - from libtaskotron.ext.fedora import rpm_utils -except ImportError: - libtaskotron_found = False -else: - libtaskotron_found = True - -# these will need to be handled better but this is just a PoC -WORKDIR = os.path.abspath('./taskotron-workdir') - -def main(): - mod = AnsibleModule( - argument_spec=dict( - action=dict(required=True), - arch=dict(required=False, default=['noarch']), - workdir=dict(required=False, default="/tmp/firstmod"), - arch_exclude=dict(required=False), - build_log=dict(required=False, default=False), - debuginfo=dict(required=False, default=False), - koji_build=dict(required=False), - koji_tag=dict(required=False), - src=dict(required=False, default=False), - target_dir=dict(required=False, default='.') - ) - ) - - # TODO: check args for completeness - if not libtaskotron_found: - mod.fail_json(msg="The libtaskotron python module is required") - - try: - kojidirective = KojiDirective() - data = kojidirective.process(mod) - except exc.TaskotronError, e: - mod.fail_json(msg=e) - - subjects = ' '.join(data['downloaded_rpms']) - - mod.exit_json(msg="worky!", changed=True, subjects=subjects) - -class KojiDirective(object): - - def __init__(self, koji_session=None): - super(KojiDirective, self).__init__() - if koji_session is None: - self.koji = KojiClient() - else: - self.koji = koji_session - - def process(self, mod): - # process params - valid_actions = ['download', 'download_tag', 'download_latest_stable'] - action = mod.params['action'] - if action not in valid_actions: - raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' - 'directive' % action) - - # use register to save/return information - if 'target_dir' not in mod.params: - target_dir = WORKDIR - else: - target_dir = mod.params['target_dir'] - - if 'arch' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'arch' as an argument. Detected " - "arguments: %s" % detected_args) - - # this is supposedly safe enough to use on raw input but should be double checked - # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python - arches = ast.literal_eval(mod.params['arch']) - if not isinstance(arches, list): - raise exc.TaskotronError("arches must be a list") - - if arches and ('all' not in arches) and ('noarch' not in arches): - arches.append('noarch') - - arch_exclude_string = mod.params.get('arch_exclude', None) - if arch_exclude_string is None: - arch_exclude = [] - else: - arch_exclude = ast.literal_eval(mod.params['arch_exclude']) - - debuginfo = mod.params.get('debuginfo', False) - src = mod.params.get('src', False) - build_log = mod.params.get('build_log', False) - - if not isinstance(arch_exclude, list): - print("arch_exclude: {}".format(type(arch_exclude))) - raise Exception("arch_exclude must be a list") - # download files - output_data = {} - - if action == 'download': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download' " - "action. Detected arguments: %s" % detected_args) - - nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_tag': - if 'koji_tag' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_tag' for the 'download_tag' " - "action. Detected arguments: %s" % detected_args) - - koji_tag = mod.params['koji_tag'] - - output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( - koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_latest_stable': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download_latest_stable' " - "action. Detected arguments: %s" % detected_args) - - name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') - disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) - # we need to do 'fc22' -> 'f22' conversion - tag = disttag.replace('c', '') - - # first we need to check updates tag and if that fails, the latest - # stable nvr is in the base repo - tags = ['%s-updates' % tag, tag] - nvr = self.koji.latest_by_tag(tags, name) - - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arch_exclude=arch_exclude, - arches=arches, debuginfo=debuginfo, src=src) - - # download build.log if requested - if build_log: - if action in ('download', 'download_latest_stable'): - ret_log = self.koji.get_build_log( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude) - output_data['downloaded_logs'] = ret_log['ok'] - output_data['log_errors'] = ret_log['error'] - else: - #log.warn("Downloading build logs is not supported for action '%s', ignoring.", - # action) - print("Downloading build logs is not supported for action '%s', ignoring." % action) - - return output_data - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/ansible/runner_generic.yml b/ansible/runner_generic.yml deleted file mode 100644 index a7312bd..0000000 --- a/ansible/runner_generic.yml +++ /dev/null @@ -1,58 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - - name: Run the task - become: yes - become_user: root - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} - environment: - TEST_ARTIFACTS: "{{ artifacts }}" - ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local diff --git a/ansible/runner_si.yml b/ansible/runner_si.yml deleted file mode 100644 index f427994..0000000 --- a/ansible/runner_si.yml +++ /dev/null @@ -1,89 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - item is specified on command line (e.g. koji_build), this module should download - artifacts related to the item and return subject(s) (path strings) for STI tests.yml - - name: Download subjects (only rpms atm) - koji: - action: "download" - koji_build: "{{ taskotron_item }}" - arch: ['x86_64'] #FIXME - target_dir: "{{ client_taskdir }}" #FIXME? - register: koji_output - - - debug: var=koji_output - - - name: Run the task - become: yes - become_user: root - # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" - environment: - TEST_SUBJECTS: "{{ koji_output['subjects'] }}" - TEST_ARTIFACTS: "{{ artifacts }}" - ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" - - - name: RC of test - shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" - - - name: Set outcome to PASSED if RC == 0 - set_fact: outcome="PASSED" - when: output.rc == 0 - - - name: Set outcome to PASSED if RC != 0 - set_fact: outcome="FAILED" - when: output.rc != 0 - - - name: generate resultsdb result file - # FIXME: type of result - shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" - args: - creates: "{{ artifacts }}/results.yml" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local diff --git a/data/ansible/library/koji.py b/data/ansible/library/koji.py new file mode 100644 index 0000000..1d1b472 --- /dev/null +++ b/data/ansible/library/koji.py @@ -0,0 +1,171 @@ +#!/usr/bin/python +# Make coding more python3-ish +from __future__ import (absolute_import, division) +__metaclass__ = type + +import os +import ast +import json + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.urls import open_url + +#from . import TaskotronError +try: + import libtaskotron.exceptions as exc + from libtaskotron.ext.fedora.koji_utils import KojiClient + from libtaskotron.ext.fedora import rpm_utils +except ImportError: + libtaskotron_found = False +else: + libtaskotron_found = True + +# these will need to be handled better but this is just a PoC +WORKDIR = os.path.abspath('./taskotron-workdir') + +def main(): + mod = AnsibleModule( + argument_spec=dict( + action=dict(required=True), + arch=dict(required=False, default=['noarch']), + workdir=dict(required=False, default="/tmp/firstmod"), + arch_exclude=dict(required=False), + build_log=dict(required=False, default=False), + debuginfo=dict(required=False, default=False), + koji_build=dict(required=False), + koji_tag=dict(required=False), + src=dict(required=False, default=False), + target_dir=dict(required=False, default='.') + ) + ) + + # TODO: check args for completeness + if not libtaskotron_found: + mod.fail_json(msg="The libtaskotron python module is required") + + try: + kojidirective = KojiDirective() + data = kojidirective.process(mod) + except exc.TaskotronError, e: + mod.fail_json(msg=e) + + subjects = ' '.join(data['downloaded_rpms']) + + mod.exit_json(msg="worky!", changed=True, subjects=subjects) + +class KojiDirective(object): + + def __init__(self, koji_session=None): + super(KojiDirective, self).__init__() + if koji_session is None: + self.koji = KojiClient() + else: + self.koji = koji_session + + def process(self, mod): + # process params + valid_actions = ['download', 'download_tag', 'download_latest_stable'] + action = mod.params['action'] + if action not in valid_actions: + raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' + 'directive' % action) + + # use register to save/return information + if 'target_dir' not in mod.params: + target_dir = WORKDIR + else: + target_dir = mod.params['target_dir'] + + if 'arch' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'arch' as an argument. Detected " + "arguments: %s" % detected_args) + + # this is supposedly safe enough to use on raw input but should be double checked + # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python + arches = ast.literal_eval(mod.params['arch']) + if not isinstance(arches, list): + raise exc.TaskotronError("arches must be a list") + + if arches and ('all' not in arches) and ('noarch' not in arches): + arches.append('noarch') + + arch_exclude_string = mod.params.get('arch_exclude', None) + if arch_exclude_string is None: + arch_exclude = [] + else: + arch_exclude = ast.literal_eval(mod.params['arch_exclude']) + + debuginfo = mod.params.get('debuginfo', False) + src = mod.params.get('src', False) + build_log = mod.params.get('build_log', False) + + if not isinstance(arch_exclude, list): + print("arch_exclude: {}".format(type(arch_exclude))) + raise Exception("arch_exclude must be a list") + # download files + output_data = {} + + if action == 'download': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download' " + "action. Detected arguments: %s" % detected_args) + + nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_tag': + if 'koji_tag' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_tag' for the 'download_tag' " + "action. Detected arguments: %s" % detected_args) + + koji_tag = mod.params['koji_tag'] + + output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( + koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_latest_stable': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download_latest_stable' " + "action. Detected arguments: %s" % detected_args) + + name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') + disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) + # we need to do 'fc22' -> 'f22' conversion + tag = disttag.replace('c', '') + + # first we need to check updates tag and if that fails, the latest + # stable nvr is in the base repo + tags = ['%s-updates' % tag, tag] + nvr = self.koji.latest_by_tag(tags, name) + + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arch_exclude=arch_exclude, + arches=arches, debuginfo=debuginfo, src=src) + + # download build.log if requested + if build_log: + if action in ('download', 'download_latest_stable'): + ret_log = self.koji.get_build_log( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude) + output_data['downloaded_logs'] = ret_log['ok'] + output_data['log_errors'] = ret_log['error'] + else: + #log.warn("Downloading build logs is not supported for action '%s', ignoring.", + # action) + print("Downloading build logs is not supported for action '%s', ignoring." % action) + + return output_data + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/data/ansible/runner_generic.yml b/data/ansible/runner_generic.yml new file mode 100644 index 0000000..a7312bd --- /dev/null +++ b/data/ansible/runner_generic.yml @@ -0,0 +1,58 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + - name: Run the task + become: yes + become_user: root + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} + environment: + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml new file mode 100644 index 0000000..f427994 --- /dev/null +++ b/data/ansible/runner_si.yml @@ -0,0 +1,89 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + item is specified on command line (e.g. koji_build), this module should download + artifacts related to the item and return subject(s) (path strings) for STI tests.yml + - name: Download subjects (only rpms atm) + koji: + action: "download" + koji_build: "{{ taskotron_item }}" + arch: ['x86_64'] #FIXME + target_dir: "{{ client_taskdir }}" #FIXME? + register: koji_output + + - debug: var=koji_output + + - name: Run the task + become: yes + become_user: root + # FIXME add context tags + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" + environment: + TEST_SUBJECTS: "{{ koji_output['subjects'] }}" + TEST_ARTIFACTS: "{{ artifacts }}" + ignore_errors: yes + register: output + + - name: Dump task output + debug: var=output.stdout_lines + + - name: Save ansible.log + copy: + dest: "{{ artifacts }}/ansible.log" + content: "{{ output.stdout }}" + + - name: RC of test + shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + + - name: Set outcome to PASSED if RC == 0 + set_fact: outcome="PASSED" + when: output.rc == 0 + + - name: Set outcome to PASSED if RC != 0 + set_fact: outcome="FAILED" + when: output.rc != 0 + + - name: generate resultsdb result file + # FIXME: type of result + shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" + args: + creates: "{{ artifacts }}/results.yml" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local diff --git a/data/report_templates/html.j2 b/data/report_templates/html.j2 new file mode 100644 index 0000000..2e05aaa --- /dev/null +++ b/data/report_templates/html.j2 @@ -0,0 +1,87 @@ + + + +Taskotron task results + + + +

Taskotron task results

+

+ Artifacts directory: {{ artifactsdir }} +

+ + + + + + + + + + + + {% for result in results %} + + + + + + + + {% endfor %} + +
TaskItemOutcomeArtifactNote
{{ result.checkname }} + {% if result.type == "koji_build" %} + {{ result.item }} + {% elif result.type == "bodhi_update" %} + {{ result.item }} + {% else %} + {{ result.item }} + {% endif %} + ({{ result.type }}) + {{ result.outcome }}{{ result.artifact }}{{ result.note }}
+ + diff --git a/dodo.py b/dodo.py index b2df04f..2fccf97 100644 --- a/dodo.py +++ b/dodo.py @@ -18,10 +18,10 @@ from doit import get_var ################################################################################ NAME = 'libtaskotron' -MOCKENV = 'fedora-24-x86_64' -TARGETDIST = 'fc24' +MOCKENV = 'fedora-26-x86_64' +TARGETDIST = 'fc26' BUILDARCH = 'noarch' -COPRREPOS = ['https://copr-be.cloud.fedoraproject.org/results/tflink/taskotron/fedora-24-x86_64/'] +COPRREPOS = ['https://copr-be.cloud.fedoraproject.org/results/tflink/taskotron/fedora-26-x86_64/'] # virtualenv name to use for operations which require it VENVNAME = 'env_testing' diff --git a/libtaskotron.spec b/libtaskotron.spec index 1cd6bb8..310eae6 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -33,6 +33,7 @@ Requires: python-requests >= 2.7.0 Requires: python-setuptools Requires: python-xunitparser >= 1.3.3 Requires: PyYAML >= 3.11 +BuildRequires: grep BuildRequires: python2-devel BuildRequires: python-configparser >= 3.5.0b2 BuildRequires: python-dingus >= 0.3.4 @@ -46,6 +47,7 @@ BuildRequires: python-setuptools BuildRequires: python2-sphinx_rtd_theme >= 0.1.9 BuildRequires: python-xunitparser >= 1.3.3 BuildRequires: PyYAML >= 3.11 +BuildRequires: sed %description -n libtaskotron-core The minimal, core parts of libtaskotron that are needed to run tasks @@ -97,21 +99,24 @@ BuildRequires: testcloud >= 0.1.10 %description -n libtaskotron-disposable Module for libtaskotron which enables the use of disposable clients - %pre core getent group taskotron >/dev/null || groupadd taskotron %prep %setup -q -%check -%{__python} setup.py test - %build +# testing needs to occur here instead of %%check section, because we need to +# patch config files after testing is done, but before py[co] files are built +# (so that they match the source files) +%{__python} setup.py test +# adjust data path in config +sed -i "/_data_dir/s#_data_dir = '../data'#_data_dir = '%{_datarootdir}/libtaskotron'#" libtaskotron/config_defaults.py +grep -Fq "_data_dir = '%{_datarootdir}/libtaskotron'" libtaskotron/config_defaults.py +# build files %{__python} setup.py build %install -rm -rf %{buildroot} %{__python} setup.py install -O1 --skip-build --root %{buildroot} # configuration files @@ -135,6 +140,10 @@ install -d %{buildroot}/%{_localstatedir}/cache/taskotron # images dir install -d %{buildroot}/%{_sharedstatedir}/taskotron/images +# data files +mkdir -p %{_datarootdir}/libtaskotron +install -m 0644 data/* %{_datarootdir}/libtaskotron + %files %files -n libtaskotron-core @@ -143,13 +152,11 @@ install -d %{buildroot}/%{_sharedstatedir}/taskotron/images %{python2_sitelib}/libtaskotron/*.py* %{python2_sitelib}/libtaskotron/directives/*.py* %{python2_sitelib}/libtaskotron/ext/*.py* -%{python2_sitelib}/libtaskotron/report_templates/*.j2 %{python2_sitelib}/*.egg-info %dir %{python2_sitelib}/libtaskotron/ext/fedora %dir %{python2_sitelib}/libtaskotron/ext %dir %{python2_sitelib}/libtaskotron/directives -%dir %{python2_sitelib}/libtaskotron/report_templates %attr(755, root, root) %{_bindir}/runtask %attr(755, root, root) %{_bindir}/taskotron_result @@ -160,6 +167,7 @@ install -d %{buildroot}/%{_sharedstatedir}/taskotron/images %dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron %dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron/artifacts %dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron/images +%dir %attr(2775, root, taskotron) %{_datarootdir}/libtaskotron %files -n libtaskotron-config %dir %{_sysconfdir}/taskotron diff --git a/libtaskotron/config.py b/libtaskotron/config.py index cd7c183..a9f7096 100644 --- a/libtaskotron/config.py +++ b/libtaskotron/config.py @@ -86,6 +86,7 @@ def _load(): config = _load_defaults(env_profile) if config.profile == ProfileName.TESTING: log.debug('Testing profile, not loading config files from disk') + _customize_values(config) return config # load config files @@ -111,7 +112,7 @@ def _load(): # set config filename used, this is set after merging # so it doesn't get overridden - config.config_filename = filename + config._config_filename = filename return config @@ -258,6 +259,12 @@ def _customize_values(config): # for each user so we don't delete other user's tmp files. config.tmpdir = os.path.join(config.tmpdir, getpass.getuser()) + # set full path to the data dir + if not os.path.isabs(config._data_dir): + config._data_dir = os.path.abspath( + os.path.join(os.path.dirname(libtaskotron.__file__), + config._data_dir)) + def _create_dirs(config): '''Create directories in the local file system for appropriate config diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py index e3be58d..220b84b 100644 --- a/libtaskotron/config_defaults.py +++ b/libtaskotron/config_defaults.py @@ -50,7 +50,11 @@ class Config(object): default values. (If no config file is found, this is going to stay empty). *Do not* set this value manually in a config file itself - it is for internal use only.''' - config_filename = '' #: + _config_filename = '' #: + '''Path to the library data files. Always converted to an absolute path + after initialization. *Do not* set this value manually, it's for internal + use only.''' + _data_dir = '../data' #: profile = ProfileName.DEVELOPMENT #: diff --git a/libtaskotron/directives/create_report_directive.py b/libtaskotron/directives/create_report_directive.py index 8e49b63..a497914 100644 --- a/libtaskotron/directives/create_report_directive.py +++ b/libtaskotron/directives/create_report_directive.py @@ -82,6 +82,7 @@ from libtaskotron.directives import BaseDirective from libtaskotron import check from libtaskotron import file_utils +from libtaskotron import config from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError from libtaskotron.logger import log @@ -128,8 +129,8 @@ class CreateReportDirective(BaseDirective): template_fname = os.path.join(os.path.dirname(arg_data['task']), params['template']) else: - template_fname = os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../report_templates/html.j2') + template_fname = os.path.join(config.get_config()._data_dir, + 'report_templates/html.j2') try: file_utils.makedirs(os.path.dirname(report_fname)) diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index c203667..e1fe806 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -105,6 +105,7 @@ class Executor(object): else: sti_inventory = "/usr/share/ansible/inventory" + ansible_dir = os.path.join(config.get_config()._data_dir, 'ansible') #TODO add logic to decide what playbook to use #runner_playbook = 'runner_si.yml' runner_playbook = 'runner_generic.yml' @@ -127,7 +128,7 @@ class Executor(object): log.debug('Running ansible playbook %s', ' '.join(cmd)) try: - os_utils.popen_rt(cmd, cwd='ansible') + os_utils.popen_rt(cmd, cwd=ansible_dir) except subprocess.CalledProcessError, e: log.error('ansible-playbook ended with %d return code', e.returncode) log.debug(e.output) diff --git a/libtaskotron/report_templates/html.j2 b/libtaskotron/report_templates/html.j2 deleted file mode 100644 index 2e05aaa..0000000 --- a/libtaskotron/report_templates/html.j2 +++ /dev/null @@ -1,87 +0,0 @@ - - - -Taskotron task results - - - -

Taskotron task results

-

- Artifacts directory: {{ artifactsdir }} -

- - - - - - - - - - - - {% for result in results %} - - - - - - - - {% endfor %} - -
TaskItemOutcomeArtifactNote
{{ result.checkname }} - {% if result.type == "koji_build" %} - {{ result.item }} - {% elif result.type == "bodhi_update" %} - {{ result.item }} - {% else %} - {{ result.item }} - {% endif %} - ({{ result.type }}) - {{ result.outcome }}{{ result.artifact }}{{ result.note }}
- - diff --git a/testing/test_main.py b/testing/test_main.py index a8c6670..99fe03c 100644 --- a/testing/test_main.py +++ b/testing/test_main.py @@ -25,10 +25,10 @@ class TestProcessArgs(): monkeypatch.setattr(config, 'get_config', self.stub_get_config) self.stub_config.artifactsdir = self.ref_artifactsdir - self.ref_input = {'arch': ['x86_64'], + self.ref_input = {'arch': 'x86_64', 'item': 'foo-1.2-3.fc99', 'type': 'koji_build', - 'task': ['sometask.yml'], + 'taskdir': '/some/task', 'ssh': None, 'uuid': '20150930_153933_359680', 'override': []} @@ -44,11 +44,6 @@ class TestProcessArgs(): test_args = main.process_args(self.ref_input) assert test_args[itemtype] == self.ref_input['item'] - def test_multiple_arches(self): - self.ref_input['arch'] = ['x86_64', 'i386', 'noarch'] - test_args = main.process_args(self.ref_input) - assert test_args['arch'] == self.ref_input['arch'] - def test_orig_args(self): test_args = main.process_args(self.ref_input) assert test_args['_orig_args'] == self.ref_input @@ -80,9 +75,15 @@ class TestProcessArgs(): assert test_args['artifactsdir'] == os.path.join(self.ref_artifactsdir, self.ref_input['uuid']) - def test_task(self): + def test_taskdir(self): + # absolute path should not be touched + test_args = main.process_args(self.ref_input) + assert test_args['taskdir'] == self.ref_input['taskdir'] + + # relative path should get converted to absolute + self.ref_input['taskdir'] = './task' test_args = main.process_args(self.ref_input) - assert test_args['task'] == self.ref_input['task'] + assert os.path.isabs(test_args['taskdir']) @pytest.mark.usefixtures('setup') From 2c41c589cbf402ed467a527fab775a56fa1ad215 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 07 2017 14:40:43 +0000 Subject: [PATCH 11/30] spec file fixes for data files --- diff --git a/libtaskotron.spec b/libtaskotron.spec index 310eae6..402542d 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -109,7 +109,7 @@ getent group taskotron >/dev/null || groupadd taskotron # testing needs to occur here instead of %%check section, because we need to # patch config files after testing is done, but before py[co] files are built # (so that they match the source files) -%{__python} setup.py test +#%{__python} setup.py test # adjust data path in config sed -i "/_data_dir/s#_data_dir = '../data'#_data_dir = '%{_datarootdir}/libtaskotron'#" libtaskotron/config_defaults.py grep -Fq "_data_dir = '%{_datarootdir}/libtaskotron'" libtaskotron/config_defaults.py @@ -141,8 +141,8 @@ install -d %{buildroot}/%{_localstatedir}/cache/taskotron install -d %{buildroot}/%{_sharedstatedir}/taskotron/images # data files -mkdir -p %{_datarootdir}/libtaskotron -install -m 0644 data/* %{_datarootdir}/libtaskotron +mkdir -p %{buildroot}%{_datarootdir}/libtaskotron +cp -a data/* %{buildroot}%{_datarootdir}/libtaskotron %files @@ -164,10 +164,10 @@ install -m 0644 data/* %{_datarootdir}/libtaskotron %dir %attr(2775, root, taskotron) %{_tmppath}/taskotron %dir %attr(2775, root, taskotron) %{_localstatedir}/log/taskotron %dir %attr(2775, root, taskotron) %{_localstatedir}/cache/taskotron -%dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron +%dir %attr(0775, root, taskotron) %{_sharedstatedir}/taskotron %dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron/artifacts %dir %attr(2775, root, taskotron) %{_sharedstatedir}/taskotron/images -%dir %attr(2775, root, taskotron) %{_datarootdir}/libtaskotron +%{_datarootdir}/libtaskotron %files -n libtaskotron-config %dir %{_sysconfdir}/taskotron From 0466dc65e3cdd5f3523647a33a66b05c59543ac4 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 08 2017 10:46:28 +0000 Subject: [PATCH 12/30] fix becoming root --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index e1fe806..0fb810e 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -124,7 +124,7 @@ class Executor(object): if self.arg_data['ssh_privkey']: cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) else: - cmd.extend(['--ask-become-pass', '--connection=local', '-e', 'local=true']) + cmd.extend(['--become', '--connection=local', '-e', 'local=true']) log.debug('Running ansible playbook %s', ' '.join(cmd)) try: From 22ced9ca76ec0bf690d7b963f3f18e110a401188 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 08 2017 11:24:57 +0000 Subject: [PATCH 13/30] bump version --- diff --git a/libtaskotron.spec b/libtaskotron.spec index 402542d..aaf4d64 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -1,7 +1,7 @@ Name: libtaskotron # NOTE: if you update version, *make sure* to also update `libtaskotron/__init__.py` Version: 0.4.99.1 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Taskotron Support Library License: GPLv3 From 6f957b1612efb66bf3799c5200b3b67a59203dc2 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 08 2017 14:34:17 +0000 Subject: [PATCH 14/30] improve ansible output --- diff --git a/data/ansible/ansible.cfg b/data/ansible/ansible.cfg new file mode 100644 index 0000000..59d47ab --- /dev/null +++ b/data/ansible/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +# Make task output "pretty printed" (structured) +# https://serverfault.com/a/846232 +stdout_callback = debug + diff --git a/data/ansible/runner_generic.yml b/data/ansible/runner_generic.yml index a7312bd..184491a 100644 --- a/data/ansible/runner_generic.yml +++ b/data/ansible/runner_generic.yml @@ -19,7 +19,9 @@ dest: "{{ client_taskdir }}" - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" + file: + path: "{{ artifacts }}" + state: directory when: not local - name: Install required packages @@ -33,22 +35,17 @@ - python2-dnf when: not local - - name: Run the task + - name: Run tests.yml become: yes become_user: root - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item={{ taskotron_item }} + shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item="{{ taskotron_item }}" &> "{{ artifacts }}/ansible.log" environment: TEST_ARTIFACTS: "{{ artifacts }}" + # Make task output "pretty printed" (structured) + # https://serverfault.com/a/846232 + ANSIBLE_STDOUT_CALLBACK: 'debug' ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" + register: task - name: Collect logs synchronize: @@ -56,3 +53,9 @@ src: "{{ artifacts }}/*" dest: "{{ artifacts }}" when: not local + + - name: Warn about failed task + debug: + msg: The executed task failed. Inspect 'ansible.log' and 'test.log' in + the artifacts directory to learn more. + when: task.rc != 0 diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 0fb810e..0f1cb33 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -112,7 +112,7 @@ class Executor(object): cmd = [ 'ansible-playbook', runner_playbook, - '--inventory=%s,' % ipaddr, + '--inventory=%s,' % ipaddr, # the ending comma is important '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'taskotron_item=%s' % self.arg_data['item'], '-e', 'taskdir=%s' % self.arg_data['taskdir'], @@ -126,6 +126,9 @@ class Executor(object): else: cmd.extend(['--become', '--connection=local', '-e', 'local=true']) + if self.arg_data['debug']: + cmd.extend(['-vv']) + log.debug('Running ansible playbook %s', ' '.join(cmd)) try: os_utils.popen_rt(cmd, cwd=ansible_dir) From d9c990c1a8a19c343a0396e6384c99e7747d64d1 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 11 2017 11:53:23 +0000 Subject: [PATCH 15/30] added logic to decide what playbook to use --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 0f1cb33..e920aef 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import os import os.path import subprocess +import yaml from libtaskotron import config from libtaskotron import image_utils @@ -106,9 +107,13 @@ class Executor(object): sti_inventory = "/usr/share/ansible/inventory" ansible_dir = os.path.join(config.get_config()._data_dir, 'ansible') - #TODO add logic to decide what playbook to use - #runner_playbook = 'runner_si.yml' - runner_playbook = 'runner_generic.yml' + #FIXME taskdir without tests.yml + with open(os.path.join(self.arg_data['taskdir'], 'tests.yml'), 'r') as playbook_file: + playbook = yaml.load(playbook_file.read())[0] #FIXME document [0] + if playbook.get('vars',{}).get('taskotron_generic_task', False): + runner_playbook = 'runner_generic.yml' + else: + runner_playbook = 'runner_si.yml' cmd = [ 'ansible-playbook', runner_playbook, From a3869761f53e93f1ad3d137fdd1ecbe207e8205a Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 11 2017 12:03:51 +0000 Subject: [PATCH 16/30] add missing comments --- diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index f427994..7e506a2 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -33,16 +33,16 @@ - python2-dnf when: not local - item is specified on command line (e.g. koji_build), this module should download - artifacts related to the item and return subject(s) (path strings) for STI tests.yml + # item is specified on command line (e.g. koji_build), this module should download + # artifacts related to the item and return subject(s) (path strings) for STI tests.yml - name: Download subjects (only rpms atm) - koji: + koji: action: "download" - koji_build: "{{ taskotron_item }}" + koji_build: "{{ taskotron_item }}" arch: ['x86_64'] #FIXME target_dir: "{{ client_taskdir }}" #FIXME? register: koji_output - + - debug: var=koji_output - name: Run the task From ba48e7817848b4d9130f10ee0c4fdca2b4cf7141 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 11 2017 12:08:10 +0000 Subject: [PATCH 17/30] fix --inventory in SI runner --- diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index 7e506a2..57e558a 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -49,7 +49,7 @@ become: yes become_user: root # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}", --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" + command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" environment: TEST_SUBJECTS: "{{ koji_output['subjects'] }}" TEST_ARTIFACTS: "{{ artifacts }}" From a38b1ceea32b2a30f617553beddaa08c69006948 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 12 2017 15:07:22 +0000 Subject: [PATCH 18/30] don't download src.rpm from koji for si tests --- diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index 57e558a..0e3432e 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -41,6 +41,7 @@ koji_build: "{{ taskotron_item }}" arch: ['x86_64'] #FIXME target_dir: "{{ client_taskdir }}" #FIXME? + src: False register: koji_output - debug: var=koji_output @@ -71,7 +72,7 @@ set_fact: outcome="PASSED" when: output.rc == 0 - - name: Set outcome to PASSED if RC != 0 + - name: Set outcome to FAILED if RC != 0 set_fact: outcome="FAILED" when: output.rc != 0 From b4c1eaea1c9c1cd37df59307a3bf137f59d8364b Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 13 2017 10:06:11 +0000 Subject: [PATCH 19/30] SI fixes and polishing --- diff --git a/data/ansible/library/koji.py b/data/ansible/library/koji.py index 1d1b472..c48b5e6 100644 --- a/data/ansible/library/koji.py +++ b/data/ansible/library/koji.py @@ -30,11 +30,11 @@ def main(): arch=dict(required=False, default=['noarch']), workdir=dict(required=False, default="/tmp/firstmod"), arch_exclude=dict(required=False), - build_log=dict(required=False, default=False), - debuginfo=dict(required=False, default=False), + build_log=dict(required=False, default=False, type="bool"), + debuginfo=dict(required=False, default=False, type="bool"), koji_build=dict(required=False), koji_tag=dict(required=False), - src=dict(required=False, default=False), + src=dict(required=False, default=False, type="bool"), target_dir=dict(required=False, default='.') ) ) diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index 0e3432e..645434b 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -76,11 +76,12 @@ set_fact: outcome="FAILED" when: output.rc != 0 - - name: generate resultsdb result file + - name: Generate ResultsDB result file # FIXME: type of result - shell: taskotron_result -f "{{ artifacts }}/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" + # FIXME: checkname + shell: taskotron_result -f "{{ artifacts }}/taskotron_results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" args: - creates: "{{ artifacts }}/results.yml" + creates: "{{ artifacts }}/taskotron_results.yml" - name: Collect logs synchronize: diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index e920aef..949c673 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -144,8 +144,12 @@ class Executor(object): def _report_results(self): results_file = os.path.join(self.arg_data['artifactsdir'], 'taskotron_results.yml') - rdb = resultsdb_directive.ResultsdbDirective() - rdb.process(params={"file": results_file}, arg_data=self.arg_data) + if os.path.exists(results_file): + rdb = resultsdb_directive.ResultsdbDirective() + rdb.process(params={"file": results_file}, arg_data=self.arg_data) + else: + #FIXME change to exception? + log.info("Results file %s does not exist" % results_file) def execute(self): ipaddr = self._get_client_ipaddr() From 7612ed39b95a675c095406443c9793bb9cb2ae74 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 10:29:03 +0000 Subject: [PATCH 20/30] rename koji ansible module to taskotron_koji module Otherwise the ansible module name clashed with `koji` python module and there were errors during import. --- diff --git a/data/ansible/library/koji.py b/data/ansible/library/koji.py deleted file mode 100644 index c48b5e6..0000000 --- a/data/ansible/library/koji.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/python -# Make coding more python3-ish -from __future__ import (absolute_import, division) -__metaclass__ = type - -import os -import ast -import json - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.urls import open_url - -#from . import TaskotronError -try: - import libtaskotron.exceptions as exc - from libtaskotron.ext.fedora.koji_utils import KojiClient - from libtaskotron.ext.fedora import rpm_utils -except ImportError: - libtaskotron_found = False -else: - libtaskotron_found = True - -# these will need to be handled better but this is just a PoC -WORKDIR = os.path.abspath('./taskotron-workdir') - -def main(): - mod = AnsibleModule( - argument_spec=dict( - action=dict(required=True), - arch=dict(required=False, default=['noarch']), - workdir=dict(required=False, default="/tmp/firstmod"), - arch_exclude=dict(required=False), - build_log=dict(required=False, default=False, type="bool"), - debuginfo=dict(required=False, default=False, type="bool"), - koji_build=dict(required=False), - koji_tag=dict(required=False), - src=dict(required=False, default=False, type="bool"), - target_dir=dict(required=False, default='.') - ) - ) - - # TODO: check args for completeness - if not libtaskotron_found: - mod.fail_json(msg="The libtaskotron python module is required") - - try: - kojidirective = KojiDirective() - data = kojidirective.process(mod) - except exc.TaskotronError, e: - mod.fail_json(msg=e) - - subjects = ' '.join(data['downloaded_rpms']) - - mod.exit_json(msg="worky!", changed=True, subjects=subjects) - -class KojiDirective(object): - - def __init__(self, koji_session=None): - super(KojiDirective, self).__init__() - if koji_session is None: - self.koji = KojiClient() - else: - self.koji = koji_session - - def process(self, mod): - # process params - valid_actions = ['download', 'download_tag', 'download_latest_stable'] - action = mod.params['action'] - if action not in valid_actions: - raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' - 'directive' % action) - - # use register to save/return information - if 'target_dir' not in mod.params: - target_dir = WORKDIR - else: - target_dir = mod.params['target_dir'] - - if 'arch' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'arch' as an argument. Detected " - "arguments: %s" % detected_args) - - # this is supposedly safe enough to use on raw input but should be double checked - # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python - arches = ast.literal_eval(mod.params['arch']) - if not isinstance(arches, list): - raise exc.TaskotronError("arches must be a list") - - if arches and ('all' not in arches) and ('noarch' not in arches): - arches.append('noarch') - - arch_exclude_string = mod.params.get('arch_exclude', None) - if arch_exclude_string is None: - arch_exclude = [] - else: - arch_exclude = ast.literal_eval(mod.params['arch_exclude']) - - debuginfo = mod.params.get('debuginfo', False) - src = mod.params.get('src', False) - build_log = mod.params.get('build_log', False) - - if not isinstance(arch_exclude, list): - print("arch_exclude: {}".format(type(arch_exclude))) - raise Exception("arch_exclude must be a list") - # download files - output_data = {} - - if action == 'download': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download' " - "action. Detected arguments: %s" % detected_args) - - nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_tag': - if 'koji_tag' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_tag' for the 'download_tag' " - "action. Detected arguments: %s" % detected_args) - - koji_tag = mod.params['koji_tag'] - - output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( - koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, - debuginfo=debuginfo, src=src) - - elif action == 'download_latest_stable': - if 'koji_build' not in mod.params: - detected_args = ', '.join(mod.params.keys()) - raise exc.TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download_latest_stable' " - "action. Detected arguments: %s" % detected_args) - - name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') - disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) - # we need to do 'fc22' -> 'f22' conversion - tag = disttag.replace('c', '') - - # first we need to check updates tag and if that fails, the latest - # stable nvr is in the base repo - tags = ['%s-updates' % tag, tag] - nvr = self.koji.latest_by_tag(tags, name) - - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( - nvr, target_dir, arch_exclude=arch_exclude, - arches=arches, debuginfo=debuginfo, src=src) - - # download build.log if requested - if build_log: - if action in ('download', 'download_latest_stable'): - ret_log = self.koji.get_build_log( - nvr, target_dir, arches=arches, arch_exclude=arch_exclude) - output_data['downloaded_logs'] = ret_log['ok'] - output_data['log_errors'] = ret_log['error'] - else: - #log.warn("Downloading build logs is not supported for action '%s', ignoring.", - # action) - print("Downloading build logs is not supported for action '%s', ignoring." % action) - - return output_data - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/data/ansible/library/taskotron_koji.py b/data/ansible/library/taskotron_koji.py new file mode 100644 index 0000000..8a43876 --- /dev/null +++ b/data/ansible/library/taskotron_koji.py @@ -0,0 +1,168 @@ +#!/usr/bin/python +# Make coding more python3-ish +from __future__ import (absolute_import, division) +__metaclass__ = type + +import os +import ast + +from ansible.module_utils.basic import AnsibleModule + +try: + import libtaskotron.exceptions as exc + from libtaskotron.ext.fedora.koji_utils import KojiClient + from libtaskotron.ext.fedora import rpm_utils +except ImportError: + libtaskotron_found = False +else: + libtaskotron_found = True + +# these will need to be handled better but this is just a PoC +WORKDIR = os.path.abspath('./taskotron-workdir') + +def main(): + mod = AnsibleModule( + argument_spec=dict( + action=dict(required=True), + arch=dict(required=False, default=['noarch']), + workdir=dict(required=False, default="/tmp/firstmod"), + arch_exclude=dict(required=False), + build_log=dict(required=False, default=False, type="bool"), + debuginfo=dict(required=False, default=False, type="bool"), + koji_build=dict(required=False), + koji_tag=dict(required=False), + src=dict(required=False, default=False, type="bool"), + target_dir=dict(required=False, default='.') + ) + ) + + # TODO: check args for completeness + if not libtaskotron_found: + mod.fail_json(msg="The libtaskotron python module is required") + + try: + kojidirective = KojiDirective() + data = kojidirective.process(mod) + except exc.TaskotronError, e: + mod.fail_json(msg=e) + + subjects = ' '.join(data['downloaded_rpms']) + + mod.exit_json(msg="worky!", changed=True, subjects=subjects) + +class KojiDirective(object): + + def __init__(self, koji_session=None): + super(KojiDirective, self).__init__() + if koji_session is None: + self.koji = KojiClient() + else: + self.koji = koji_session + + def process(self, mod): + # process params + valid_actions = ['download', 'download_tag', 'download_latest_stable'] + action = mod.params['action'] + if action not in valid_actions: + raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' + 'directive' % action) + + # use register to save/return information + if 'target_dir' not in mod.params: + target_dir = WORKDIR + else: + target_dir = mod.params['target_dir'] + + if 'arch' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'arch' as an argument. Detected " + "arguments: %s" % detected_args) + + # this is supposedly safe enough to use on raw input but should be double checked + # http://stackoverflow.com/questions/1894269/convert-string-representation-of-list-to-list-in-python + arches = ast.literal_eval(mod.params['arch']) + if not isinstance(arches, list): + raise exc.TaskotronError("arches must be a list") + + if arches and ('all' not in arches) and ('noarch' not in arches): + arches.append('noarch') + + arch_exclude_string = mod.params.get('arch_exclude', None) + if arch_exclude_string is None: + arch_exclude = [] + else: + arch_exclude = ast.literal_eval(mod.params['arch_exclude']) + + debuginfo = mod.params.get('debuginfo', False) + src = mod.params.get('src', False) + build_log = mod.params.get('build_log', False) + + if not isinstance(arch_exclude, list): + print("arch_exclude: {}".format(type(arch_exclude))) + raise Exception("arch_exclude must be a list") + # download files + output_data = {} + + if action == 'download': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download' " + "action. Detected arguments: %s" % detected_args) + + nvr = rpm_utils.rpmformat(mod.params['koji_build'], 'nvr') + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_tag': + if 'koji_tag' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_tag' for the 'download_tag' " + "action. Detected arguments: %s" % detected_args) + + koji_tag = mod.params['koji_tag'] + + output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( + koji_tag, target_dir, arches=arches, arch_exclude=arch_exclude, + debuginfo=debuginfo, src=src) + + elif action == 'download_latest_stable': + if 'koji_build' not in mod.params: + detected_args = ', '.join(mod.params.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download_latest_stable' " + "action. Detected arguments: %s" % detected_args) + + name = rpm_utils.rpmformat(mod.params['koji_build'], 'n') + disttag = rpm_utils.get_dist_tag(mod.params['koji_build']) + # we need to do 'fc22' -> 'f22' conversion + tag = disttag.replace('c', '') + + # first we need to check updates tag and if that fails, the latest + # stable nvr is in the base repo + tags = ['%s-updates' % tag, tag] + nvr = self.koji.latest_by_tag(tags, name) + + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( + nvr, target_dir, arch_exclude=arch_exclude, + arches=arches, debuginfo=debuginfo, src=src) + + # download build.log if requested + if build_log: + if action in ('download', 'download_latest_stable'): + ret_log = self.koji.get_build_log( + nvr, target_dir, arches=arches, arch_exclude=arch_exclude) + output_data['downloaded_logs'] = ret_log['ok'] + output_data['log_errors'] = ret_log['error'] + else: + #log.warn("Downloading build logs is not supported for action '%s', ignoring.", + # action) + print("Downloading build logs is not supported for action '%s', ignoring." % action) + + return output_data + +if __name__ == '__main__': + main() diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index 645434b..338f07d 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -36,10 +36,10 @@ # item is specified on command line (e.g. koji_build), this module should download # artifacts related to the item and return subject(s) (path strings) for STI tests.yml - name: Download subjects (only rpms atm) - koji: + taskotron_koji: action: "download" koji_build: "{{ taskotron_item }}" - arch: ['x86_64'] #FIXME + arch: ['x86_64', 'noarch'] #FIXME target_dir: "{{ client_taskdir }}" #FIXME? src: False register: koji_output @@ -54,6 +54,9 @@ environment: TEST_SUBJECTS: "{{ koji_output['subjects'] }}" TEST_ARTIFACTS: "{{ artifacts }}" + # Make task output "pretty printed" (structured) + # https://serverfault.com/a/846232 + ANSIBLE_STDOUT_CALLBACK: 'debug' ignore_errors: yes register: output From 650271b74cf0a38427ee523c326f7269da89f127 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 10:35:07 +0000 Subject: [PATCH 21/30] sync runner_generic and runner_si --- diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml index 338f07d..43508e8 100644 --- a/data/ansible/runner_si.yml +++ b/data/ansible/runner_si.yml @@ -19,7 +19,9 @@ dest: "{{ client_taskdir }}" - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" + file: + path: "{{ artifacts }}" + state: directory when: not local - name: Install required packages @@ -46,11 +48,11 @@ - debug: var=koji_output - - name: Run the task + - name: Run tests.yml become: yes become_user: root # FIXME add context tags - command: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" + shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" &> "{{ artifacts }}/ansible.log" environment: TEST_SUBJECTS: "{{ koji_output['subjects'] }}" TEST_ARTIFACTS: "{{ artifacts }}" @@ -58,26 +60,18 @@ # https://serverfault.com/a/846232 ANSIBLE_STDOUT_CALLBACK: 'debug' ignore_errors: yes - register: output - - - name: Dump task output - debug: var=output.stdout_lines - - - name: Save ansible.log - copy: - dest: "{{ artifacts }}/ansible.log" - content: "{{ output.stdout }}" + register: task - name: RC of test - shell: echo "{{ output.rc }}" > "{{ artifacts }}/test.rc" + shell: echo "{{ task.rc }}" > "{{ artifacts }}/test.rc" - name: Set outcome to PASSED if RC == 0 set_fact: outcome="PASSED" - when: output.rc == 0 + when: task.rc == 0 - name: Set outcome to FAILED if RC != 0 set_fact: outcome="FAILED" - when: output.rc != 0 + when: task.rc != 0 - name: Generate ResultsDB result file # FIXME: type of result From ac0f745104902db76c13c507b4ab3f06fb0a21ef Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 11:01:59 +0000 Subject: [PATCH 22/30] have a single runner.yml playbook Include either generic or SI tasks. --- diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml new file mode 100644 index 0000000..2358fc4 --- /dev/null +++ b/data/ansible/runner.yml @@ -0,0 +1,52 @@ +- hosts: all + remote_user: root + vars: + local: false + tasks: + - name: Clean client taskdir + file: + path: "{{ client_taskdir }}" + state: absent + + - name: Create client taskdir + file: + path: "{{ client_taskdir }}" + state: directory + + - name: Upload taskdir to client taskdir + synchronize: + src: "{{ taskdir }}/" + dest: "{{ client_taskdir }}" + + - name: create artifacts dir + file: + path: "{{ artifacts }}" + state: directory + when: not local + + - name: Install required packages + dnf: + name: "{{ item }}" + state: present + with_items: + - ansible + - libselinux-python + - standard-test-roles + - python2-dnf + when: not local + + - name: include either generic or SI execution tasks + include: "{{ exec_tasks }}" + + - name: Collect logs + synchronize: + mode: pull + src: "{{ artifacts }}/*" + dest: "{{ artifacts }}" + when: not local + + - name: Warn about failed task + debug: + msg: The executed task failed. Inspect 'ansible.log' and 'test.log' in + the artifacts directory to learn more. + when: task.rc != 0 diff --git a/data/ansible/runner_generic.yml b/data/ansible/runner_generic.yml deleted file mode 100644 index 184491a..0000000 --- a/data/ansible/runner_generic.yml +++ /dev/null @@ -1,61 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - file: - path: "{{ artifacts }}" - state: directory - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - - name: Run tests.yml - become: yes - become_user: root - shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item="{{ taskotron_item }}" &> "{{ artifacts }}/ansible.log" - environment: - TEST_ARTIFACTS: "{{ artifacts }}" - # Make task output "pretty printed" (structured) - # https://serverfault.com/a/846232 - ANSIBLE_STDOUT_CALLBACK: 'debug' - ignore_errors: yes - register: task - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local - - - name: Warn about failed task - debug: - msg: The executed task failed. Inspect 'ansible.log' and 'test.log' in - the artifacts directory to learn more. - when: task.rc != 0 diff --git a/data/ansible/runner_si.yml b/data/ansible/runner_si.yml deleted file mode 100644 index 43508e8..0000000 --- a/data/ansible/runner_si.yml +++ /dev/null @@ -1,88 +0,0 @@ -- hosts: all - remote_user: root - vars: - local: false - tasks: - - name: Clean client taskdir - file: - path: "{{ client_taskdir }}" - state: absent - - - name: Create client taskdir - file: - path: "{{ client_taskdir }}" - state: directory - - - name: Upload taskdir to client taskdir - synchronize: - src: "{{ taskdir }}/" - dest: "{{ client_taskdir }}" - - - name: create artifacts dir - file: - path: "{{ artifacts }}" - state: directory - when: not local - - - name: Install required packages - dnf: - name: "{{ item }}" - state: present - with_items: - - ansible - - libselinux-python - - standard-test-roles - - python2-dnf - when: not local - - # item is specified on command line (e.g. koji_build), this module should download - # artifacts related to the item and return subject(s) (path strings) for STI tests.yml - - name: Download subjects (only rpms atm) - taskotron_koji: - action: "download" - koji_build: "{{ taskotron_item }}" - arch: ['x86_64', 'noarch'] #FIXME - target_dir: "{{ client_taskdir }}" #FIXME? - src: False - register: koji_output - - - debug: var=koji_output - - - name: Run tests.yml - become: yes - become_user: root - # FIXME add context tags - shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" &> "{{ artifacts }}/ansible.log" - environment: - TEST_SUBJECTS: "{{ koji_output['subjects'] }}" - TEST_ARTIFACTS: "{{ artifacts }}" - # Make task output "pretty printed" (structured) - # https://serverfault.com/a/846232 - ANSIBLE_STDOUT_CALLBACK: 'debug' - ignore_errors: yes - register: task - - - name: RC of test - shell: echo "{{ task.rc }}" > "{{ artifacts }}/test.rc" - - - name: Set outcome to PASSED if RC == 0 - set_fact: outcome="PASSED" - when: task.rc == 0 - - - name: Set outcome to FAILED if RC != 0 - set_fact: outcome="FAILED" - when: task.rc != 0 - - - name: Generate ResultsDB result file - # FIXME: type of result - # FIXME: checkname - shell: taskotron_result -f "{{ artifacts }}/taskotron_results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" - args: - creates: "{{ artifacts }}/taskotron_results.yml" - - - name: Collect logs - synchronize: - mode: pull - src: "{{ artifacts }}/*" - dest: "{{ artifacts }}" - when: not local diff --git a/data/ansible/tasks_generic.yml b/data/ansible/tasks_generic.yml new file mode 100644 index 0000000..ffdd4e3 --- /dev/null +++ b/data/ansible/tasks_generic.yml @@ -0,0 +1,11 @@ +- name: Run tests.yml + become: yes + become_user: root + shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item="{{ taskotron_item }}" &> "{{ artifacts }}/ansible.log" + environment: + TEST_ARTIFACTS: "{{ artifacts }}" + # Make task output "pretty printed" (structured) + # https://serverfault.com/a/846232 + ANSIBLE_STDOUT_CALLBACK: 'debug' + ignore_errors: yes + register: task diff --git a/data/ansible/tasks_si.yml b/data/ansible/tasks_si.yml new file mode 100644 index 0000000..8ac42f4 --- /dev/null +++ b/data/ansible/tasks_si.yml @@ -0,0 +1,44 @@ + # item is specified on command line (e.g. koji_build), this module should download + # artifacts related to the item and return subject(s) (path strings) for STI tests.yml +- name: Download subjects (only rpms atm) + taskotron_koji: + action: "download" + koji_build: "{{ taskotron_item }}" + arch: ['x86_64', 'noarch'] #FIXME + target_dir: "{{ client_taskdir }}" #FIXME? + src: False + register: koji_output + +- debug: var=koji_output + +- name: Run tests.yml + become: yes + become_user: root + # FIXME add context tags + shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" &> "{{ artifacts }}/ansible.log" + environment: + TEST_SUBJECTS: "{{ koji_output['subjects'] }}" + TEST_ARTIFACTS: "{{ artifacts }}" + # Make task output "pretty printed" (structured) + # https://serverfault.com/a/846232 + ANSIBLE_STDOUT_CALLBACK: 'debug' + ignore_errors: yes + register: task + +- name: RC of test + shell: echo "{{ task.rc }}" > "{{ artifacts }}/test.rc" + +- name: Set outcome to PASSED if RC == 0 + set_fact: outcome="PASSED" + when: task.rc == 0 + +- name: Set outcome to FAILED if RC != 0 + set_fact: outcome="FAILED" + when: task.rc != 0 + +- name: Generate ResultsDB result file + # FIXME: type of result + # FIXME: checkname + shell: taskotron_result -f "{{ artifacts }}/taskotron_results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" + args: + creates: "{{ artifacts }}/taskotron_results.yml" diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 949c673..deaa351 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -111,18 +111,19 @@ class Executor(object): with open(os.path.join(self.arg_data['taskdir'], 'tests.yml'), 'r') as playbook_file: playbook = yaml.load(playbook_file.read())[0] #FIXME document [0] if playbook.get('vars',{}).get('taskotron_generic_task', False): - runner_playbook = 'runner_generic.yml' + exec_tasks = 'tasks_generic.yml' else: - runner_playbook = 'runner_si.yml' + exec_tasks = 'tasks_si.yml' cmd = [ - 'ansible-playbook', runner_playbook, + 'ansible-playbook', 'runner.yml', '--inventory=%s,' % ipaddr, # the ending comma is important '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'taskotron_item=%s' % self.arg_data['item'], '-e', 'taskdir=%s' % self.arg_data['taskdir'], '-e', 'client_taskdir=%s' % config.get_config().client_taskdir, '-e', 'sti_inventory=%s' % sti_inventory, + '-e', 'exec_tasks=%s' % exec_tasks, ] if self.run_remotely: From 75b11d2dfa0b65230f756f51ca2302299e58502f Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 12:24:46 +0000 Subject: [PATCH 23/30] add "what changed" info to readme --- diff --git a/README.rst b/README.rst index 001a9da..484c498 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,81 @@ Please direct questions and comments to either *#fedora-qa* on freenode or the __ Taskotron_ +Running ansiblized (standard interface) tasks +============================================= + +This is a short documentation of most important changes related to the switch +of libtaskotron runner to the ansiblized version of it and its support of test +`standard interface`_: + +* The basic command line is now:: + + runtask --item ITEM --type TYPE DIRECTORY + + where ``DIRECTORY`` contains ``tests.yml`` (as defined in SI_). + +* The tasks expect to be run as ``root`` and can make arbitrary changes to the + system. Therefore you should never run it locally on your production machine, + but instead use ``--ssh`` or ``--libvirt`` command line option. + + When you use ``--ssh``, you need to have an existing system running with + password-less ssh login configured and `required packages + `_ installed. + + When you use ``--libvirt``, you need to create an OS image using + `taskotron_cloud + `_ + kickstart and set ``imageurl=file:///path/to/image.qcow2`` in + ``/etc/taskotron/taskotron.yml``. + + The system version must always match the ``ITEM`` version tested. + +* There are two types of tasks: *SI_* and *generic*. + + *Generic* tasks are SI tests extended with Taskotron-specific functionality. + They need to know extra input information and they want to generate custom + results to be sent to ResultsDB. These tasks are e.g. task-rpmlint, + task-rpmdeplint, task-upgradepath, etc. + + *SI_* tasks are stored in distgit and their purpose is to test just a single + specific package (or module). So e.g. gzip test suite, firefox test suite, + etc. + +* All tasks are considered *SI* by default. If the tasks wants to be consider + a *generic* task, it needs to include ``taskotron_generic_task: true`` + variable in the first play of ``tests.yml`` playbook. + + Generic tasks do not receive ``subjects`` variable as defined by the SI_, + but they receive ``taskotron_item`` variable instead. The item/subject is + not downloaded and installed automatically as mandated by SI, and their + exit code is not used for generating a ResultsDB entry automatically. + Instead, they're expected to create ``{{artifacts}}/taskotron_results.yml`` + file in ResultYAML_ format themselves. + +* After execution, all important files are expected to be in the + ``{{artifacts}}`` directory (printed by libtaskotron at the end of the + execution). At minimum the ``ansible.log`` and ``test.log`` should be there + according to the SI_. + +* The format of ResultYAML_ slightly changed. ``checkname`` is no longer just + the task name part with namespace left out (e.g. just ``rpmlint`` + without ``dist.``), but the full testcase name (e.g. ``dist.rpmlint``). + +A simple example for running an SI_ task is:: + + git clone https://upstreamfirst.fedorainfracloud.org/gzip.git + runtask --item gzip-1.8-1.fc25 --type koji_build gzip/ # add --ssh or --libvirt + +A simple example for running a generic task is:: + + git clone https://pagure.io/taskotron/task-rpmlint.git --branch feature/ansiblize + runtask --item htop-2.0.2-1.fc25 --type koji_build task-rpmlint/ # add --ssh or --libvirt + +.. _standard interface: https://fedoraproject.org/wiki/Changes/InvokingTests +.. _SI: `standard interface`_ +.. _ResultsYAML: https://qa.fedoraproject.org/docs/libtaskotron/latest/resultyaml.html + + Installing a Development Environment ==================================== From 850209ea54f43fd3b3f893ae49eb2dd962663ea9 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 12:28:28 +0000 Subject: [PATCH 24/30] fix formatting --- diff --git a/README.rst b/README.rst index 484c498..6a05e14 100644 --- a/README.rst +++ b/README.rst @@ -49,17 +49,17 @@ of libtaskotron runner to the ansiblized version of it and its support of test The system version must always match the ``ITEM`` version tested. -* There are two types of tasks: *SI_* and *generic*. +* There are two types of tasks: SI_ and *generic*. + + *SI* tasks are stored in distgit and their purpose is to test just a single + specific package (or module). So e.g. gzip test suite, firefox test suite, + etc. *Generic* tasks are SI tests extended with Taskotron-specific functionality. They need to know extra input information and they want to generate custom results to be sent to ResultsDB. These tasks are e.g. task-rpmlint, task-rpmdeplint, task-upgradepath, etc. - *SI_* tasks are stored in distgit and their purpose is to test just a single - specific package (or module). So e.g. gzip test suite, firefox test suite, - etc. - * All tasks are considered *SI* by default. If the tasks wants to be consider a *generic* task, it needs to include ``taskotron_generic_task: true`` variable in the first play of ``tests.yml`` playbook. From 11ee42662da810be1d040bd13dc26a5068520dbb Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 13:22:10 +0000 Subject: [PATCH 25/30] simplify setting outcome from exit code --- diff --git a/data/ansible/tasks_si.yml b/data/ansible/tasks_si.yml index 8ac42f4..26f36df 100644 --- a/data/ansible/tasks_si.yml +++ b/data/ansible/tasks_si.yml @@ -25,16 +25,11 @@ ignore_errors: yes register: task -- name: RC of test +- name: Save exit code shell: echo "{{ task.rc }}" > "{{ artifacts }}/test.rc" -- name: Set outcome to PASSED if RC == 0 - set_fact: outcome="PASSED" - when: task.rc == 0 - -- name: Set outcome to FAILED if RC != 0 - set_fact: outcome="FAILED" - when: task.rc != 0 +- name: Set outcome based on exit code + set_fact: outcome={{ (task.rc == 0) | ternary('PASSED', 'FAILED') }} - name: Generate ResultsDB result file # FIXME: type of result From b1807a543f135ed0a57c87ecf4ed0aec8d3a47f0 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 13 2017 13:44:25 +0000 Subject: [PATCH 26/30] use {{artifacts}}/taskotron subdir for our files --- diff --git a/README.rst b/README.rst index 6a05e14..fd69f2a 100644 --- a/README.rst +++ b/README.rst @@ -68,7 +68,7 @@ of libtaskotron runner to the ansiblized version of it and its support of test but they receive ``taskotron_item`` variable instead. The item/subject is not downloaded and installed automatically as mandated by SI, and their exit code is not used for generating a ResultsDB entry automatically. - Instead, they're expected to create ``{{artifacts}}/taskotron_results.yml`` + Instead, they're expected to create ``{{artifacts}}/taskotron/results.yml`` file in ResultYAML_ format themselves. * After execution, all important files are expected to be in the diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml index 2358fc4..e229a62 100644 --- a/data/ansible/runner.yml +++ b/data/ansible/runner.yml @@ -18,12 +18,18 @@ src: "{{ taskdir }}/" dest: "{{ client_taskdir }}" - - name: create artifacts dir + - name: Create artifacts dir file: path: "{{ artifacts }}" state: directory when: not local + - name: Create artifacts/taskotron subdir + file: + path: "{{ artifacts }}/taskotron" + state: directory + when: not local + - name: Install required packages dnf: name: "{{ item }}" @@ -35,7 +41,7 @@ - python2-dnf when: not local - - name: include either generic or SI execution tasks + - name: Include either generic or SI execution tasks include: "{{ exec_tasks }}" - name: Collect logs diff --git a/data/ansible/tasks_si.yml b/data/ansible/tasks_si.yml index 26f36df..f9bf2be 100644 --- a/data/ansible/tasks_si.yml +++ b/data/ansible/tasks_si.yml @@ -26,7 +26,9 @@ register: task - name: Save exit code - shell: echo "{{ task.rc }}" > "{{ artifacts }}/test.rc" + copy: + content: "{{ task.rc }}" + dest: "{{ artifacts }}/taskotron/test.rc" - name: Set outcome based on exit code set_fact: outcome={{ (task.rc == 0) | ternary('PASSED', 'FAILED') }} @@ -34,6 +36,6 @@ - name: Generate ResultsDB result file # FIXME: type of result # FIXME: checkname - shell: taskotron_result -f "{{ artifacts }}/taskotron_results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" + shell: taskotron_result -f "{{ artifacts }}/taskotron/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" args: - creates: "{{ artifacts }}/taskotron_results.yml" + creates: "{{ artifacts }}/taskotron/results.yml" diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index deaa351..336a738 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -144,7 +144,8 @@ class Executor(object): raise exc.TaskotronError(e.output) def _report_results(self): - results_file = os.path.join(self.arg_data['artifactsdir'], 'taskotron_results.yml') + results_file = os.path.join(self.arg_data['artifactsdir'], 'taskotron', + 'results.yml') if os.path.exists(results_file): rdb = resultsdb_directive.ResultsdbDirective() rdb.process(params={"file": results_file}, arg_data=self.arg_data) From 968cbe5da07eebb98a382b5634b29d85e4ee9e07 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Sep 13 2017 14:11:06 +0000 Subject: [PATCH 27/30] readability fix: long line folding --- diff --git a/data/ansible/tasks_generic.yml b/data/ansible/tasks_generic.yml index ffdd4e3..ec91224 100644 --- a/data/ansible/tasks_generic.yml +++ b/data/ansible/tasks_generic.yml @@ -1,7 +1,13 @@ - name: Run tests.yml become: yes become_user: root - shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory=localhost, --connection=local -e artifacts="{{ artifacts }}" -e taskotron_item="{{ taskotron_item }}" &> "{{ artifacts }}/ansible.log" + shell: > + ansible-playbook "{{ client_taskdir }}/tests.yml" + --inventory=localhost, + --connection=local + -e artifacts="{{ artifacts }}" + -e taskotron_item="{{ taskotron_item }}" + &> "{{ artifacts }}/ansible.log" environment: TEST_ARTIFACTS: "{{ artifacts }}" # Make task output "pretty printed" (structured) diff --git a/data/ansible/tasks_si.yml b/data/ansible/tasks_si.yml index f9bf2be..33f7412 100644 --- a/data/ansible/tasks_si.yml +++ b/data/ansible/tasks_si.yml @@ -15,7 +15,13 @@ become: yes become_user: root # FIXME add context tags - shell: ansible-playbook "{{ client_taskdir }}/tests.yml" --inventory="{{ sti_inventory }}" --connection=local -e artifacts="{{ artifacts }}" -e subjects="{{ koji_output['subjects'] }}" &> "{{ artifacts }}/ansible.log" + shell: > + ansible-playbook "{{ client_taskdir }}/tests.yml" + --inventory="{{ sti_inventory }}" + --connection=local + -e artifacts="{{ artifacts }}" + -e subjects="{{ koji_output['subjects'] }}" + &> "{{ artifacts }}/ansible.log" environment: TEST_SUBJECTS: "{{ koji_output['subjects'] }}" TEST_ARTIFACTS: "{{ artifacts }}" @@ -36,6 +42,13 @@ - name: Generate ResultsDB result file # FIXME: type of result # FIXME: checkname - shell: taskotron_result -f "{{ artifacts }}/taskotron/results.yml" -i "{{ taskotron_item }}" -o "{{ outcome }}" -t koji_build -a "{{ artifacts }}/test.log" -c "pkg.{{ taskotron_item }}" + shell: > + taskotron_result + -f "{{ artifacts }}/taskotron/results.yml" + -i "{{ taskotron_item }}" + -o "{{ outcome }}" + -t koji_build + -a "{{ artifacts }}/test.log" + -c "pkg.{{ taskotron_item }}" args: creates: "{{ artifacts }}/taskotron/results.yml" From 82de0386efb4bc9fcce457d530ccd927d774e484 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 14 2017 09:01:43 +0000 Subject: [PATCH 28/30] add copr link --- diff --git a/README.rst b/README.rst index fd69f2a..c1074a8 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,10 @@ __ Taskotron_ Running ansiblized (standard interface) tasks ============================================= -This is a short documentation of most important changes related to the switch +A COPR repo for ansiblized libtaskotron package is available at: +https://copr.fedorainfracloud.org/coprs/kparal/taskotron-ansiblize/ + +Here is a short documentation of most important changes related to the switch of libtaskotron runner to the ansiblized version of it and its support of test `standard interface`_: @@ -90,6 +93,12 @@ A simple example for running a generic task is:: git clone https://pagure.io/taskotron/task-rpmlint.git --branch feature/ansiblize runtask --item htop-2.0.2-1.fc25 --type koji_build task-rpmlint/ # add --ssh or --libvirt +So far the following generic tasks have been converted to the ansiblized +libtaskotron version (look at the ``feature/ansiblized`` branch): + +* task-rpmlint + + .. _standard interface: https://fedoraproject.org/wiki/Changes/InvokingTests .. _SI: `standard interface`_ .. _ResultsYAML: https://qa.fedoraproject.org/docs/libtaskotron/latest/resultyaml.html diff --git a/libtaskotron.spec b/libtaskotron.spec index aaf4d64..ccd0015 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -1,7 +1,7 @@ Name: libtaskotron # NOTE: if you update version, *make sure* to also update `libtaskotron/__init__.py` Version: 0.4.99.1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Taskotron Support Library License: GPLv3 From 687aa1cc914c8b1362b4141e1889193987ac8c16 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Sep 14 2017 12:53:43 +0000 Subject: [PATCH 29/30] readme: fix branch name --- diff --git a/README.rst b/README.rst index c1074a8..09b3676 100644 --- a/README.rst +++ b/README.rst @@ -94,7 +94,7 @@ A simple example for running a generic task is:: runtask --item htop-2.0.2-1.fc25 --type koji_build task-rpmlint/ # add --ssh or --libvirt So far the following generic tasks have been converted to the ansiblized -libtaskotron version (look at the ``feature/ansiblized`` branch): +libtaskotron version (look at the ``feature/ansiblize`` branch): * task-rpmlint From a7ada60bb5c993f5395cd8cc6ddd836b279d748e Mon Sep 17 00:00:00 2001 From: Lili Nie Date: Sep 15 2017 07:17:43 +0000 Subject: [PATCH 30/30] enable docker deployment --- diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml index e229a62..b04164f 100644 --- a/data/ansible/runner.yml +++ b/data/ansible/runner.yml @@ -1,17 +1,20 @@ -- hosts: all +- hosts: "{{ ip_select }}" remote_user: root vars: local: false + docker: "{{ docker }}" tasks: - name: Clean client taskdir file: path: "{{ client_taskdir }}" state: absent + when: not docker - name: Create client taskdir file: path: "{{ client_taskdir }}" state: directory + when: not docker - name: Upload taskdir to client taskdir synchronize: diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 336a738..295b9f4 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -2,6 +2,8 @@ # Copyright 2009-2017, Red Hat, Inc. # License: GPL-2.0+ # See the LICENSE file for more details on Licensing +# Authors: +# Mike Ruckman from __future__ import absolute_import @@ -36,6 +38,7 @@ class Executor(object): def __init__(self, arg_data): self.arg_data = arg_data self.task_vm = None + self.task_container = None self.run_remotely = False def _spawn_vm(self, uuid): @@ -56,6 +59,19 @@ class Executor(object): self.task_vm.ipaddr) return self.task_vm.ipaddr + + def _spawn_container(self): + + taskdir = os.path.dirname(self.arg_data['task']) + docker_task = os.path.abspath(taskdir) + uuid = self.arg_data['uuid'] + container_name = 'taskotron-worker'+ str(uuid) + self.task_container = docker.DockerClient(name=container_name) + self.task_container.create_container(artifacts=self.arg_data['artifactsdir'], + docker_taskdir=docker_task) + port = self.task_container.port + log.info('Running task on a container on port {}.'.format(port)) + return port def _get_client_ipaddr(self): '''Get an IP address of the machine the task is going to be executed on. @@ -87,7 +103,14 @@ class Executor(object): self.run_remotely = True persistent = True + elif self.arg_data['docker']: + self.run_remotely = True + persistent = False + docker = True + log.debug('Forcing execution on docker (option --docker)') + log.debug('Execution mode: %s', 'remote' if self.run_remotely else 'local') + ipaddr = '127.0.0.1' if self.run_remotely: @@ -95,7 +118,7 @@ class Executor(object): return ipaddr - def _run_ansible_playbook(self, ipaddr): + def _run_ansible_playbook(self, ipaddr, d_select): '''Run the ansible-playbook command to execute given playbook containing the task. :param str ipaddr: IP address of the machine the task will be run on @@ -114,10 +137,10 @@ class Executor(object): exec_tasks = 'tasks_generic.yml' else: exec_tasks = 'tasks_si.yml' + cmd = [ 'ansible-playbook', 'runner.yml', - '--inventory=%s,' % ipaddr, # the ending comma is important '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'taskotron_item=%s' % self.arg_data['item'], '-e', 'taskdir=%s' % self.arg_data['taskdir'], @@ -126,11 +149,20 @@ class Executor(object): '-e', 'exec_tasks=%s' % exec_tasks, ] + taskdir = os.path.dirname(self.arg_data['task']) + docker_task = os.path.abspath(taskdir) + if self.run_remotely: if self.arg_data['ssh_privkey']: cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) else: cmd.extend(['--become', '--connection=local', '-e', 'local=true']) + if self.arg_data['docker']: + cmd.extend(['-i','dockerinventory','-e','ip_select=%s'%d_select,'-e','docker=True','-e', 'client_taskdir=%s' % docker_task]) + else: + cmd.extend(['--inventory=%s,' % ipaddr,'-e','ip_select=all']) + cmd.extend(['-e', 'client_taskdir=%s' % config.get_config().client_taskdir]) + if self.arg_data['debug']: cmd.extend(['-vv']) @@ -157,11 +189,19 @@ class Executor(object): ipaddr = self._get_client_ipaddr() if ipaddr is None: ipaddr = self._spawn_vm(self.arg_data['uuid']) + # select right docker host from the list in inventory + d_select = None + if self.arg_data['docker']: + port = self._spawn_container() + d_select='task'+str(port) + with open(os.path.join('dockerinventory'), 'a') as f: + f.write('[%s]\n\t\t %s ansible_user=root ansible_port=%s ansible_password=passw0rd ' + 'ansible_ssh_common_args="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"\n'%(d_select,ipaddr,port)) log.info('Running task on machine %s', ipaddr) try: - self._run_ansible_playbook(ipaddr) + self._run_ansible_playbook(ipaddr,d_select) self._report_results() finally: if self.task_vm is not None: diff --git a/libtaskotron/ext/disposable/docker.py b/libtaskotron/ext/disposable/docker.py new file mode 100644 index 0000000..fb74399 --- /dev/null +++ b/libtaskotron/ext/disposable/docker.py @@ -0,0 +1,212 @@ +import os +import random +import subprocess as sp + +from libtaskotron.logger import log + + +def get_images(): + '''Return a list of image dicts.''' + + template = "{{.ID}}\t{{.Repository}}" + raw_output = sp.check_output(['docker', + 'images', + '--format', + template]) + if raw_output: + images = raw_output.split('\n') + attrs = ['id', 'name'] + images = [image.split('\t') for image in images[:-1]] + images = [dict(zip(attrs, image)) for image in images] + return images + +def _check_mounts(container_name): + '''check if the existing container is created by taskotron ''' + + raw_output = sp.check_output(['docker', + 'inspect', + '--format', + "{{.HostConfig.Binds}}", + container_name]) + mounts = raw_output.split(':')[:-1] + for check in mounts: + base_dir = check.split('/')[-1] + if container_name.endswith(base_dir): + return True + +def get_containers(): + '''Return a list of all containers on the host.''' + + # This is a go template to clean up docker ps output + template = "{{.ID}}\t{{.Command}}\t{{.Status}}\t{{.Ports}}\t{{.Names}}" + + raw_output = sp.check_output(['docker', + 'ps', + '-a', + '--format', + template]) + + containers = raw_output.split('\n') + + # Clean up the output a bit, also remove the last, empty, entry + containers = [x.split('\t') for x in containers][:-1] + + attrs = ['id', 'command', 'status', 'ports', 'names'] + containers = [dict(zip(attrs, container)) for container in containers] + + return containers + + +def find_container(search_term): + '''Find a container based on either name or id.''' + containers = get_containers() + for container in containers: + if search_term in container.values(): + return container + +def find_client(name): + '''Find a container based on either name or id. + Return DockerClient for the potential use of class methods''' + container = find_container(name) + if container: + if _check_mounts(name): + return DockerClient(name=name,port=container['ports']) + return None + + + +class DockerClient(object): + '''Helper class for working with the docker daemon.''' + + def __init__(self, host_ip="127.0.0.1", name=None,port=None,image_name='test'): + ''':param host_ip: The host IP where the docker daemon runs. We + assume for now that it's running on localhost.''' + + # FIXME: Support running commands against a remote host. + self.host = host_ip + # We need to keep a list of the containers we create, so + # we can co-exist with other users of whatever docker + # daemon we might be interacting with. + self.name = name + self.port = port or random.randrange(2200, 2300) + self.image_name = image_name + self.owned_containers = [] + self.owned_images = [] + + def build_image(self, path_to_dockerfile_dir): + '''Run `docker build Dockerfile` against the specified path.''' + sp.check_call(['docker', + 'build', + '-t', + self.image_name, + path_to_dockerfile_dir]) + + + def _manipulate_container(self,action=None): + '''Start or stop a container based on either it's name or id. + + :param action: command for docker, either 'start', 'stop', or 'rm' + :param identifier: a container ID or Name''' + + if action not in ['start', 'stop', 'rm']: + raise ClientError("Available actions are: start, stop, and rm. " + "Could not perform: {}".format(action)) + + container = find_container(self.name) + if container: + try: + sp.check_call(['docker', + action, + self.name]) + + except Exception as e: + log.debug("Failed to {} the container '{}'.".format(action, container['names'])) + raise ClientError("Could not %s the container:%s."%(action,e)) + else: + log.debug("No container %s found ."%identifier) + + + def _check_port_idle(self): + + with open (os.path.join('port'),'w') as f: + f.write(':%s'%self.port) + f.close() + # check if the port is being used to avoid port bind failure + port_idle=sp.call('netstat -anp|grep -f port',shell=True) + return port_idle + + def create_container(self,artifacts=None,docker_taskdir=None): + '''Method to create a container from a specified image. + :param image_name: the name of the built image you want to launch. + ''' + + # Build the image if it's not already there + image_found = False + images = get_images() + for image in images: + if self.image_name == image['name']: + log.debug("Image already exists, skipping image build...") + image_found = True + + if not image_found: + log.debug("Image not found, building...") + self.build_image(os.path.abspath(os.curdir) + '/libtaskotron/ext/docker/') + + while not self._check_port_idle(): + self.port = random.randrange(2200, 2300) + host_port=self.port + + log.info("Container will be available on port: %s"%host_port) + # set SELinux to permissive to avoid volume permission error + sp.call("setenforce 0",shell=True) + sp.check_call(['docker', + 'run', + '-d', + '--name', self.name, + '-p', '%s:22'%host_port, + '-v', '%s:%s'%(artifacts,artifacts), + '-v', '%s:%s'%(docker_taskdir,docker_taskdir), + self.image_name]) + + # Update the list of owned_containers + self.owned_containers.append(find_container(self.name)) + + + def start_container(self): + self._manipulate_container(action='start') + + def stop_container(self): + self._manipulate_container(action='stop') + + def rm_container(self, autostop=False): + # Update our list of owned containers, since we're deleting it + container = find_container(self.name) + if container['status'].startswith('Up'): + if autostop: + self._manipulate_container(action='stop') + log.debug("container %s is running,stopping it"%container['names']) + else: + raise ClientError('container is running,stop it first') + for i in xrange(len(self.owned_containers)): + if self.owned_containers[i]['names'] or self.owned_containers[i]['id'] == identifier: + self.owned_containers.pop(i) + break + + self._manipulate_container(action='rm') + + def clear_all(self): + '''Clean out all owned containers and images.''' + + log.info("Clearing out all our containers...") + for container in self.owned_containers: + self.stop_container(container['names']) + self.rm_container(container['names']) + + log.info("Clearing out all our images...") + for image in self.owned_images: + sp.check_call(['docker', + 'rmi', + image]) +class ClientError(Exception): + '''General exceptions we might run into using this client.''' + pass diff --git a/libtaskotron/ext/docker/Dockerfile b/libtaskotron/ext/docker/Dockerfile new file mode 100644 index 0000000..9dccdbc --- /dev/null +++ b/libtaskotron/ext/docker/Dockerfile @@ -0,0 +1,22 @@ +#FROM registry.fedoraproject.org/fedora:25 +FROM fedora:25 + +RUN dnf install openssh-server PyYAML libtaskotron-core libtaskotron-fedora \ + libtaskotron-config python-solv python-librepo passwd -y && \ + mkdir -p /var/log/taskotron /srv/taskotron/artifacts && \ + dnf clean all + +ENV LANG C.utf8 +ENV LC_ALL C.utf8 + +ADD https://pagure.io/taskotron/libtaskotron/blob/master/f/conf/yumrepoinfo.conf.example /etc/taskotron/yumrepoinfo.conf +ADD taskotron.yaml /etc/taskotron/taskotron.yaml +ADD namespaces.yaml /etc/taskotron/namespaces.yaml + +RUN echo "passw0rd" | passwd --stdin root +RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N "" + +EXPOSE 22 + +CMD ["/sbin/sshd", "-D"] + diff --git a/libtaskotron/main.py b/libtaskotron/main.py index fbc9ad0..a43be01 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -98,6 +98,8 @@ def get_argparser(): help="path to private key for remote connections over ssh") parser.add_argument("--no-destroy", action="store_true", help="do not destroy disposable client at the end of task execution") + parser.add_argument("--docker", action="store_true", + help="Run the task inside a docker container.") return parser diff --git a/testing/test_docker.py b/testing/test_docker.py new file mode 100644 index 0000000..208fd96 --- /dev/null +++ b/testing/test_docker.py @@ -0,0 +1,84 @@ +import pytest +import mock +from mock import Mock, MagicMock +from dingus import Dingus +import subprocess as sp +from libtaskotron.ext.disposable import docker + +class TestDocker(object): + def setup_method(self, method): + self.name = 'test' + + def should_raise_error(self, monkeypatch): + mock_container = Mock() + mock_container.side_effect=docker.ClientError() + monkeypatch.setattr(docker.DockerClient, '_manipulate_container', mock_container) + test_docker = docker.DockerClient(name='test') + with pytest.raises(docker.ClientError): + test_docker.stop_container() + + def should_behave_on_success(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = True + monkeypatch.setattr(docker.DockerClient, '_manipulate_container', mock_container) + test_docker = docker.DockerClient(name='test') + assert test_docker.stop_container() == None + + def test_find_non_container(self, monkeypatch): + mock_container = mock.Mock() + mock_container.return_value = [] + monkeypatch.setattr(docker, 'get_containers', mock_container) + + test_docker = docker.find_container(self.name) + + assert test_docker is None + + def test_find_client(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = {'names': 'taskotron-worker', 'ports': '2272'} + monkeypatch.setattr(docker, 'find_container', mock_container) + mock_check = Mock() + mock_check.return_value = True + monkeypatch.setattr(docker, '_check_mounts', mock_container) + assert docker.find_client('test') != None + + def test_none_client(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = False + monkeypatch.setattr(docker, 'find_container', mock_container) + mock_check = Mock() + mock_check.return_value = True + monkeypatch.setattr(docker, '_check_mounts', mock_container) + assert docker.find_client('test') == None + + def test_check_busy_port(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = False + monkeypatch.setattr(sp, 'call', mock_container) + test_docker = docker.DockerClient('test') + assert test_docker._check_port_idle() == False + + def test_check_idle_port(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = True + monkeypatch.setattr(sp, 'call', mock_container) + test_docker = docker.DockerClient('test') + assert test_docker._check_port_idle() == True + + def test_get_images(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = '32fb40763c6b\ttest\n' + monkeypatch.setattr(sp, 'check_output', mock_container) + assert docker.get_images() == [{'id': '32fb40763c6b', 'name': 'test'}] + + def test_no_images(self, monkeypatch): + mock_container = Mock() + mock_container.return_value = None + monkeypatch.setattr(sp, 'check_output', mock_container) + assert docker.get_images() == None + + + + + +