From 81f65eecb111de36d77b980ae2b7c0c1cd6d7f08 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Jun 20 2017 09:52:45 +0000 Subject: [PATCH 1/5] Merge branch 'develop' into feature/ansiblize --- diff --git a/docs/generate_directive_docs.py b/docs/generate_directive_docs.py index 9329c24..494b381 100644 --- a/docs/generate_directive_docs.py +++ b/docs/generate_directive_docs.py @@ -345,7 +345,7 @@ def process_category(category, categories, options, env, template, outputname): links_list = [] for module in modules: - if 'pyc' in module: + if not module.endswith('.py'): continue result = process_module(module, diff --git a/docs/source/resultyaml.rst b/docs/source/resultyaml.rst index b1ae880..86ee486 100644 --- a/docs/source/resultyaml.rst +++ b/docs/source/resultyaml.rst @@ -110,7 +110,7 @@ the per-check results separately (e.g. ``repo-sanity`` could output ``depcheck``, ``upgradepath`` and ``repoclosure`` results in one run). ``note`` adds a short valuable information that should be easily visible in the - result overview. +result overview. ``arch`` and ``package`` are custom fields that do not have any special meaning for the Taskotron tooling, but provide additional information about the check diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index 52a5601..d3266d1 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -103,7 +103,6 @@ from libtaskotron.directives import BaseDirective from libtaskotron import check from libtaskotron import config -from libtaskotron import buildbot_utils from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError from libtaskotron.logger import log from libtaskotron.ext.fedora import rpm_utils @@ -279,17 +278,13 @@ class ResultsdbDirective(BaseDirective): return check.export_YAML(check_details) checkname = '%s.%s' % (arg_data['namespace'], arg_data['checkname']) + artifactsdir_url = '%s/all/%s' % (self.artifacts_baseurl, arg_data['uuid']) # find out if the task is allowed to post results into the namespace if config.get_config().profile == config.ProfileName.PRODUCTION: self.check_namespace(checkname, arg_data) # for now, we're creating the resultsdb group at reporting time - _, log_url = buildbot_utils.get_urls(arg_data['jobid'], - self.masterurl, - self.task_stepname) - artifactsdir_url = '%s/all/%s' % (self.artifacts_baseurl, arg_data['uuid']) - group_data = self.create_resultsdb_group(uuid=arg_data['uuid'], name=checkname) log.info('Posting %s results to ResultsDB...' % len(check_details)) @@ -298,7 +293,7 @@ class ResultsdbDirective(BaseDirective): detail.checkname or arg_data['checkname']) self.ensure_testcase_exists(checkname) - result_log_url = log_url + result_log_url = artifactsdir_url if detail.artifact: artifact_path = self.get_artifact_path( arg_data['artifactsdir'], diff --git a/testing/functest_resultsdb_directive.py b/testing/functest_resultsdb_directive.py index d26ecb7..b156f5e 100644 --- a/testing/functest_resultsdb_directive.py +++ b/testing/functest_resultsdb_directive.py @@ -105,10 +105,7 @@ class TestResultsdbDirective(): call_data = call[2] # the log url depends on the arg_data, so construct it here - ref_builder, ref_jobid = self.ref_arg_data['jobid'].split('/') - ref_log_url = '%s/builders/%s/builds/%s/steps/%s/logs/stdio' %\ - (self.conf.taskotron_master, ref_builder, ref_jobid, - self.conf.buildbot_task_step) + ref_log_url = '%s/all/%s' % (self.conf.artifacts_baseurl, self.ref_arg_data['uuid']) ref_testcase_name = '%s.%s' % (self.ref_arg_data['namespace'], self.ref_arg_data['checkname']) diff --git a/testing/test_resultsdb_directive.py b/testing/test_resultsdb_directive.py index 130f156..3cd2bcf 100644 --- a/testing/test_resultsdb_directive.py +++ b/testing/test_resultsdb_directive.py @@ -153,10 +153,7 @@ class TestResultsdbReport(): call_data = call[2] # the log url depends on the arg_data, so construct it here - ref_builder, ref_jobid = self.ref_arg_data['jobid'].split('/') - ref_log_url = '%s/builders/%s/builds/%s/steps/%s/logs/stdio' %\ - (self.conf.taskotron_master, ref_builder, ref_jobid, - self.conf.buildbot_task_step) + ref_log_url = '%s/all/%s' % (self.conf.artifacts_baseurl, self.ref_arg_data['uuid']) ref_testcase_name = '%s.%s' % (self.ref_arg_data['namespace'], self.ref_arg_data['checkname']) From 6d5ad44536bd641bf253d68d0d2bb0ccb6eb75e2 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Jul 31 2017 12:47:11 +0000 Subject: [PATCH 2/5] create artifacts dir when not local execution --- diff --git a/runner.yml b/runner.yml index 0f44e14..8aaf7ab 100644 --- a/runner.yml +++ b/runner.yml @@ -18,6 +18,10 @@ src: "{{ taskdir }}/" dest: "{{ client_taskdir }}" + - name: create artifacts dir + command: mkdir -p "{{ artifacts }}" + when: not local + - name: Install required packages dnf: name: "{{ item }}" From af0df1c2245345afbb15853ada78fc36bdd7947f Mon Sep 17 00:00:00 2001 From: Lili Nie Date: Aug 28 2017 03:25:26 +0000 Subject: [PATCH 3/5] modify the code to avoid ssh Permission denied errors when no private key provided --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 6a7118a..0b5e2b6 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -99,21 +99,24 @@ class Executor(object): :param str ipaddr: IP address of the machine the task will be run on ''' - taskdir = os.path.dirname(self.arg_data['task']) - taskfile = os.path.basename(self.arg_data['task']) + if os.path.isfile(os.path.join(self.arg_data['taskdir'], 'inventory')): + sti_inventory = os.path.join(config.get_config().client_taskdir, "inventory") + else: + sti_inventory = "/usr/share/ansible/inventory" cmd = [ 'ansible-playbook', 'runner.yml', '--inventory=%s,' % ipaddr, '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'subjects=%s' % self.arg_data['item'], - '-e', 'taskdir=%s' % taskdir, - '-e', 'taskfile=%s' % taskfile, + '-e', 'taskdir=%s' % self.arg_data['taskdir'], '-e', 'client_taskdir=%s' % config.get_config().client_taskdir, + '-e', 'sti_inventory=%s' % sti_inventory, ] if self.run_remotely: - cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) + if 'ssh_privkey' in self.arg_data: + cmd.extend(['--private-key=%s' % self.arg_data['ssh_privkey']]) else: cmd.extend(['--ask-become-pass', '--connection=local', '-e', 'local=true']) diff --git a/libtaskotron/main.py b/libtaskotron/main.py index f3d46c9..c054991 100644 --- a/libtaskotron/main.py +++ b/libtaskotron/main.py @@ -65,7 +65,7 @@ def get_argparser(): pass parser = argparse.ArgumentParser(epilog=ITEM_TYPE_DOCS, formatter_class=CustomFormatter) - parser.add_argument("task", help="task playbook to run") + parser.add_argument("taskdir", help="taskdir with STI playbook to run") parser.add_argument("-a", "--arch", choices=["i386", "x86_64", "armhfp", "noarch"], default='noarch', help="architecture specifying the item to be checked. 'noarch' value " @@ -147,6 +147,10 @@ def process_args(raw_args): args['machine'] = machine args['port'] = 22 + # avoid using './libtaskotron/None' when no ssh_privkey is provided + if args['ssh_privkey'] == None: + del args['ssh_privkey'] + # set paths args['artifactsdir'] = os.path.join(config.get_config().artifactsdir, args['uuid']) @@ -186,6 +190,10 @@ def main(): executor = Executor(arg_data) executor.execute() + # report results + # test_rc = read_test_rc() + # resultsdb.report(subject, rc, hurr, durr) + # finalize log.info('Execution finished at: %s. Task artifacts were saved in: %s', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'), From 6f6a4aa2efa832aece07ad4c60c961bff346534b Mon Sep 17 00:00:00 2001 From: Lili Nie Date: Aug 28 2017 03:40:02 +0000 Subject: [PATCH 4/5] do not remove the vm when no-destroy is specified --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 0b5e2b6..dadd54a 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -138,5 +138,5 @@ class Executor(object): try: self._run_ansible_playbook(ipaddr) finally: - if self.task_vm is not None: + if self.task_vm is not None and not self.arg_data['no_destroy']: self.task_vm.teardown() diff --git a/runner.yml b/runner.yml index 8aaf7ab..ece2a18 100644 --- a/runner.yml +++ b/runner.yml @@ -19,7 +19,9 @@ dest: "{{ client_taskdir }}" - name: create artifacts dir - command: mkdir -p "{{ artifacts }}" + file: + path: "{{ artifacts }}" + state: directory when: not local - name: Install required packages From 24de7061341fbfa157820a291ebfbe5e806b3c7b Mon Sep 17 00:00:00 2001 From: Lili Nie Date: Aug 28 2017 03:56:56 +0000 Subject: [PATCH 5/5] do not ask user to input yes when using unknown hosts --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index dadd54a..5732dbe 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -93,7 +93,7 @@ class Executor(object): return ipaddr - def _run_ansible_playbook(self, ipaddr): + def _run_ansible_playbook(self): '''Run the ansible-playbook command to execute given playbook containing the task. :param str ipaddr: IP address of the machine the task will be run on @@ -106,7 +106,7 @@ class Executor(object): cmd = [ 'ansible-playbook', 'runner.yml', - '--inventory=%s,' % ipaddr, + '-i', 'inventory', '-e', 'artifacts=%s' % self.arg_data['artifactsdir'], '-e', 'subjects=%s' % self.arg_data['item'], '-e', 'taskdir=%s' % self.arg_data['taskdir'], @@ -132,11 +132,13 @@ class Executor(object): ipaddr = self._get_client_ipaddr() if ipaddr is None: ipaddr = self._spawn_vm(self.arg_data['uuid']) + with open(os.path.join('inventory'), 'w') as f: + f.write('[taskotron]\n%s ansible_ssh_common_args="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"'% ipaddr) log.info('Running task on machine %s', ipaddr) try: - self._run_ansible_playbook(ipaddr) + self._run_ansible_playbook() finally: if self.task_vm is not None and not self.arg_data['no_destroy']: self.task_vm.teardown()