From a76bdfa464e806300ba2113005f10a04a22751e5 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Jun 04 2018 15:52:53 +0000 Subject: [PATCH 1/3] Enable grabbing secrets from Vault server --- diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml index 69c4cb6..321ae2a 100644 --- a/data/ansible/runner.yml +++ b/data/ansible/runner.yml @@ -81,6 +81,12 @@ register: result until: result.rc == 0 + - name: Upload secrets + copy: + src: '{{taskotron_secrets_file}}' + dest: '{{taskotron_secrets_file}}' + when: not local + - name: Start heartbeat command: > ./heartbeat.sh start {{ heartbeat_file }} {{ heartbeat_interval }} @@ -95,6 +101,15 @@ static: yes # variable 'task' is registered here + - name: Delete secrets + file: + state: absent + path: '{{taskotron_secrets_file}}' + tags: + - failsafe + when: not local + + - name: Kill heartbeat command: ./heartbeat.sh stop {{ heartbeat_file }} when: taskotron_keepalive_minutes|int > 0 diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py index ebd569c..54c50f9 100644 --- a/libtaskotron/config_defaults.py +++ b/libtaskotron/config_defaults.py @@ -74,6 +74,10 @@ class Config(object): taskotron_master = 'http://localhost/taskmaster' #: artifacts_baseurl = 'http://localhost/artifacts' #: download_cache_enabled = True #: + vault_enabled = False #: + vault_server = 'http://localhost:4999/api/v1' #: + vault_username = 'taskotron' #: + vault_password = 'taskotron' #: tmpdir = '/var/tmp/taskotron' #: logdir = '/var/log/taskotron' #: diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index 469f2ad..9fb226c 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -109,6 +109,18 @@ summary will be printed out into the log, like this:: directive_class = 'ResultsdbDirective' +def git_origin_url(taskdir): + try: + gitconfig_filename = os.path.join(taskdir, '.git/config') + gitconfig = configparser.ConfigParser() + gitconfig.read(gitconfig_filename) + task_repo_url = gitconfig['remote "origin"']['url'] + except TypeError as e: + log.exception(e) + task_repo_url = None + return task_repo_url + + class ResultsdbDirective(BaseDirective): @@ -217,17 +229,11 @@ class ResultsdbDirective(BaseDirective): else: raise TaskotronDirectiveError('No namespace for task %s exists.' % checkname) - try: - taskdir = os.path.dirname(os.path.abspath(arg_data['task'])) - gitconfig_filename = os.path.join(taskdir, '.git/config') - - gitconfig = configparser.ConfigParser() - gitconfig.read(gitconfig_filename) - task_repo_url = gitconfig['remote "origin"']['url'] - except TypeError as e: - log.exception(e) + taskdir = os.path.dirname(os.path.abspath(arg_data['task'])) + task_repo_url = git_origin_url(taskdir) + if not task_repo_url: raise TaskotronDirectiveError("Could not find task's git remote 'origin' url" - "in %s" % gitconfig_filename) + "in %s" % os.path.join(taskdir, '.git/config')) try: if not [ns_repo for ns_repo in ns_repos if task_repo_url.strip().startswith(ns_repo)]: diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 3d0aa0a..7378914 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -12,6 +12,10 @@ import yaml import fnmatch import signal import json +import tempfile +import configparser +import requests +import re from libtaskotron import config from libtaskotron import image_utils @@ -181,6 +185,48 @@ class Executor(object): vars_[var] = val return vars_ + def _get_vault_secrets(self, taskdir): + '''Load secrets from the Vault server and store them in a file + + :param str taskdir: path to the directory with test suite (on overlord) + :return: a filename with decrypted secrets + ''' + cfg = config.get_config() + secrets = {} + if cfg.vault_enabled: + task_repo_url = resultsdb_directive.git_origin_url(taskdir) + if task_repo_url: + re_enabler = re.compile(r'taskotron_enable\(([^)]*)\)') + try: + r = requests.get( + "%s/buckets" % cfg.vault_server, + auth=(cfg.vault_username, cfg.vault_password), + ) + except requests.exceptions.RequestException: + r = None + if r and r.ok: + data = r.json()['data'] + valid_buckets = [] + for b in data: + desc = b['description'] + if not desc: + continue + enabled_for = ', '.join(re_enabler.findall(desc)) + if not task_repo_url in enabled_for: + continue + valid_buckets.append(b) + for b in valid_buckets: + secrets[b['uuid']] = b['secrets'] + + if config.get_config().profile == config.ProfileName.TESTING: + return secrets + + fd, fname = tempfile.mkstemp(prefix='taskotron_secrets') + os.close(fd) + with open(fname, 'w') as fd: + fd.write(json.dumps(secrets, indent=2, sort_keys=True)) + return fname + def _create_playbook_vars(self, test_playbook): '''Create and return dictionary containing all variables to be used with our ansible playbook. @@ -230,6 +276,8 @@ class Executor(object): whether VM guest distro has to match taskotron_item taskotron_match_host_release whether VM guest release has to match taskotron_item + taskotron_secrets_file + path to the file with secrets appropriate for the task taskotron_supported_arches list of base architectures supported by Taskotron (e.g. 'armhfp') @@ -274,6 +322,7 @@ class Executor(object): vars_['taskotron_arch'] = self.arg_data['arch'] vars_['taskotron_item'] = self.arg_data['item'] vars_['taskotron_item_type'] = self.arg_data['type'] + vars_['taskotron_secrets_file'] = self._get_vault_secrets(taskdir=vars_['taskdir']) vars_['taskotron_supported_arches'] = cfg.supported_arches vars_['taskotron_supported_binary_arches'] = [binarch for arch in cfg.supported_arches for binarch in arch_utils.Arches.binary[arch]] @@ -446,6 +495,7 @@ class Executor(object): failed = [] for test_playbook in test_playbooks: + playbook_vars = None try: # syntax check self._check_playbook_syntax(os.path.join( @@ -487,7 +537,12 @@ class Executor(object): break else: self.task_vm.teardown() - + try: + if playbook_vars and config.get_config().profile != config.ProfileName.TESTING: + os.remove(playbook_vars['taskotron_secrets_file']) + except OSError: + log.info("Could not delete the secrets file at %r", + playbook_vars['taskotron_secrets_file']) log.info('Playbook execution finished: %s', test_playbook) if failed: diff --git a/testing/functest_executor.py b/testing/functest_executor.py index a7c2a48..40771ee 100644 --- a/testing/functest_executor.py +++ b/testing/functest_executor.py @@ -89,3 +89,44 @@ class TestExecutor(): assert len(matched) == 1 warn_task_line = lines.index(matched[0]) assert 'skipping' in lines[warn_task_line + 1] + + def test_get_vault_secrets(self, monkeypatch): + class MockRequests(object): + def __init__(self, retval): + self.ok = True + self.retval = retval + self.exceptions = mock.Mock() + def get(self, *args, **kwargs): + return self + def json(self, *args, **kwargs): + return self.retval + + self.conf.vault_enabled = True + + mock_git_origin_url = mock.Mock(return_value='fake://repo.git') + monkeypatch.setattr(executor.resultsdb_directive, 'git_origin_url', mock_git_origin_url) + + mock_requests = MockRequests({'data': [ + { + 'description': 'dat description', + 'secrets': 'dem secrets', + 'uuid': 'dat uuid', + } + ]}) + monkeypatch.setattr(executor, 'requests', mock_requests) + secrets = self.executor._get_vault_secrets(taskdir=None) + assert secrets == {} + + + mock_requests = MockRequests({'data': [ + { + 'description': 'dat_description, taskotron_enable(fake://repo.git)', + 'secrets': 'dem_secrets', + 'uuid': 'dat uuid', + } + ]}) + monkeypatch.setattr(executor, 'requests', mock_requests) + secrets = self.executor._get_vault_secrets(taskdir=None) + assert secrets == {'dat uuid': 'dem_secrets'} + + From 8036ea3b3abe38809f9158ee14910bfdc7f9a333 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Jun 06 2018 13:22:46 +0000 Subject: [PATCH 2/3] Fixed to reflect review --- diff --git a/conf/taskotron.yaml.example b/conf/taskotron.yaml.example index 76ab182..13eacc2 100644 --- a/conf/taskotron.yaml.example +++ b/conf/taskotron.yaml.example @@ -88,6 +88,19 @@ ## Please make sure the URL doesn't have a trailing slash. #execdb_server: http://localhost:5003 +## URL of VAULT server API interface, which stores secrets. +## Please make sure the URL doesn't have a trailing slash. +#vault_server: http://localhost:4999/api/v1 + +## Use vault to grab secrets +#vault_enabled: False + +## Username for vault server +#vault_username: taskotron + +## Password for vault server +#vault_password: taskotron + ## URL of taskotron buildmaster, to construct log URLs from. ## Please make sure the URL doesn't have a trailing slash. #taskotron_master: http://localhost/taskmaster diff --git a/docs/source/writingtasks.rst b/docs/source/writingtasks.rst index f6cdaae..6b998d4 100644 --- a/docs/source/writingtasks.rst +++ b/docs/source/writingtasks.rst @@ -151,6 +151,14 @@ Taskotron-specific variables: tasks might want to ignore this (for example, if they're capable of checking all architectures in a single run). +``taskotron_secrets_file`` + (string) Path to the file containing relevant secrets grabbed from the + Vault server. If a bucket owned by the taskotron user has a string like + ``taskotron_enable(XYZ)`` in the ``description`` field, and the ``XYZ`` + is the same as the current task's git source repo, the bucket's secrets + are included in the file. This file contains a json-encoded dictionary, + where bucket's uuids are keys, and a dict of secrets is the respective value. + ``taskotron_supported_arches`` (list of strings) A list of all base architectures (i.e. ``x86_64``, ``i386``, ``armhfp``) that are officially supported by this Taskotron diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 7378914..cc218bc 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -61,6 +61,7 @@ class Executor(object): 'taskotron_arch', 'taskotron_item_type', 'taskotron_item', + 'taskotron_secrets_file', 'taskotron_supported_arches', 'taskotron_supported_binary_arches', ] @@ -196,17 +197,20 @@ class Executor(object): if cfg.vault_enabled: task_repo_url = resultsdb_directive.git_origin_url(taskdir) if task_repo_url: - re_enabler = re.compile(r'taskotron_enable\(([^)]*)\)') try: - r = requests.get( + session = file_utils._get_session() + r = session.get( "%s/buckets" % cfg.vault_server, auth=(cfg.vault_username, cfg.vault_password), ) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + log.error("Connection to Vault server failed. %s", e) r = None + if r and r.ok: data = r.json()['data'] valid_buckets = [] + re_enabler = re.compile(r'taskotron_enable\((.*?)\)') for b in data: desc = b['description'] if not desc: @@ -217,6 +221,8 @@ class Executor(object): valid_buckets.append(b) for b in valid_buckets: secrets[b['uuid']] = b['secrets'] + elif r and not r.ok: + log.error("Could not get data from vault. %r, %r", r.status_code, r.reason) if config.get_config().profile == config.ProfileName.TESTING: return secrets @@ -540,9 +546,9 @@ class Executor(object): try: if playbook_vars and config.get_config().profile != config.ProfileName.TESTING: os.remove(playbook_vars['taskotron_secrets_file']) - except OSError: - log.info("Could not delete the secrets file at %r", - playbook_vars['taskotron_secrets_file']) + except OSError as e: + log.warning("Could not delete the secrets file at %r. %s", + playbook_vars['taskotron_secrets_file'], e) log.info('Playbook execution finished: %s', test_playbook) if failed: diff --git a/testing/functest_executor.py b/testing/functest_executor.py index 40771ee..5920f28 100644 --- a/testing/functest_executor.py +++ b/testing/functest_executor.py @@ -113,7 +113,8 @@ class TestExecutor(): 'uuid': 'dat uuid', } ]}) - monkeypatch.setattr(executor, 'requests', mock_requests) + mock_session = mock.Mock(return_value=mock_requests) + monkeypatch.setattr(executor.file_utils, '_get_session', mock_session) secrets = self.executor._get_vault_secrets(taskdir=None) assert secrets == {} @@ -125,7 +126,8 @@ class TestExecutor(): 'uuid': 'dat uuid', } ]}) - monkeypatch.setattr(executor, 'requests', mock_requests) + mock_session = mock.Mock(return_value=mock_requests) + monkeypatch.setattr(executor.file_utils, '_get_session', mock_session) secrets = self.executor._get_vault_secrets(taskdir=None) assert secrets == {'dat uuid': 'dem_secrets'} From 8df72216d0654d2236cdc2587a0f578e099ec74b Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Jun 06 2018 13:25:29 +0000 Subject: [PATCH 3/3] . --- diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index cc218bc..f90978f 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -535,6 +535,12 @@ class Executor(object): test_playbook, e) failed.append(test_playbook) finally: + try: + if playbook_vars and config.get_config().profile != config.ProfileName.TESTING: + os.remove(playbook_vars['taskotron_secrets_file']) + except OSError as e: + log.warning("Could not delete the secrets file at %r. %s", + playbook_vars['taskotron_secrets_file'], e) if self.task_vm is not None: if self.arg_data['no_destroy']: log.info('Not destroying disposable client as ' @@ -543,12 +549,6 @@ class Executor(object): break else: self.task_vm.teardown() - try: - if playbook_vars and config.get_config().profile != config.ProfileName.TESTING: - os.remove(playbook_vars['taskotron_secrets_file']) - except OSError as e: - log.warning("Could not delete the secrets file at %r. %s", - playbook_vars['taskotron_secrets_file'], e) log.info('Playbook execution finished: %s', test_playbook) if failed: