From de77e62e251916a6b6526c663699cfe6804787bb Mon Sep 17 00:00:00 2001 From: Petr Šplíchal Date: Jul 18 2018 11:38:28 +0000 Subject: [PATCH 1/2] Support for a simple fmf filter in beakerlib role This adds basic support for test selection based on metadata. The new "filter" variable and dict key under "repositories" accepts a single fmf filter. Separate script "str-filter-tests" is used to check provided paths and filters and lists appropriate tests on the standard output. Manual test listing can be combined with the filter functionality as well. --- diff --git a/roles/standard-test-beakerlib/README.md b/roles/standard-test-beakerlib/README.md index 9b69440..ae1574e 100644 --- a/roles/standard-test-beakerlib/README.md +++ b/roles/standard-test-beakerlib/README.md @@ -6,6 +6,7 @@ testing. You can redefine the following variables in `roles/standard-test-beakerlib/vars/main.yml`: * **tests**: A list of beakerlib test files + * **filter**: An fmf filter to select tests based on test metadata [1] * **artifacts**: An artifacts directory on localhost to store logs * **remote_artifacts**: The directory on the system under test where the logs are stored. Note: if this variable is left undefined, it will default to @@ -21,6 +22,7 @@ testing. You can redefine the following variables in url ... repository url (required) dest ... destination directory (required) version ... desired branch (optional, defaults to master) + filter ... fmf filter (optional, used for test selection) Example usage: @@ -39,3 +41,30 @@ Example usage: required_packages: - expect - which + +Instead of manually listing all tests to be executed it is also +possible to provide an fmf [1] filter in the following way: + + - hosts: localhost + roles: + - role: standard-test-beakerlib + tags: + - classic + repositories: + - repo: "https://src.fedoraproject.org/tests/shell.git" + dest: "shell" + filter: "tier: 1" + +Filter can be used also if tests are stored directly in the git: + + - hosts: localhost + roles: + - role: standard-test-beakerlib + tags: + - classic + filter: "tier: 1" + +See the Fedora CI metadata wiki [1] for more details and examples +on test metadata. + +[1] https://fedoraproject.org/wiki/CI/Metadata diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml index 43b9d4a..14e4a4a 100644 --- a/roles/standard-test-beakerlib/tasks/main.yml +++ b/roles/standard-test-beakerlib/tasks/main.yml @@ -5,6 +5,7 @@ with_flattened: # tests - is defined at tests.yml playbook - "{{ tests }}" + - "{{ filter_tests }}" register: deps delegate_to: "{{ test_runner_inventory_name }}" delegate_facts: True @@ -112,6 +113,7 @@ fi with_items: - "{{ tests }}" + - "{{ filter_tests }}" always: - name: Make the master tests summary log artifact diff --git a/roles/str-common-init/tasks/trunner.yml b/roles/str-common-init/tasks/trunner.yml index b14fca5..4838237 100644 --- a/roles/str-common-init/tasks/trunner.yml +++ b/roles/str-common-init/tasks/trunner.yml @@ -43,6 +43,40 @@ "{{ repositories }}" when: repositories is defined + # Prepare the filter command (check 'filter' variable for tests + # stored directly in the repo and also 'filter' key for each + # fetched test repo dictionary in the 'repositories' list). + - set_fact: + filter_command: "str-filter-tests" + - set_fact: + filter_command: "{{ filter_command }} . '{{ filter }}'" + when: filter is defined + - set_fact: + filter_command: "{{ filter_command }} '{{ item.dest }}' '{{ item.filter }}'" + with_items: + - "{{ repositories }}" + when: repositories is defined and item.filter is defined + - debug: + var: filter_command + verbosity: 1 + + # Filter tests based on the gathered parameters + - set_fact: + filter_tests: [] + - name: Filter tests according to provided fmf filters + shell: "{{ filter_command }}" + register: filter_output + when: filter_command != "str-filter-tests" + - debug: + var: filter_output + verbosity: 2 + - set_fact: + filter_tests: "{{ filter_output.stdout_lines }}" + when: filter_output.stdout_lines is defined + - debug: + var: filter_tests + verbosity: 1 + - debug: msg: > {{ test_runner_inventory_name }} is: diff --git a/scripts/README.md b/scripts/README.md index bf5da45..6f33e9d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,3 +28,10 @@ environment variables set in the inventory script. export TEST_DOCKER_EXTRA_ARGS="--cap-add=SYS_ADMIN" exec merge-standard-inventory "$@" ``` + +## `str-filter-tests` + +This script is used to filter tests for execution based on their +metadata stored in the Flexible Metadata Format [1]. It accepts +pairs of the form "path filter" for each metadata tree to be +investigated. Returns list of filtered tests on the stdout. diff --git a/scripts/str-filter-tests b/scripts/str-filter-tests new file mode 100755 index 0000000..f9a4085 --- /dev/null +++ b/scripts/str-filter-tests @@ -0,0 +1,40 @@ +#!/bin/env python +""" +Filter tests based on metadata and provided path and filter. +Usage: {0} path filter [path filter...] +Directories with no metadata present are silently ignored. +""" + +from __future__ import unicode_literals + +import fmf +import sys + + +def usage(): + """ Show usage message and exit """ + print(__doc__.format(sys.argv[0])) + raise SystemExit(1) + + +if __name__ == "__main__": + # Check for correct arguments + pairs = sys.argv[1:] + if len(pairs) < 2: + usage() + + # Filter tests for each path-filter pair + while pairs: + try: + path = pairs.pop(0).rstrip('/') + filtr = pairs.pop(0) + except IndexError: + usage() + + try: + tree = fmf.Tree(path) + except fmf.utils.RootError: + fmf.utils.info("No metadata found for '{0}'.".format(path)) + continue + for node in tree.prune(keys=['test'], filters=[filtr]): + print(path + (node.data.get('path') or node.name)) From 117b095d25fd565c0c55f5b411e77d6c98e3bd04 Mon Sep 17 00:00:00 2001 From: Petr Šplíchal Date: Aug 16 2018 13:58:49 +0000 Subject: [PATCH 2/2] Incorporate pull request feedback * Rename parameter to "fmf_filter" to make it more distinct * Extend comment for the str-filter-tests command * Gather results from filtered tests as well * Add forgotten link to fmf documentation --- diff --git a/roles/standard-test-beakerlib/README.md b/roles/standard-test-beakerlib/README.md index ae1574e..09897f5 100644 --- a/roles/standard-test-beakerlib/README.md +++ b/roles/standard-test-beakerlib/README.md @@ -6,7 +6,7 @@ testing. You can redefine the following variables in `roles/standard-test-beakerlib/vars/main.yml`: * **tests**: A list of beakerlib test files - * **filter**: An fmf filter to select tests based on test metadata [1] + * **fmf_filter**: An fmf filter to select tests based on test metadata [1] * **artifacts**: An artifacts directory on localhost to store logs * **remote_artifacts**: The directory on the system under test where the logs are stored. Note: if this variable is left undefined, it will default to @@ -22,7 +22,7 @@ testing. You can redefine the following variables in url ... repository url (required) dest ... destination directory (required) version ... desired branch (optional, defaults to master) - filter ... fmf filter (optional, used for test selection) + fmf_filter ... fmf filter (optional, used for test selection) Example usage: @@ -53,7 +53,7 @@ possible to provide an fmf [1] filter in the following way: repositories: - repo: "https://src.fedoraproject.org/tests/shell.git" dest: "shell" - filter: "tier: 1" + fmf_filter: "tier: 1" Filter can be used also if tests are stored directly in the git: @@ -62,7 +62,7 @@ Filter can be used also if tests are stored directly in the git: - role: standard-test-beakerlib tags: - classic - filter: "tier: 1" + fmf_filter: "tier: 1" See the Fedora CI metadata wiki [1] for more details and examples on test metadata. diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml index 14e4a4a..ef61fd2 100644 --- a/roles/standard-test-beakerlib/tasks/main.yml +++ b/roles/standard-test-beakerlib/tasks/main.yml @@ -136,6 +136,7 @@ mv ${logfile} {{ remote_artifacts }}/${status}_${log_file_name} with_items: - "{{ tests }}" + - "{{ filter_tests }}" # Can't go in block. See # https://github.com/ansible/ansible/issues/20736 diff --git a/roles/str-common-init/tasks/trunner.yml b/roles/str-common-init/tasks/trunner.yml index 4838237..d93d656 100644 --- a/roles/str-common-init/tasks/trunner.yml +++ b/roles/str-common-init/tasks/trunner.yml @@ -49,18 +49,21 @@ - set_fact: filter_command: "str-filter-tests" - set_fact: - filter_command: "{{ filter_command }} . '{{ filter }}'" - when: filter is defined + filter_command: "{{ filter_command }} . '{{ fmf_filter }}'" + when: fmf_filter is defined - set_fact: - filter_command: "{{ filter_command }} '{{ item.dest }}' '{{ item.filter }}'" + filter_command: "{{ filter_command }} '{{ item.dest }}' '{{ item.fmf_filter }}'" with_items: - "{{ repositories }}" - when: repositories is defined and item.filter is defined + when: repositories is defined and item.fmf_filter is defined - debug: var: filter_command verbosity: 1 - # Filter tests based on the gathered parameters + # Filter tests based on the gathered parameters. (The filter + # command is extended for each provided filter. That's why + # there's check for filter_command != "str-filter-tests" which + # ensures that there has been at least one filter defined.) - set_fact: filter_tests: [] - name: Filter tests according to provided fmf filters diff --git a/scripts/README.md b/scripts/README.md index 6f33e9d..fb8c642 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -35,3 +35,5 @@ This script is used to filter tests for execution based on their metadata stored in the Flexible Metadata Format [1]. It accepts pairs of the form "path filter" for each metadata tree to be investigated. Returns list of filtered tests on the stdout. + +[1] https://github.com/psss/fmf