From ea7af886e722bdd4813a5cc8d172b0abd06ba139 Mon Sep 17 00:00:00 2001 From: Bruno Goncalves Date: May 20 2019 08:37:08 +0000 Subject: support to timeout --- diff --git a/roles/standard-test-basic/README.md b/roles/standard-test-basic/README.md index 6541a93..5675da8 100644 --- a/roles/standard-test-basic/README.md +++ b/roles/standard-test-basic/README.md @@ -18,6 +18,12 @@ to run testing. You can redefine the following variables in * **required_services**: A list of services to be started before running tests. Please use `required_packages` for specifying packages that provide the services. +## Test case parameters + + * **dir**: test directory. default: is the test name + * **run**: command to run the test. default: ./runtest.sh + * **timeout**: abort test case after this time. More details on [timeout][1]. default: 0 + Example usage: - hosts: localhost @@ -35,6 +41,7 @@ Example usage: - smoke37: dir: python/smoke run: VERSION=3.7 ./venv.sh + timeout: 1h required_packages: - python27 - python37 @@ -42,3 +49,5 @@ Example usage: - python3-virtualenv - python2-devel - python3-devel + +[1]: http://man7.org/linux/man-pages/man1/timeout.1.html diff --git a/roles/standard-test-basic/files/run-basic-test b/roles/standard-test-basic/files/run-basic-test index 5be3556..c0deae3 100755 --- a/roles/standard-test-basic/files/run-basic-test +++ b/roles/standard-test-basic/files/run-basic-test @@ -23,6 +23,7 @@ Options: -w, --workdir test environment work dir. Directory for test execution. -a, --artifactsdir test environment dir to store artifacts -t, --testname name of the test + --timeout test timeout -c, --cmd shell command to run the test EOF } @@ -37,9 +38,10 @@ STR_VERBOSE="${STR_VERBOSE:-}" STR_WORKDIR="${STR_WORKDIR:-}" STR_TEST_NAME="${STR_TEST_NAME:-}" STR_ARTIFACTS_DIR="${STR_ARTIFACTS_DIR:-/tmp}" +STR_TIMEOUT="${STR_TIMEOUT:-0}" # http://wiki.bash-hackers.org/howto/getopts_tutorial -opt=$(getopt -n "$0" --options "hvt:w:a:c:" --longoptions "help,verbose,cmd:,testname:,workdir:,artifactsdir:" -- "$@") +opt=$(getopt -n "$0" --options "hvt:w:a:c:" --longoptions "help,verbose,cmd:,testname:,workdir:,artifactsdir:,timeout:" -- "$@") eval set -- "$opt" while [[ $# -gt 0 ]]; do case "$1" in @@ -59,6 +61,10 @@ while [[ $# -gt 0 ]]; do STR_ARTIFACTS_DIR="$2" shift 2 ;; + --timeout) + STR_TIMEOUT="$2" + shift 2 + ;; -v|--verbose) DEBUG="-v" shift @@ -91,6 +97,7 @@ debug "Test: $STR_TEST_NAME" debug "Command: $STR_CMD" debug "Work dir: $STR_WORKDIR" debug "Artifacts dir: $STR_ARTIFACTS_DIR" +debug "Timeout: $STR_TIMEOUT" # Up to this point any fail is considered as ci-sytem fail. Exit code != 0. # Starting from this point and bellow any fail is considered as a test fail. Exit code == 0. @@ -111,6 +118,10 @@ clean_exit() { if [[ $rc -eq 127 ]]; then echo "$STR_TEST_NAME (problem with test execution)" >&2 status="ERROR" + elif [[ $rc -eq 124 ]]; then + # test case timed out + echo "$STR_TEST_NAME (test aborted due to timeout)" >&2 + status="ERROR" elif [[ $rc -eq 0 ]]; then status="PASS" fi @@ -143,6 +154,6 @@ cd "$STR_WORKDIR" if [ -f "$STR_CMD" ]; then chmod 0775 "$STR_CMD" fi -bash -c "$STR_CMD" +timeout "$STR_TIMEOUT" bash -c "$STR_CMD" # Explicit return code exit $? diff --git a/roles/standard-test-basic/tasks/main.yml b/roles/standard-test-basic/tasks/main.yml index 61b8cd9..86948b9 100644 --- a/roles/standard-test-basic/tasks/main.yml +++ b/roles/standard-test-basic/tasks/main.yml @@ -18,6 +18,7 @@ --workdir "{{ tenv_workdir }}/{{ item if item.keys is not defined else item[(item.keys()|list)[0]]['dir']|default((item.keys()|list)[0]) }}" \ --artifactsdir "{{ remote_artifacts }}" \ --test "{{ item if item.keys is not defined else (item.keys()|list)[0] }}" \ + --timeout "{{ '0' if item.keys is not defined else item[(item.keys()|list)[0]]['timeout']|default('0') }}" \ --cmd "{{ './runtest.sh' if item.keys is not defined else item[(item.keys()|list)[0]]['run']|default('./runtest.sh') | regex_replace('\\', '\\\\') | regex_replace('\"', '\"') | regex_replace('\$', '\\$') }}" with_items: - "{{ tests }}" diff --git a/roles/standard-test-beakerlib/README.md b/roles/standard-test-beakerlib/README.md index 2dee24f..3b22150 100644 --- a/roles/standard-test-beakerlib/README.md +++ b/roles/standard-test-beakerlib/README.md @@ -24,6 +24,10 @@ testing. You can redefine the following variables in version ... desired branch (optional, defaults to master) fmf_filter ... fmf filter (optional, used for test selection) +## Test case parameters + + * **timeout**: abort test case after this time. More details on [timeout][1]. default: 0 + Example usage: ```yaml @@ -38,7 +42,8 @@ Example usage: tests: - shell/func - shell/login - - shell/smoke + - shell/smoke: + timeout: 1h required_packages: - expect - which diff --git a/roles/standard-test-beakerlib/files/run-beakerlib-test b/roles/standard-test-beakerlib/files/run-beakerlib-test index a46bf4b..4918d2f 100755 --- a/roles/standard-test-beakerlib/files/run-beakerlib-test +++ b/roles/standard-test-beakerlib/files/run-beakerlib-test @@ -23,6 +23,7 @@ Options: -w, --workdir test environment work dir -a, --artifactsdir test environment dir to store artifacts -t, --test test to run, can be a path to directory or a runtest.sh file + --timeout test timeout EOF } @@ -35,9 +36,10 @@ STR_VERBOSE="${STR_VERBOSE:-}" STR_BKR_TEST="${STR_BKR_TEST:-}" STR_WORKDIR="${STR_WORKDIR:-}" STR_ARTIFACTS_DIR="${STR_ARTIFACTS_DIR:-}" +STR_TIMEOUT="${STR_TIMEOUT:-0}" # http://wiki.bash-hackers.org/howto/getopts_tutorial -opt=$(getopt -n "$0" --options "hvt:w:a:" --longoptions "help,verbose,test:,workdir:,artifactsdir:" -- "$@") +opt=$(getopt -n "$0" --options "hvt:w:a:" --longoptions "help,verbose,test:,workdir:,artifactsdir:,timeout:" -- "$@") eval set -- "$opt" while [[ $# -gt 0 ]]; do case "$1" in @@ -53,6 +55,10 @@ while [[ $# -gt 0 ]]; do STR_ARTIFACTS_DIR="$2" shift 2 ;; + --timeout) + STR_TIMEOUT="$2" + shift 2 + ;; -v|--verbose) DEBUG="-v" shift @@ -80,6 +86,7 @@ fi debug "Test: $STR_BKR_TEST" debug "Work dir: $STR_WORKDIR" debug "Artifacts dir: $STR_ARTIFACTS_DIR" +debug "Timeout: $STR_TIMEOUT" STR_BKR_TEST_DASHED=$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g') # Up to this point any fail is considered as ci-sytem fail. Exit code != 0. @@ -101,6 +108,10 @@ clean_exit() { local status if [[ $rc -eq 127 ]]; then status="ERROR" + elif [[ $rc -eq 124 ]]; then + # test case timed out + echo "$STR_BKR_TEST (test aborted due to timeout)" >&2 + status="ERROR" elif grep -q "RESULT: WARN" "$log_file_path"; then status="ERROR" elif grep -q "RESULT: FAIL" "$log_file_path"; then @@ -146,7 +157,7 @@ fi if [ -f "$STR_BKR_TEST" ]; then debug "Running test from file: $STR_BKR_TEST" cd $(dirname "$STR_BKR_TEST") - /bin/sh -e ./$(basename "$STR_BKR_TEST") || : + timeout "$STR_TIMEOUT" /bin/sh -e ./$(basename "$STR_BKR_TEST") exit fi @@ -157,10 +168,10 @@ if [ -d "$STR_BKR_TEST" ]; then get-test-deps -i . if [ -f "Makefile" ] && command -p -v "make" >"/dev/null" 2>&1; then debug "Running test from Makefile" - make run || : + timeout "$STR_TIMEOUT" make run elif [ -f "runtest.sh" ]; then debug "Running test from runtest.sh" - /bin/sh -e ./runtest.sh || : + timeout "$STR_TIMEOUT" /bin/sh -e ./runtest.sh else echo "FAIL test $STR_BKR_TEST do not know how to run test" fi diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml index 30d57f6..7d99091 100644 --- a/roles/standard-test-beakerlib/tasks/main.yml +++ b/roles/standard-test-beakerlib/tasks/main.yml @@ -52,7 +52,10 @@ - block: - name: Run beakerlib tests - script: run-beakerlib-test --workdir {{ tenv_workdir }} --artifactsdir {{ remote_artifacts }} --test {{ item }} + script: run-beakerlib-test --workdir {{ tenv_workdir }} \ + --artifactsdir {{ remote_artifacts }} \ + --test "{{ item if item.keys is not defined else (item.keys()|list)[0] }}" \ + --timeout "{{ '0' if item.keys is not defined else item[(item.keys()|list)[0]]['timeout']|default('0') }}" with_items: - "{{ tests }}" - "{{ filter_tests }}" diff --git a/tests/basic.yml b/tests/basic.yml index 4ca1217..2667143 100644 --- a/tests/basic.yml +++ b/tests/basic.yml @@ -57,3 +57,38 @@ - import_tasks: shared-tasks/verify_failed_test.yml - import_tasks: shared-tasks/artifacts_test_env.yml - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role passes on timeout +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-basic + tests: + - test-basic-timeout: + run: ls; sleep 1 + timeout: 5 + tasks: + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role fails on timeout +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-basic + tests: + - test-basic-timeout-fail: + run: ls; sleep 5 + timeout: 1 + ignore_errors: yes + tasks: + # 'verify_error_test' tasks should run after 'test-basic-timeout-fail' + - import_tasks: shared-tasks/verify_error_test.yml + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml diff --git a/tests/beakerlib.yml b/tests/beakerlib.yml index 5f84bda..84c2d91 100644 --- a/tests/beakerlib.yml +++ b/tests/beakerlib.yml @@ -33,3 +33,36 @@ - import_tasks: shared-tasks/verify_failed_test.yml - import_tasks: shared-tasks/artifacts_test_env.yml - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role behaves correctly with timeout +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-beakerlib + tests: + - test-beakerlib-timeout: + timeout: 10 + tasks: + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role behaves correctly if timeout fails +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-beakerlib + tests: + - test-beakerlib-timeout: + timeout: 1 + ignore_errors: yes + tasks: + # 'tests_verify_error_test' tasks should run after 'test-beakerlib-timeout' + - import_tasks: shared-tasks/verify_error_test.yml + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml diff --git a/tests/shared-tasks/verify_error_test.yml b/tests/shared-tasks/verify_error_test.yml new file mode 100644 index 0000000..5d1253d --- /dev/null +++ b/tests/shared-tasks/verify_error_test.yml @@ -0,0 +1,11 @@ +# These tasks should run after as test that fails +- name: Read test.log on test environment + shell: "cat {{remote_artifacts}}/test.log" + register: test_log + +- name: Check for ERROR on test.log on test environment + fail: msg="Could not find ERROR on test log" + when: test_log.stdout.find("ERROR") == -1 + # We should fail when we can not find ERROR string on log + # -1 means string not found + diff --git a/tests/test-beakerlib-timeout/runtest.sh b/tests/test-beakerlib-timeout/runtest.sh new file mode 100644 index 0000000..9b78f46 --- /dev/null +++ b/tests/test-beakerlib-timeout/runtest.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="bash" + +rlJournalStart + rlPhaseStartTest "beakerlib timeout test" + rlRun "ls /; sleep 5" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd