From 820ab1fec0f341650ecfe83222a94d884596447b Mon Sep 17 00:00:00 2001
From: Petr Šplíchal <psplicha@redhat.com>
Date: May 14 2019 11:56:45 +0000
Subject: Clearly show overall result of the testing


Report role result task has been updated to show actual result of
testing, which means: ERROR upon any test execution problem, FAIL
when any test fails and PASS if there is at least one test passed.

---

diff --git a/roles/standard-test-basic/tasks/main.yml b/roles/standard-test-basic/tasks/main.yml
index 83b4e55..61b8cd9 100644
--- a/roles/standard-test-basic/tasks/main.yml
+++ b/roles/standard-test-basic/tasks/main.yml
@@ -25,15 +25,27 @@
   # Can't go in block. See
   # https://github.com/ansible/ansible/issues/20736
   - name: Check the results
-    shell: grep "^ERROR" {{ remote_artifacts }}/test.log
-    register: test_error
-    # Never fail at this step. Just store result of tests.
-    failed_when: False
+    shell: |
+      log="{{ remote_artifacts }}/test.log"
+      if [ ! -f "$log" ]; then
+          echo ERROR
+          echo "Test results not found." 1>&2
+      elif grep ^ERROR "$log" 1>&2; then
+          echo ERROR
+      elif grep ^FAIL "$log" 1>&2; then
+          echo FAIL
+      elif grep -q ^PASS "$log"; then
+          echo PASS
+      else
+          echo ERROR
+          echo "No test results found." 1>&2
+      fi
+    register: test_results
 
   - name: Set role result
     set_fact:
-      role_result_error: "{{ (test_error.stdout|d|length > 0) or (test_error.stderr|d|length > 0) }}"
-      role_result_msg: "{{ test_error.stdout|d('test execution error.') }}"
+      role_result: "{{ test_results.stdout }}"
+      role_message: "{{ test_results.stderr|d('test execution error.') }}"
 
   - include_role:
       name: str-common-final
diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml
index 64e221b..30d57f6 100644
--- a/roles/standard-test-beakerlib/tasks/main.yml
+++ b/roles/standard-test-beakerlib/tasks/main.yml
@@ -63,15 +63,27 @@
   # Can't go in block. See
   # https://github.com/ansible/ansible/issues/20736
   - name: Check the results
-    shell: grep "^ERROR" {{ remote_artifacts }}/test.log
-    register: test_error
-    # Never fail at this step. Just store result of tests.
-    failed_when: False
+    shell: |
+      log="{{ remote_artifacts }}/test.log"
+      if [ ! -f "$log" ]; then
+          echo ERROR
+          echo "Test results not found." 1>&2
+      elif grep ^ERROR "$log" 1>&2; then
+          echo ERROR
+      elif grep ^FAIL "$log" 1>&2; then
+          echo FAIL
+      elif grep -q ^PASS "$log"; then
+          echo PASS
+      else
+          echo ERROR
+          echo "No test results found." 1>&2
+      fi
+    register: test_results
 
   - name: Set role result
     set_fact:
-      role_result_error: "{{ (test_error.stdout|d|length > 0) or (test_error.stderr|d|length > 0) }}"
-      role_result_msg: "{{ test_error.stdout|d('test execution error.') }}"
+      role_result: "{{ test_results.stdout }}"
+      role_message: "{{ test_results.stderr|d('test execution error.') }}"
 
   - include_role:
       name: str-common-final
diff --git a/roles/str-common-final/tasks/main.yml b/roles/str-common-final/tasks/main.yml
index 55f4299..01a7a18 100644
--- a/roles/str-common-final/tasks/main.yml
+++ b/roles/str-common-final/tasks/main.yml
@@ -11,8 +11,8 @@
 - name: Report role result
   vars:
     msg: |
-       Tests error: {{ role_result_error|d('Undefined') }}
-       Tests msg: {{ role_result_msg|d('None') }}
+       Result: {{ role_result|d('Undefined') }}
+       {{ role_message|d('None') }}
   debug:
     msg: "{{ msg.split('\n') }}"
-  failed_when: "role_result_error|bool"
+  failed_when: role_result == 'ERROR'