Fix yum integration tests (#35177)

* use tests instead of filters

* Don't check 'rc', use success/failed tests

- yum module use 'fail_json' which set 'failed'
- 'rc' is for backward compatibility:
  bbd6b8bb42/lib/ansible/executor/task_executor.py (L571)
- besides yum module doesn't document any return values

* Allow to run tests twice in a row

ansible-test is executed with '--retry-on-error' switch, then tests must
be able to executed twice in a row.

Error was:
TASK [yum : install group] ***
ok: [testhost] => {
    "attempts": 1,
    "changed": false,
    "failed": false,
    "invocation": { [...] },
    "msg": "",
    "rc": 0,
    "results": []
}

TASK [yum : verify installation of the group] ***
fatal: [testhost]: FAILED! => {
    "assertion": "yum_result.changed",
    "attempts": 1,
    "changed": false,
    "evaluated_to": false,
    "failed": true
}

test/integration/targets/yum/tasks/main.yml

@@ -18,7 +18,32 @@
 
 # Note: We install the yum package onto Fedora so that this will work on dnf systems
 # We want to test that for people who don't want to upgrade their systems.
-- include: 'yum.yml'
+
+- block:
+  - include: 'yum.yml'
+  always:
+    - name: remove installed packages
+      yum:
+        name:
+          - bc
+          - sos
+        state: absent
+
+    # On CentOS 6 'yum groupremove "Development Tools"' fails when groupremove_leaf_only
+    # isn't enabled, that's why a package belonging to "Development Tools" (autoconf)
+    # is removed instead of the whole group.
+    - name: remove installed group
+      yum:
+        name: "@Development tools"
+        state: absent
+      when:
+        - (ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and ansible_distribution_major_version|int > 6) or ansible_distribution in ['Fedora']
+    - name: remove a package from a group
+      yum:
+        name: 'autoconf'
+        state: absent
+      when:
+        - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux'] and ansible_distribution_major_version|int <= 6
   when:
     - ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux', 'Fedora']
     - ansible_python.version.major == 2

test/integration/targets/yum/tasks/yum.yml

@@ -5,14 +5,14 @@
 
 - name: check sos with rpm
   shell: rpm -q sos
-  failed_when: False
+  ignore_errors: True
   register: rpm_result
 
 - name: verify uninstallation of sos
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "rpm_result.rc == 1"
+        - "yum_result is success"
+        - "rpm_result is failed"
 
 # UNINSTALL AGAIN
 - name: uninstall sos again in check mode
@@ -23,7 +23,7 @@
 - name: verify no change on re-uninstall in check mode
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: uninstall sos again
   yum: name=sos state=removed
@@ -32,7 +32,7 @@
 - name: verify no change on re-uninstall
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 # INSTALL
 - name: install sos in check mode
@@ -43,32 +43,28 @@
 - name: verify installation of sos in check mode
   assert:
     that:
-        - "yum_result.changed"
+        - "yum_result is changed"
 
 - name: install sos
   yum: name=sos state=present
   register: yum_result
 
-- name: check sos with rpm
-  shell: rpm -q sos
-  failed_when: False
-  register: rpm_result
-
 - name: verify installation of sos
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
-        - "rpm_result.rc == 0"
+        - "yum_result is success"
+        - "yum_result is changed"
 
 - name: verify yum module outputs
   assert:
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
+- name: check sos with rpm
+  shell: rpm -q sos
+
 # INSTALL AGAIN
 - name: install sos again in check mode
   yum: name=sos state=present
@@ -77,7 +73,7 @@
 - name: verify no change on second install in check mode
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: install sos again
   yum: name=sos state=present
@@ -85,7 +81,7 @@
 - name: verify no change on second install
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 # INSTALL AGAIN WITH LATEST
 - name: install sos again with state latest in check mode
@@ -95,7 +91,7 @@
 - name: verify install sos again with state latest in check mode
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: install sos again with state latest idempotence
   yum: name=sos state=latest
@@ -103,7 +99,7 @@
 - name: verify install sos again with state latest idempotence
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 # INSTALL WITH LATEST
 - name: uninstall sos
@@ -121,7 +117,7 @@
 - name: verify install sos with state latest in check mode
   assert:
     that:
-        - "yum_result.changed"
+        - "yum_result is changed"
 
 - name: install sos with state latest
   yum: name=sos state=latest
@@ -129,7 +125,7 @@
 - name: verify install sos with state latest
   assert:
     that:
-        - "yum_result.changed"
+        - "yum_result is changed"
 
 - name: install sos with state latest idempotence
   yum: name=sos state=latest
@@ -137,50 +133,43 @@
 - name: verify install sos with state latest idempotence
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 # Multiple packages
 - name: uninstall sos and bc
   yum: name=sos,bc state=removed
-  register: yum_result
 
 - name: check sos with rpm
   shell: rpm -q sos
-  failed_when: False
+  ignore_errors: True
   register: rpm_sos_result
 
 - name: check bc with rpm
   shell: rpm -q bc
-  failed_when: False
+  ignore_errors: True
   register: rpm_bc_result
 
 - name: verify packages installed
   assert:
     that:
-        - "rpm_sos_result.rc != 0"
-        - "rpm_bc_result.rc != 0"
+        - "rpm_sos_result is failed"
+        - "rpm_bc_result is failed"
 
 - name: install sos and bc as comma separated
   yum: name=sos,bc state=present
   register: yum_result
 
+- name: verify packages installed
+  assert:
+    that:
+        - "yum_result is success"
+        - "yum_result is changed"
+
 - name: check sos with rpm
   shell: rpm -q sos
-  failed_when: False
-  register: rpm_sos_result
 
 - name: check bc with rpm
   shell: rpm -q bc
-  failed_when: False
-  register: rpm_bc_result
-
-- name: verify packages installed
-  assert:
-    that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
-        - "rpm_sos_result.rc == 0"
-        - "rpm_bc_result.rc == 0"
 
 - name: uninstall sos and bc
   yum: name=sos,bc state=removed
@@ -194,23 +183,17 @@
     state: present
   register: yum_result
 
+- name: verify packages installed
+  assert:
+    that:
+        - "yum_result is success"
+        - "yum_result is changed"
+
 - name: check sos with rpm
   shell: rpm -q sos
-  failed_when: False
-  register: rpm_sos_result
 
 - name: check bc with rpm
   shell: rpm -q bc
-  failed_when: False
-  register: rpm_bc_result
-
-- name: verify packages installed
-  assert:
-    that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
-        - "rpm_sos_result.rc == 0"
-        - "rpm_bc_result.rc == 0"
 
 - name: uninstall sos and bc
   yum: name=sos,bc state=removed
@@ -222,23 +205,17 @@
     state: present
   register: yum_result
 
+- name: verify packages installed
+  assert:
+    that:
+        - "yum_result is success"
+        - "yum_result is changed"
+
 - name: check sos with rpm
   shell: rpm -q sos
-  failed_when: False
-  register: rpm_sos_result
 
 - name: check sos with rpm
   shell: rpm -q bc
-  failed_when: False
-  register: rpm_bc_result
-
-- name: verify packages installed
-  assert:
-    that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
-        - "rpm_sos_result.rc == 0"
-        - "rpm_bc_result.rc == 0"
 
 - name: uninstall sos and bc
   yum: name=sos,bc state=removed
@@ -260,26 +237,22 @@
   yum: name=sos state=present installroot='/'
   register: yum_result
 
-- name: check sos with rpm
-  shell: rpm -q sos --root=/
-  failed_when: False
-  register: rpm_result
-
 - name: verify installation of sos
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
-        - "rpm_result.rc == 0"
+        - "yum_result is success"
+        - "yum_result is changed"
 
 - name: verify yum module outputs
   assert:
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
+- name: check sos with rpm
+  shell: rpm -q sos --root=/
+
 - name: uninstall sos
   yum:
     name: sos
@@ -296,15 +269,14 @@
 - name: verify installation of the group
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
+        - "yum_result is success"
+        - "yum_result is changed"
 
 - name: verify yum module outputs
   assert:
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: install the group again
@@ -316,15 +288,14 @@
 - name: verify nothing changed
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "not yum_result.changed"
+        - "yum_result is success"
+        - "not yum_result is changed"
 
 - name: verify yum module outputs
   assert:
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: install the group again but also with a package that is not yet installed
@@ -338,15 +309,14 @@
 - name: verify sos is installed
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
+        - "yum_result is success"
+        - "yum_result is changed"
 
 - name: verify yum module outputs
   assert:
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: try to install the group again, with --check to check 'changed'
@@ -359,7 +329,7 @@
 - name: verify nothing changed
   assert:
     that:
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: verify yum module outputs
   assert:
@@ -378,8 +348,8 @@
 - name: verify installation of the non existing group failed
   assert:
     that:
-        - "yum_result.rc == 1"
-        - "not yum_result.changed"
+        - "yum_result is failed"
+        - "not yum_result is changed"
         - "yum_result is failed"
 
 - name: verify yum module outputs
@@ -387,7 +357,6 @@
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: try to install non existing file
@@ -401,7 +370,7 @@
   assert:
     that:
         - "yum_result is failed"
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: verify yum module outputs
   assert:
@@ -420,7 +389,7 @@
   assert:
     that:
         - "yum_result is failed"
-        - "not yum_result.changed"
+        - "not yum_result is changed"
 
 - name: verify yum module outputs
   assert:
@@ -454,7 +423,7 @@
 - name: verify httpd not installed
   assert:
     that:
-      - "not yum_result.changed"
+      - "not yum_result is changed"
       - "'Packages providing httpd not installed due to update_only specified' in yum_result.results"
 
 - name: try to install not compatible arch rpm, should fail
@@ -467,8 +436,7 @@
 - name: verify that yum failed
   assert:
     that:
-        - "yum_result.rc == 1"
-        - "not yum_result.changed"
+        - "not yum_result is changed"
         - "yum_result is failed"
 
 # setup for testing installing an RPM from url
@@ -499,8 +467,8 @@
 - name: verify installation
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
+        - "yum_result is success"
+        - "yum_result is changed"
         - "yum_result is not failed"
 
 - name: verify yum module outputs
@@ -508,7 +476,6 @@
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: install the downloaded rpm again
@@ -520,8 +487,8 @@
 - name: verify installation
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "not yum_result.changed"
+        - "yum_result is success"
+        - "not yum_result is changed"
         - "yum_result is not failed"
 
 - name: verify yum module outputs
@@ -529,7 +496,6 @@
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: clean up
@@ -546,8 +512,8 @@
 - name: verify installation
   assert:
     that:
-        - "yum_result.rc == 0"
-        - "yum_result.changed"
+        - "yum_result is success"
+        - "yum_result is changed"
         - "yum_result is not failed"
 
 - name: verify yum module outputs
@@ -555,7 +521,6 @@
     that:
         - "'changed' in yum_result"
         - "'msg' in yum_result"
-        - "'rc' in yum_result"
         - "'results' in yum_result"
 
 - name: Create a temp RPM file which does not contain nevra information