docker modules: make sure everything works with older docker-py versions (#55258)

* General test improvements.

* Adjust tests to older docker-py versions.

* docker_swarm_server_info: work around problems with older docker-py versions

* Bump minimal docker-py version for options network_filters and disk_usage.

* More general test improvements.

* Correct usage of docker_image.

* Put files into output directory.

* Speed up test.

* Remove old check.

changelogs/fragments/55258-docker_host_info-old-docker-py.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- "docker_host_info - ``network_filters`` needs docker-py 2.0.2, ``disk_usage`` needs docker-py 2.2.0."

changelogs/fragments/55258-docker_swarm_service_info-old-docker-py.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- "docker_swarm_service_info - work around problems with older docker-py versions such as 2.0.2."

lib/ansible/module_utils/docker/swarm.py

@@ -6,7 +6,7 @@ import json
 from time import sleep
 
 try:
-    from docker.errors import APIError
+    from docker.errors import APIError, NotFound
 except ImportError:
     # missing Docker SDK for Python handled in ansible.module_utils.docker.common
     pass
@@ -259,15 +259,16 @@ class AnsibleDockerSwarmClient(AnsibleDockerClient):
             Single service information structure
         """
         try:
-            service_info = self.inspect_service(service=service_id)
+            service_info = self.inspect_service(service_id)
+        except NotFound as exc:
+            if skip_missing is False:
+                self.fail("Error while reading from Swarm manager: %s" % to_native(exc))
+            else:
+                return None
         except APIError as exc:
             if exc.status_code == 503:
                 self.fail("Cannot inspect service: To inspect service execute module on Swarm Manager")
-            if exc.status_code == 404:
-                if skip_missing is False:
-                    self.fail("Error while reading from Swarm manager: %s" % to_native(exc))
-                else:
-                    return None
+            self.fail("Error inspecting swarm service: %s" % exc)
         except Exception as exc:
             self.fail("Error inspecting swarm service: %s" % exc)
 

lib/ansible/modules/cloud/docker/docker_host_info.py

@@ -245,15 +245,18 @@ class DockerHostManager(DockerBaseClass):
         header_images = ['Id', 'RepoTags', 'Created', 'Size']
         header_networks = ['Id', 'Driver', 'Name', 'Scope']
 
+        filter_arg = dict()
+        if filters:
+            filter_arg['filters'] = filters
         try:
             if docker_object == 'containers':
-                items = self.client.containers(filters=filters)
+                items = self.client.containers(**filter_arg)
             elif docker_object == 'networks':
-                items = self.client.networks(filters=filters)
+                items = self.client.networks(**filter_arg)
             elif docker_object == 'images':
-                items = self.client.images(filters=filters)
+                items = self.client.images(**filter_arg)
             elif docker_object == 'volumes':
-                items = self.client.volumes(filters=filters)
+                items = self.client.volumes(**filter_arg)
         except APIError as exc:
             self.client.fail("Error inspecting docker host for object '%s': %s" %
                              (docker_object, to_native(exc)))
@@ -301,11 +304,17 @@ def main():
         verbose_output=dict(type='bool', default=False),
     )
 
+    option_minimal_versions = dict(
+        network_filters=dict(docker_py_version='2.0.2'),
+        disk_usage=dict(docker_py_version='2.2.0'),
+    )
+
     client = AnsibleDockerClient(
         argument_spec=argument_spec,
         supports_check_mode=True,
         min_docker_version='1.10.0',
         min_docker_api_version='1.21',
+        option_minimal_versions=option_minimal_versions,
         fail_results=dict(
             can_talk_to_docker=False,
         ),

test/integration/targets/docker_container/tasks/tests/options.yml

@@ -937,6 +937,7 @@
     dns_search_domains:
     - example.org
     - example.com
+    force_kill: yes
   register: dns_search_domains_3
 
 - name: dns_search_domains (changed elements)
@@ -1000,6 +1001,7 @@
     dns_servers:
     - 8.8.8.8
     - 1.1.1.1
+    force_kill: yes
   register: dns_servers_3
 
 - name: dns_servers (changed elements)

test/integration/targets/docker_container_info/tasks/main.yml

@@ -27,7 +27,6 @@
       image: alpine:3.8
       command: '/bin/sh -c "sleep 10m"'
       state: started
-      auto_remove: yes
       force_kill: yes
 
   - name: Inspect a present container
@@ -45,12 +44,6 @@
   - name: Dump docker inspect result
     debug: var=docker_inspect_result
 
-  - name: Cleanup
-    docker_container:
-      name: "{{ cname }}"
-      state: absent
-      force_kill: yes
-
   - assert:
       that:
       - result.exists
@@ -58,6 +51,13 @@
       - "result.container"
       - "result.container == docker_inspect_result[0]"
 
+  always:
+  - name: Cleanup
+    docker_container:
+      name: "{{ cname }}"
+      state: absent
+      force_kill: yes
+
   when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.20', '>=')
 
 - fail: msg="Too old docker / docker-py version to run docker_container_info tests!"

test/integration/targets/docker_host_info/tasks/test_host_info.yml

@@ -183,6 +183,7 @@
     docker_host_info:
       disk_usage: yes
     register: output
+    ignore_errors: yes
 
   - name: assert reading docker host facts when docker is running and get disk usage
     assert:
@@ -194,12 +195,19 @@
          - 'output.images is not defined'
          - 'output.disk_usage.LayersSize is number'
          - 'output.disk_usage.BuilderSize is not defined'
+    when: docker_py_version is version('2.2.0', '>=')
+  - assert:
+      that:
+        - output is failed
+        - "('version is ' ~ docker_py_version ~'. Minimum version required is 2.2.0') in output.msg"
+    when: docker_py_version is version('2.2.0', '<')
 
   - name: Get info on Docker host and get disk usage with verbose output
     docker_host_info:
       disk_usage: yes
       verbose_output: yes
     register: output
+    ignore_errors: yes
 
   - name: assert reading docker host facts when docker is running and get disk usage with verbose output
     assert:
@@ -211,6 +219,12 @@
          - 'output.images is not defined'
          - 'output.disk_usage.LayersSize is number'
          - 'output.disk_usage.BuilderSize is number'
+    when: docker_py_version is version('2.2.0', '>=')
+  - assert:
+      that:
+        - output is failed
+        - "('version is ' ~ docker_py_version ~'. Minimum version required is 2.2.0') in output.msg"
+    when: docker_py_version is version('2.2.0', '<')
 
   - name: Get info on Docker host, disk usage and get all lists together
     docker_host_info:
@@ -218,7 +232,7 @@
       containers: yes
       networks: yes
       images: yes
-      disk_usage: yes
+      disk_usage: "{{ docker_py_version is version('2.2.0', '>=') }}"
     register: output
 
   - name: assert reading docker host facts when docker is running, disk usage and get lists together
@@ -233,8 +247,11 @@
          - 'output.volumes[0].Mountpoint is not defined'
          - 'output.images[0].Id is string'
          - 'output.images[0].ParentId is not defined'
+  - assert:
+      that:
          - 'output.disk_usage.LayersSize is number'
          - 'output.disk_usage.BuilderSize is not defined'
+    when: docker_py_version is version('2.2.0', '>=')
 
   - name: Get info on Docker host, disk usage and get all lists together with verbose output
     docker_host_info:
@@ -242,7 +259,7 @@
       containers: yes
       networks: yes
       images: yes
-      disk_usage: yes
+      disk_usage: "{{ docker_py_version is version('2.2.0', '>=') }}"
       verbose_output: yes
     register: output
 
@@ -258,8 +275,11 @@
          - 'output.volumes[0].Mountpoint is string'
          - 'output.images[0].Id is string'
          - 'output.images[0].ParentId is string'
+  - assert:
+      that:
          - 'output.disk_usage.LayersSize is number'
          - 'output.disk_usage.BuilderSize is number'
+    when: docker_py_version is version('2.2.0', '>=')
 
   always:
   - name: Delete container

test/integration/targets/docker_image/tasks/tests/basic.yml

@@ -46,21 +46,24 @@
 
 - name: Tag image with alias
   docker_image:
+    source: local
     name: "hello-world:latest"
     repository: "hello-world:alias"
   register: tag_1
 
 - name: Tag image with alias (idempotent)
   docker_image:
+    source: local
     name: "hello-world:latest"
     repository: "hello-world:alias"
   register: tag_2
 
 - name: Tag image with alias (force, still idempotent)
   docker_image:
+    source: local
     name: "hello-world:latest"
     repository: "hello-world:alias"
-    force: yes
+    force_tag: yes
   register: tag_3
 
 - assert:
@@ -105,7 +108,8 @@
     name: "hello-world:latest"
     repository: "{{ registry_address }}/test/hello-world"
     push: yes
-    force: yes
+    source: local
+    force_tag: yes
   register: push_3
 
 - assert:

test/integration/targets/docker_image/tasks/tests/options.yml

@@ -228,7 +228,7 @@
 - name: Archive image
   docker_image:
     name: "hello-world:latest"
-    archive_path: image.tar
+    archive_path: "{{ output_dir }}/image.tar"
     source: pull
   register: archive_image
 
@@ -241,14 +241,14 @@
 - name: load image (changed)
   docker_image:
     name: "hello-world:latest"
-    load_path: image.tar
+    load_path: "{{ output_dir }}/image.tar"
     source: load
   register: load_image
 
 - name: load image (idempotency)
   docker_image:
     name: "hello-world:latest"
-    load_path: image.tar
+    load_path: "{{ output_dir }}/image.tar"
     source: load
   register: load_image_1
 

test/integration/targets/docker_image_info/tasks/main.yml

@@ -17,7 +17,7 @@
   - name: Make sure images are there
     docker_image:
       name: "{{ item }}"
-      pull: yes
+      source: pull
       state: present
     loop:
     - "hello-world:latest"

test/integration/targets/docker_network/tasks/tests/ipam.yml

@@ -221,6 +221,7 @@
     ipam_driver_options:
       a: b
   register: network_1
+  ignore_errors: yes
 - name: Create network with IPAM driver options (idempotence)
   docker_network:
     name: "{{ nname_ipam_3 }}"
@@ -229,6 +230,7 @@
       a: b
   diff: yes
   register: network_2
+  ignore_errors: yes
 - name: Create network with IPAM driver options (change)
   docker_network:
     name: "{{ nname_ipam_3 }}"
@@ -237,6 +239,7 @@
       a: c
   diff: yes
   register: network_3
+  ignore_errors: yes
 - name: Cleanup network
   docker_network:
     name: "{{ nname_ipam_3 }}"
@@ -247,3 +250,9 @@
       - network_1 is changed
       - network_2 is not changed
       - network_3 is changed
+  when: docker_py_version is version('2.0.0', '>=')
+- assert:
+    that:
+      - network_1 is failed
+      - "('version is ' ~ docker_py_version ~'. Minimum version required is 2.0.0') in network_1.msg"
+  when: docker_py_version is version('2.0.0', '<')

test/integration/targets/docker_node/tasks/test_node.yml

@@ -19,6 +19,7 @@
   - name: Create a Swarm cluster
     docker_swarm:
       state: present
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
     register: output
 
   - name: assert changed when create a new swarm cluster

test/integration/targets/docker_node_info/tasks/test_node_info.yml

@@ -19,6 +19,7 @@
   - name: Create a Swarm cluster
     docker_swarm:
       state: present
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
     register: output
 
   - name: assert changed when create a new swarm cluster

test/integration/targets/docker_prune/tasks/main.yml

@@ -45,7 +45,8 @@
       - volume.volume.Name in result.volumes
       - "'volumes_space_reclaimed' in result"
       # builder_cache
-      - "'builder_cache_space_reclaimed' in result"
+      - "'builder_cache_space_reclaimed' in result or docker_py_version is version('3.3.0', '<')"
+      - "'builder_cache_space_reclaimed' not in result or docker_py_version is version('3.3.0', '>=')"
 
   # Test with filters
   - docker_prune:

test/integration/targets/docker_swarm/tasks/tests/options.yml

@@ -65,6 +65,7 @@
   docker_swarm:
     state: present
     force: yes
+    advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
     autolock_managers: yes
   diff: yes
   register: output_7

test/integration/targets/docker_swarm_info/tasks/test_swarm_info.yml

@@ -23,6 +23,7 @@
   - name: Create a Swarm cluster
     docker_swarm:
       state: present
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
     register: output
 
   - name: assert changed when create a new swarm cluster
@@ -135,6 +136,7 @@
     docker_swarm_info:
       unlock_key: yes
     register: output
+    ignore_errors: yes
 
   - name: assert reading swarm facts and non existing swarm unlock key
     assert:
@@ -143,17 +145,26 @@
         - 'output.can_talk_to_docker == true'
         - 'output.docker_swarm_active == true'
         - 'output.docker_swarm_manager == true'
+    when: docker_py_version is version('2.7.0', '>=')
+  - assert:
+      that:
+        - output is failed
+        - "('version is ' ~ docker_py_version ~'. Minimum version required is 2.7.0') in output.msg"
+    when: docker_py_version is version('2.7.0', '<')
 
   - name: Update swarm cluster to be locked
     docker_swarm:
       state: present
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
       autolock_managers: true
     register: autolock_managers_update_output
+    ignore_errors: yes
 
   - name: Try to get docker_swarm_info and swarm_unlock_key
     docker_swarm_info:
       unlock_key: yes
     register: output
+    ignore_errors: yes
 
   - name: assert reading swarm facts and swarm unlock key
     assert:
@@ -163,6 +174,12 @@
         - 'output.can_talk_to_docker == true'
         - 'output.docker_swarm_active == true'
         - 'output.docker_swarm_manager == true'
+    when: docker_py_version is version('2.7.0', '>=')
+  - assert:
+      that:
+        - output is failed
+        - "('version is ' ~ docker_py_version ~'. Minimum version required is 2.7.0') in output.msg"
+    when: docker_py_version is version('2.7.0', '<')
 
   always:
   - name: Cleanup

test/integration/targets/docker_swarm_service/tasks/main.yml

@@ -18,7 +18,7 @@
   - name: Create a Swarm cluster
     docker_swarm:
       state: present
-      advertise_addr: "{{ansible_default_ipv4.address}}"
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
 
   - include_tasks: run-test.yml
     with_fileglob:

test/integration/targets/docker_swarm_service/tasks/tests/options.yml

@@ -919,8 +919,8 @@
     resolve_image: no
     command: '/bin/sh -v -c "sleep 10m"'
     groups:
-      - 1234
-      - 5678
+      - "1234"
+      - "5678"
   register: groups_1
   ignore_errors: yes
 
@@ -931,8 +931,8 @@
     resolve_image: no
     command: '/bin/sh -v -c "sleep 10m"'
     groups:
-      - 1234
-      - 5678
+      - "1234"
+      - "5678"
   register: groups_2
   ignore_errors: yes
 
@@ -943,7 +943,7 @@
     resolve_image: no
     command: '/bin/sh -v -c "sleep 10m"'
     groups:
-      - 1234
+      - "1234"
   register: groups_3
   ignore_errors: yes
 
@@ -1001,7 +1001,7 @@
       test:
         - CMD
         - sleep
-        - 1
+        - "1"
       timeout: 2s
       interval: 0h0m2s3ms4us
       retries: 2
@@ -1035,7 +1035,7 @@
       test:
         - CMD
         - sleep
-        - 1
+        - "1"
       timeout: 3s
       interval: 0h1m2s3ms4us
       retries: 3
@@ -1855,6 +1855,7 @@
     command: '/bin/sh -v -c "sleep 10m"'
     resolve_image: true
   register: resolve_image_3
+  ignore_errors: yes
 
 - name: cleanup
   docker_swarm_service:
@@ -1872,7 +1873,8 @@
     that:
       - resolve_image_1 is changed
       - resolve_image_2 is not changed
-      - resolve_image_3 is not changed
+      - resolve_image_3 is failed
+      - "('version is ' ~ docker_py_version ~'. Minimum version required is 3.2.0') in resolve_image_3.msg"
   when: docker_api_version is version('1.30', '<') or docker_py_version is version('3.2.0', '<')
 
 ####################################################################

test/integration/targets/docker_swarm_service/tasks/tests/rollback_config.yml

@@ -276,12 +276,12 @@
       - rollback_config_order_1 is changed
       - rollback_config_order_2 is not changed
       - rollback_config_order_3 is changed
-  when: docker_api_version is version('1.29', '>=') and docker_py_version is version('2.7.0', '>=')
+  when: docker_api_version is version('1.29', '>=') and docker_py_version is version('3.5.0', '>=')
 - assert:
     that:
       - rollback_config_order_1 is failed
       - "'Minimum version required' in rollback_config_order_1.msg"
-  when: docker_api_version is version('1.29', '<') or docker_py_version is version('2.7.0', '<')
+  when: docker_api_version is version('1.29', '<') or docker_py_version is version('3.5.0', '<')
 
 ###################################################################
 ## rollback_config.parallelism ######################################
@@ -331,9 +331,9 @@
       - rollback_config_parallelism_1 is changed
       - rollback_config_parallelism_2 is not changed
       - rollback_config_parallelism_3 is changed
-  when: docker_api_version is version('1.28', '>=') and docker_py_version is version('2.7.0', '>=')
+  when: docker_api_version is version('1.28', '>=') and docker_py_version is version('3.5.0', '>=')
 - assert:
     that:
       - rollback_config_parallelism_1 is failed
       - "'Minimum version required' in rollback_config_parallelism_1.msg"
-  when: docker_api_version is version('1.28', '<') or docker_py_version is version('2.7.0', '<')
+  when: docker_api_version is version('1.28', '<') or docker_py_version is version('3.5.0', '<')

test/integration/targets/docker_swarm_service_info/tasks/test_docker_swarm_service_info.yml

@@ -29,6 +29,7 @@
   - name: Create a Swarm cluster
     docker_swarm:
       state: present
+      advertise_addr: "{{ansible_default_ipv4.address | default('127.0.0.1')}}"
     register: output
 
   - name: Create services
@@ -74,3 +75,9 @@
       docker_swarm:
         state: absent
         force: true
+
+  # Maximum of 1.24 (docker API version for docker_swarm_service_info) and 1.25 (docker API version for docker_swarm) is 1.25
+  when: docker_py_version is version('2.0.2', '>=') and docker_api_version is version('1.25', '>=')
+
+- fail: msg="Too old docker / docker-py version to run docker_swarm_service_info tests!"
+  when: not(docker_py_version is version('2.0.2', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6)