Re-enable old_style_cache_plugin test target (#75748)

Fix tests for cache plugins written before ConfigManager

test/integration/targets/old_style_cache_plugins/aliases

@@ -1,5 +1,6 @@
+destructive
+needs/root
 shippable/posix/group3
 context/controller
 skip/osx
 skip/macos
-disabled

test/integration/targets/old_style_cache_plugins/cleanup.yml

@@ -0,0 +1,41 @@
+---
+- hosts: localhost
+  gather_facts: no
+  ignore_errors: yes
+  tasks:
+    - command: redis-cli keys
+
+    - name: delete cache keys
+      command: redis-cli del {{ item }}
+      loop:
+        - ansible_facts_localhost
+        - ansible_inventory_localhost
+        - ansible_cache_keys
+
+    - name: shutdown the server
+      command: redis-cli shutdown
+
+    - name: cleanup set up files
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - redis-stable.tar.gz
+
+    - name: remove executables
+      file:
+        state: absent
+        path: "/usr/local/bin/{{ item }}"
+        follow: no
+      become: yes
+      loop:
+        - redis-server
+        - redis-cli
+
+    - name: clean the rest of the files
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - ./redis-stable.tar.gz
+        - ./redis-stable

test/integration/targets/old_style_cache_plugins/inspect_cache.yml

@@ -0,0 +1,36 @@
+---
+- hosts: localhost
+  gather_facts: no
+  vars:
+    json_cache: "{{ cache.stdout | from_json }}"
+  tasks:
+    - command: redis-cli get ansible_facts_localhost
+      register: cache
+      tags:
+        - always
+
+    - name: test that the cache only contains the set_fact var
+      assert:
+        that:
+          - "json_cache | length  == 1"
+          - "json_cache.foo ==  ansible_facts.foo"
+      tags:
+        - set_fact
+
+    - name: test that the cache contains gathered facts and the var
+      assert:
+        that:
+          - "json_cache | length  > 1"
+          - "json_cache.foo == 'bar'"
+          - "json_cache.ansible_distribution is defined"
+      tags:
+        - additive_gather_facts
+
+    - name: test that the cache contains only gathered facts
+      assert:
+        that:
+          - "json_cache | length > 1"
+          - "json_cache.foo is undefined"
+          - "json_cache.ansible_distribution is defined"
+      tags:
+        - gather_facts

test/integration/targets/old_style_cache_plugins/plugins/cache/configurable_redis.py

@@ -0,0 +1,147 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    cache: configurable_redis
+    short_description: Use Redis DB for cache
+    description:
+        - This cache uses JSON formatted, per host records saved in Redis.
+    version_added: "1.9"
+    requirements:
+      - redis>=2.4.5 (python lib)
+    options:
+      _uri:
+        description:
+          - A colon separated string of connection information for Redis.
+        required: True
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_CONNECTION
+        ini:
+          - key: fact_caching_connection
+            section: defaults
+      _prefix:
+        description: User defined prefix to use when creating the DB entries
+        default: ansible_facts
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_PREFIX
+        ini:
+          - key: fact_caching_prefix
+            section: defaults
+      _timeout:
+        default: 86400
+        description: Expiration timeout for the cache plugin data
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_TIMEOUT
+        ini:
+          - key: fact_caching_timeout
+            section: defaults
+        type: integer
+'''
+
+import time
+import json
+
+from ansible import constants as C
+from ansible.errors import AnsibleError
+from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
+from ansible.plugins.cache import BaseCacheModule
+from ansible.utils.display import Display
+
+try:
+    from redis import StrictRedis, VERSION
+except ImportError:
+    raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
+
+display = Display()
+
+
+class CacheModule(BaseCacheModule):
+    """
+    A caching module backed by redis.
+    Keys are maintained in a zset with their score being the timestamp
+    when they are inserted. This allows for the usage of 'zremrangebyscore'
+    to expire keys. This mechanism is used or a pattern matched 'scan' for
+    performance.
+    """
+    def __init__(self, *args, **kwargs):
+        connection = []
+
+        super(CacheModule, self).__init__(*args, **kwargs)
+        if self.get_option('_uri'):
+            connection = self.get_option('_uri').split(':')
+        self._timeout = float(self.get_option('_timeout'))
+        self._prefix = self.get_option('_prefix')
+
+        self._cache = {}
+        self._db = StrictRedis(*connection)
+        self._keys_set = 'ansible_cache_keys'
+
+    def _make_key(self, key):
+        return self._prefix + key
+
+    def get(self, key):
+
+        if key not in self._cache:
+            value = self._db.get(self._make_key(key))
+            # guard against the key not being removed from the zset;
+            # this could happen in cases where the timeout value is changed
+            # between invocations
+            if value is None:
+                self.delete(key)
+                raise KeyError
+            self._cache[key] = json.loads(value, cls=AnsibleJSONDecoder)
+
+        return self._cache.get(key)
+
+    def set(self, key, value):
+
+        value2 = json.dumps(value, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
+        if self._timeout > 0:  # a timeout of 0 is handled as meaning 'never expire'
+            self._db.setex(self._make_key(key), int(self._timeout), value2)
+        else:
+            self._db.set(self._make_key(key), value2)
+
+        if VERSION[0] == 2:
+            self._db.zadd(self._keys_set, time.time(), key)
+        else:
+            self._db.zadd(self._keys_set, {key: time.time()})
+        self._cache[key] = value
+
+    def _expire_keys(self):
+        if self._timeout > 0:
+            expiry_age = time.time() - self._timeout
+            self._db.zremrangebyscore(self._keys_set, 0, expiry_age)
+
+    def keys(self):
+        self._expire_keys()
+        return self._db.zrange(self._keys_set, 0, -1)
+
+    def contains(self, key):
+        self._expire_keys()
+        return (self._db.zrank(self._keys_set, key) is not None)
+
+    def delete(self, key):
+        if key in self._cache:
+            del self._cache[key]
+        self._db.delete(self._make_key(key))
+        self._db.zrem(self._keys_set, key)
+
+    def flush(self):
+        for key in self.keys():
+            self.delete(key)
+
+    def copy(self):
+        # TODO: there is probably a better way to do this in redis
+        ret = dict()
+        for key in self.keys():
+            ret[key] = self.get(key)
+        return ret
+
+    def __getstate__(self):
+        return dict()
+
+    def __setstate__(self, data):
+        self.__init__()

test/integration/targets/old_style_cache_plugins/runme.sh

@@ -4,77 +4,44 @@ set -eux
 
 source virtualenv.sh
 
-# Run test if dependencies are installed
-failed_dep_1=$(ansible localhost -m pip -a "name=redis>=2.4.5 state=present" "$@" | tee out.txt | grep -c 'FAILED!' || true)
-cat out.txt
+trap 'ansible-playbook cleanup.yml' EXIT
 
-installed_redis=$(ansible localhost -m package -a "name=redis-server state=present" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
-failed_dep_2=$(grep out.txt -ce 'FAILED!' || true)
-cat out.txt
+export PATH="$PATH:/usr/local/bin"
 
-started_redis=$(ansible localhost -m service -a "name=redis-server state=started" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
-failed_dep_3=$(grep out.txt -ce 'FAILED!' || true)
-cat out.txt
+ansible-playbook setup_redis_cache.yml "$@"
 
-CLEANUP_REDIS () { if [ "${installed_redis}" -eq 1 ] ; then ansible localhost -m package -a "name=redis-server state=absent" --become ; fi }
-STOP_REDIS () { if [ "${installed_redis}" -ne 1 ] && [ "${started_redis}" -eq 1 ] ; then ansible localhost -m service -a "name=redis-server state=stopped" --become ; fi }
+# Cache should start empty
+redis-cli keys ansible_
+[ "$(redis-cli keys ansible_)" = "" ]
 
-if [ "${failed_dep_1}" -eq 1 ] || [ "${failed_dep_2}" -eq 1 ] || [ "${failed_dep_3}" -eq 1 ] ; then
-    STOP_REDIS
-    CLEANUP_REDIS
-    exit 0
-fi
-
-export ANSIBLE_CACHE_PLUGIN=redis
-export ANSIBLE_CACHE_PLUGIN_CONNECTION=localhost:6379:0
 export ANSIBLE_CACHE_PLUGINS=./plugins/cache
+export ANSIBLE_CACHE_PLUGIN_CONNECTION=localhost:6379:0
+export ANSIBLE_CACHE_PLUGIN_PREFIX='ansible_facts_'
+
+# Test legacy cache plugins (that use ansible.constants) and
+# new cache plugins that use config manager both work for facts.
+for fact_cache in legacy_redis configurable_redis; do
 
-# Use old redis for fact caching
-count=$(ansible-playbook test_fact_gathering.yml -vvv 2>&1 "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
-failed_dep_version=$(grep out.txt -ce "'redis' python module (version 2.4.5 or newer) is required" || true)
-cat out.txt
-if [ "${failed_dep_version}" -eq 1 ] ; then
-    STOP_REDIS
-    CLEANUP_REDIS
-    exit 0
-fi
-if [ "${count}" -ne 1 ] ; then
-    STOP_REDIS
-    CLEANUP_REDIS
-    exit 1
-fi
+    export ANSIBLE_CACHE_PLUGIN="$fact_cache"
 
-# Attempt to use old redis for inventory caching; should not work
-export ANSIBLE_INVENTORY_CACHE=True
-export ANSIBLE_INVENTORY_CACHE_PLUGIN=redis
-export ANSIBLE_INVENTORY_ENABLED=test
-export ANSIBLE_INVENTORY_PLUGINS=./plugins/inventory
+    # test set_fact with cacheable: true
+    ansible-playbook test_fact_gathering.yml --tags set_fact "$@"
+    [ "$(redis-cli keys ansible_facts_localhost | wc -l)" -eq 1 ]
+    ansible-playbook inspect_cache.yml --tags set_fact "$@"
 
-ansible-inventory -i inventory_config --graph 2>&1 "$@" | tee out.txt | grep 'Cache options were provided but may not reconcile correctly unless set via set_options'
-res=$?
-cat out.txt
-if [ "${res}" -eq 1 ] ; then
-    STOP_REDIS
-    CLEANUP_REDIS
-    exit 1
-fi
+    # cache gathered facts in addition
+    ansible-playbook test_fact_gathering.yml --tags gather_facts "$@"
+    ansible-playbook inspect_cache.yml --tags additive_gather_facts "$@"
 
-# Use new style redis for fact caching
-unset ANSIBLE_CACHE_PLUGINS
-count=$(ansible-playbook test_fact_gathering.yml -vvv "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
-cat out.txt
-if [ "${count}" -ne 1 ] ; then
-    STOP_REDIS
-    CLEANUP_REDIS
-    exit 1
-fi
+    # flush cache and only cache gathered facts
+    ansible-playbook test_fact_gathering.yml --flush-cache --tags gather_facts --tags flush "$@"
+    ansible-playbook inspect_cache.yml --tags gather_facts "$@"
 
-# Use new redis for inventory caching
-ansible-inventory -i inventory_config --graph "$@" 2>&1 | tee out.txt | grep 'host2'
-res=$?
-cat out.txt
+    redis-cli del ansible_facts_localhost
+    unset ANSIBLE_CACHE_PLUGIN
 
-STOP_REDIS
-CLEANUP_REDIS
+done
 
-exit $res
+# Legacy cache plugins need to be updated to use set_options/get_option to be compatible with inventory plugins.
+# Inventory plugins load cache options with the config manager.
+ansible-playbook test_inventory_cache.yml "$@"

test/integration/targets/old_style_cache_plugins/setup_redis_cache.yml

@@ -0,0 +1,51 @@
+---
+- hosts: localhost
+  vars:
+    make: "{{ ( ansible_distribution != 'FreeBSD' ) | ternary('make', 'gmake') }}"
+  tasks:
+    - name: name ensure make is available
+      command: "which {{ make }}"
+      register: has_make
+      ignore_errors: yes
+
+    - command: apk add --no-cache make
+      when: "has_make is failed and ansible_distribution == 'Alpine'"
+      become: yes
+
+    - package:
+        name: "{{ make }}"
+        state: present
+      become: yes
+      when: "has_make is failed and ansible_distribution != 'Alpine'"
+
+    - name: get the latest stable redis server release
+      get_url:
+        url: http://download.redis.io/redis-stable.tar.gz
+        dest: ./
+
+    - name: unzip download
+      unarchive:
+        src: redis-stable.tar.gz
+        dest: ./
+
+    - command: "{{ make }}"
+      args:
+        chdir: redis-stable
+
+    - name: copy the executable into the path
+      copy:
+        src: "redis-stable/src/{{ item }}"
+        dest: /usr/local/bin/
+        mode: 755
+      become: yes
+      loop:
+        - redis-server
+        - redis-cli
+
+    - name: start the redis server in the background
+      command: redis-server --daemonize yes
+
+    - name: install dependency for the cache plugin
+      pip:
+        name: redis>2.4.5
+        state: present

test/integration/targets/old_style_cache_plugins/test_fact_gathering.yml

@@ -1,6 +1,22 @@
 ---
 - hosts: localhost
   gather_facts: no
+  tags:
+    - flush
+  tasks:
+    - meta: clear_facts
 
 - hosts: localhost
   gather_facts: yes
+  gather_subset: min
+  tags:
+    - gather_facts
+
+- hosts: localhost
+  gather_facts: no
+  tags:
+    - set_fact
+  tasks:
+    - set_fact:
+        foo: bar
+        cacheable: true

test/integration/targets/old_style_cache_plugins/test_inventory_cache.yml

@@ -0,0 +1,45 @@
+- hosts: localhost
+  gather_facts: no
+  vars:
+    reset_color: '\x1b\[0m'
+    color: '\x1b\[[0-9];[0-9]{2}m'
+    base_environment:
+      ANSIBLE_INVENTORY_PLUGINS: ./plugins/inventory
+      ANSIBLE_INVENTORY_ENABLED: test
+      ANSIBLE_INVENTORY_CACHE: true
+      ANSIBLE_CACHE_PLUGINS: ./plugins/cache
+      ANSIBLE_CACHE_PLUGIN_CONNECTION: localhost:6379:0
+      ANSIBLE_CACHE_PLUGIN_PREFIX: 'ansible_inventory_'
+    legacy_cache:
+      ANSIBLE_INVENTORY_CACHE_PLUGIN: legacy_redis
+    updated_cache:
+      ANSIBLE_INVENTORY_CACHE_PLUGIN: configurable_redis
+  tasks:
+    - name: legacy-style cache plugin should cause a warning
+      command: ansible-inventory -i inventory_config --graph
+      register: result
+      environment: "{{ base_environment | combine(legacy_cache) }}"
+
+    - name: test warning message
+      assert:
+        that:
+          - expected_warning in warning
+          - "'No inventory was parsed, only implicit localhost is available' in warning"
+      vars:
+        warning: "{{ result.stderr | regex_replace(reset_color) | regex_replace(color) | regex_replace('\\n', ' ') }}"
+        expected_warning: "Cache options were provided but may not reconcile correctly unless set via set_options"
+
+    - name: cache plugin updated to use config manager should work
+      command: ansible-inventory -i inventory_config --graph
+      register: result
+      environment: "{{ base_environment | combine(updated_cache) }}"
+
+    - name: test warning message
+      assert:
+        that:
+          - unexpected_warning not in warning
+          - "'No inventory was parsed, only implicit localhost is available' not in warning"
+          - '"host1" in result.stdout'
+      vars:
+        warning: "{{ result.stderr | regex_replace(reset_color) | regex_replace(color) | regex_replace('\\n', ' ') }}"
+        unexpected_warning: "Cache options were provided but may not reconcile correctly unless set via set_options"