mirror of https://github.com/ansible/ansible.git
Fix inventory cache interface (#50446)
* Replace InventoryFileCacheModule with a better developer-interface Use new interface for inventory plugins with backwards compatibility Auto-update the backing cache-plugin if the cache has changed after parsing the inventory plugin * Update CacheModules to use the config system and add a deprecation warning if they are being imported directly rather than using cache_loader * Fix foreman inventory caching * Add tests * Add integration test to check that fact caching works normally with cache plugins using ansible.constants and inventory caching provides a helpful error for non-compatible cache plugins * Add some developer documentation for inventory and cache plugins * Add user documentation for inventory caching * Add deprecation docs * Apply suggestions from docs review * Add changelogpull/53455/head
parent
831f068f98
commit
9687879840
@ -0,0 +1,9 @@
|
|||||||
|
minor_changes:
|
||||||
|
- inventory plugins - Inventory plugins that support caching can now use any cache plugin
|
||||||
|
shipped with Ansible.
|
||||||
|
deprecated_features:
|
||||||
|
- inventory plugins - Inventory plugins using self.cache is deprecated and will be removed
|
||||||
|
in 2.12. Inventory plugins should use self._cache as a dictionary to store results.
|
||||||
|
- cache plugins - Importing cache plugins directly is deprecated and will be removed in 2.12.
|
||||||
|
Cache plugins should use the cache_loader instead so cache options can be reconciled via
|
||||||
|
the configuration system rather than constants.
|
@ -0,0 +1,31 @@
|
|||||||
|
---
|
||||||
|
- hosts: localhost
|
||||||
|
vars:
|
||||||
|
foreman_stub_host: "{{ lookup('env', 'FOREMAN_HOST') }}"
|
||||||
|
foreman_stub_port: "{{ lookup('env', 'FOREMAN_PORT') }}"
|
||||||
|
foreman_stub_api_path: /api/v2
|
||||||
|
cached_hosts_key: "http://{{ foreman_stub_host }}:{{ foreman_stub_port }}{{ foreman_stub_api_path }}/hosts"
|
||||||
|
tasks:
|
||||||
|
- name: verify a cache file was created
|
||||||
|
find:
|
||||||
|
path:
|
||||||
|
- ./foreman_cache
|
||||||
|
register: matching_files
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- matching_files.matched == 1
|
||||||
|
- name: read the cached inventory
|
||||||
|
set_fact:
|
||||||
|
contents: "{{ lookup('file', matching_files.files.0.path) }}"
|
||||||
|
|
||||||
|
- name: extract all the host names
|
||||||
|
set_fact:
|
||||||
|
cached_hosts: "{{ contents[cached_hosts_key] | json_query('[*].name') }}"
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
"'{{ item }}' in cached_hosts"
|
||||||
|
loop:
|
||||||
|
- "v6.example-780.com"
|
||||||
|
- "c4.j1.y5.example-487.com"
|
@ -0,0 +1,2 @@
|
|||||||
|
shippable/posix/group3
|
||||||
|
skip/osx
|
@ -0,0 +1 @@
|
|||||||
|
# inventory config file for consistent source
|
@ -0,0 +1,141 @@
|
|||||||
|
# (c) 2014, Brian Coca, Josh Drake, et al
|
||||||
|
# (c) 2017 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
DOCUMENTATION = '''
|
||||||
|
cache: redis
|
||||||
|
short_description: Use Redis DB for cache
|
||||||
|
description:
|
||||||
|
- This cache uses JSON formatted, per host records saved in Redis.
|
||||||
|
version_added: "1.9"
|
||||||
|
requirements:
|
||||||
|
- redis>=2.4.5 (python lib)
|
||||||
|
options:
|
||||||
|
_uri:
|
||||||
|
description:
|
||||||
|
- A colon separated string of connection information for Redis.
|
||||||
|
required: True
|
||||||
|
env:
|
||||||
|
- name: ANSIBLE_CACHE_PLUGIN_CONNECTION
|
||||||
|
ini:
|
||||||
|
- key: fact_caching_connection
|
||||||
|
section: defaults
|
||||||
|
_prefix:
|
||||||
|
description: User defined prefix to use when creating the DB entries
|
||||||
|
env:
|
||||||
|
- name: ANSIBLE_CACHE_PLUGIN_PREFIX
|
||||||
|
ini:
|
||||||
|
- key: fact_caching_prefix
|
||||||
|
section: defaults
|
||||||
|
_timeout:
|
||||||
|
default: 86400
|
||||||
|
description: Expiration timeout for the cache plugin data
|
||||||
|
env:
|
||||||
|
- name: ANSIBLE_CACHE_PLUGIN_TIMEOUT
|
||||||
|
ini:
|
||||||
|
- key: fact_caching_timeout
|
||||||
|
section: defaults
|
||||||
|
type: integer
|
||||||
|
'''
|
||||||
|
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
|
||||||
|
from ansible import constants as C
|
||||||
|
from ansible.errors import AnsibleError
|
||||||
|
from ansible.plugins.cache import BaseCacheModule
|
||||||
|
|
||||||
|
try:
|
||||||
|
from redis import StrictRedis, VERSION
|
||||||
|
except ImportError:
|
||||||
|
raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
|
||||||
|
|
||||||
|
|
||||||
|
class CacheModule(BaseCacheModule):
|
||||||
|
"""
|
||||||
|
A caching module backed by redis.
|
||||||
|
Keys are maintained in a zset with their score being the timestamp
|
||||||
|
when they are inserted. This allows for the usage of 'zremrangebyscore'
|
||||||
|
to expire keys. This mechanism is used or a pattern matched 'scan' for
|
||||||
|
performance.
|
||||||
|
"""
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
if C.CACHE_PLUGIN_CONNECTION:
|
||||||
|
connection = C.CACHE_PLUGIN_CONNECTION.split(':')
|
||||||
|
else:
|
||||||
|
connection = []
|
||||||
|
|
||||||
|
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
|
||||||
|
self._prefix = C.CACHE_PLUGIN_PREFIX
|
||||||
|
self._cache = {}
|
||||||
|
self._db = StrictRedis(*connection)
|
||||||
|
self._keys_set = 'ansible_cache_keys'
|
||||||
|
|
||||||
|
def _make_key(self, key):
|
||||||
|
return self._prefix + key
|
||||||
|
|
||||||
|
def get(self, key):
|
||||||
|
|
||||||
|
if key not in self._cache:
|
||||||
|
value = self._db.get(self._make_key(key))
|
||||||
|
# guard against the key not being removed from the zset;
|
||||||
|
# this could happen in cases where the timeout value is changed
|
||||||
|
# between invocations
|
||||||
|
if value is None:
|
||||||
|
self.delete(key)
|
||||||
|
raise KeyError
|
||||||
|
self._cache[key] = json.loads(value)
|
||||||
|
|
||||||
|
return self._cache.get(key)
|
||||||
|
|
||||||
|
def set(self, key, value):
|
||||||
|
|
||||||
|
value2 = json.dumps(value)
|
||||||
|
if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire'
|
||||||
|
self._db.setex(self._make_key(key), int(self._timeout), value2)
|
||||||
|
else:
|
||||||
|
self._db.set(self._make_key(key), value2)
|
||||||
|
|
||||||
|
if VERSION[0] == 2:
|
||||||
|
self._db.zadd(self._keys_set, time.time(), key)
|
||||||
|
else:
|
||||||
|
self._db.zadd(self._keys_set, {key: time.time()})
|
||||||
|
self._cache[key] = value
|
||||||
|
|
||||||
|
def _expire_keys(self):
|
||||||
|
if self._timeout > 0:
|
||||||
|
expiry_age = time.time() - self._timeout
|
||||||
|
self._db.zremrangebyscore(self._keys_set, 0, expiry_age)
|
||||||
|
|
||||||
|
def keys(self):
|
||||||
|
self._expire_keys()
|
||||||
|
return self._db.zrange(self._keys_set, 0, -1)
|
||||||
|
|
||||||
|
def contains(self, key):
|
||||||
|
self._expire_keys()
|
||||||
|
return (self._db.zrank(self._keys_set, key) is not None)
|
||||||
|
|
||||||
|
def delete(self, key):
|
||||||
|
if key in self._cache:
|
||||||
|
del self._cache[key]
|
||||||
|
self._db.delete(self._make_key(key))
|
||||||
|
self._db.zrem(self._keys_set, key)
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
for key in self.keys():
|
||||||
|
self.delete(key)
|
||||||
|
|
||||||
|
def copy(self):
|
||||||
|
# TODO: there is probably a better way to do this in redis
|
||||||
|
ret = dict()
|
||||||
|
for key in self.keys():
|
||||||
|
ret[key] = self.get(key)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def __getstate__(self):
|
||||||
|
return dict()
|
||||||
|
|
||||||
|
def __setstate__(self, data):
|
||||||
|
self.__init__()
|
@ -0,0 +1,59 @@
|
|||||||
|
# Copyright (c) 2019 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
DOCUMENTATION = '''
|
||||||
|
name: test
|
||||||
|
plugin_type: inventory
|
||||||
|
short_description: test inventory source
|
||||||
|
extends_documentation_fragment:
|
||||||
|
- inventory_cache
|
||||||
|
'''
|
||||||
|
|
||||||
|
from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
|
||||||
|
|
||||||
|
|
||||||
|
class InventoryModule(BaseInventoryPlugin, Cacheable):
|
||||||
|
|
||||||
|
NAME = 'test'
|
||||||
|
|
||||||
|
def populate(self, hosts):
|
||||||
|
for host in list(hosts.keys()):
|
||||||
|
self.inventory.add_host(host, group='all')
|
||||||
|
for hostvar, hostval in hosts[host].items():
|
||||||
|
self.inventory.set_variable(host, hostvar, hostval)
|
||||||
|
|
||||||
|
def get_hosts(self):
|
||||||
|
return {'host1': {'one': 'two'}, 'host2': {'three': 'four'}}
|
||||||
|
|
||||||
|
def parse(self, inventory, loader, path, cache=True):
|
||||||
|
super(InventoryModule, self).parse(inventory, loader, path)
|
||||||
|
|
||||||
|
self.load_cache_plugin()
|
||||||
|
|
||||||
|
cache_key = self.get_cache_key(path)
|
||||||
|
|
||||||
|
# cache may be True or False at this point to indicate if the inventory is being refreshed
|
||||||
|
# get the user's cache option
|
||||||
|
cache_setting = self.get_option('cache')
|
||||||
|
|
||||||
|
attempt_to_read_cache = cache_setting and cache
|
||||||
|
cache_needs_update = cache_setting and not cache
|
||||||
|
|
||||||
|
# attempt to read the cache if inventory isn't being refreshed and the user has caching enabled
|
||||||
|
if attempt_to_read_cache:
|
||||||
|
try:
|
||||||
|
results = self._cache[cache_key]
|
||||||
|
except KeyError:
|
||||||
|
# This occurs if the cache_key is not in the cache or if the cache_key expired, so the cache needs to be updated
|
||||||
|
cache_needs_update = True
|
||||||
|
|
||||||
|
if cache_needs_update:
|
||||||
|
results = self.get_hosts()
|
||||||
|
|
||||||
|
# set the cache
|
||||||
|
self._cache[cache_key] = results
|
||||||
|
|
||||||
|
self.populate(results)
|
@ -0,0 +1,91 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# We don't set -u here, due to pypa/virtualenv#150
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
|
||||||
|
|
||||||
|
trap 'rm -rf "${MYTMPDIR}"' EXIT
|
||||||
|
|
||||||
|
# This is needed for the ubuntu1604py3 tests
|
||||||
|
# Ubuntu patches virtualenv to make the default python2
|
||||||
|
# but for the python3 tests we need virtualenv to use python3
|
||||||
|
PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
|
||||||
|
|
||||||
|
virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/redis-cache"
|
||||||
|
source "${MYTMPDIR}/redis-cache/bin/activate"
|
||||||
|
|
||||||
|
# Run test if dependencies are installed
|
||||||
|
failed_dep_1=$(ansible localhost -m pip -a "name=redis>=2.4.5 state=present" "$@" | tee out.txt | grep -c 'FAILED!' || true)
|
||||||
|
cat out.txt
|
||||||
|
|
||||||
|
installed_redis=$(ansible localhost -m package -a "name=redis-server state=present" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
|
||||||
|
failed_dep_2=$(grep out.txt -ce 'FAILED!' || true)
|
||||||
|
cat out.txt
|
||||||
|
|
||||||
|
started_redis=$(ansible localhost -m service -a "name=redis-server state=started" --become "$@" | tee out.txt | grep -c '"changed": true' || true)
|
||||||
|
failed_dep_3=$(grep out.txt -ce 'FAILED!' || true)
|
||||||
|
cat out.txt
|
||||||
|
|
||||||
|
CLEANUP_REDIS () { if [ "${installed_redis}" -eq 1 ] ; then ansible localhost -m package -a "name=redis-server state=absent" --become ; fi }
|
||||||
|
STOP_REDIS () { if [ "${installed_redis}" -ne 1 ] && [ "${started_redis}" -eq 1 ] ; then ansible localhost -m service -a "name=redis-server state=stopped" --become ; fi }
|
||||||
|
|
||||||
|
if [ "${failed_dep_1}" -eq 1 ] || [ "${failed_dep_2}" -eq 1 ] || [ "${failed_dep_3}" -eq 1 ] ; then
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
export ANSIBLE_CACHE_PLUGIN=redis
|
||||||
|
export ANSIBLE_CACHE_PLUGIN_CONNECTION=localhost:6379:0
|
||||||
|
export ANSIBLE_CACHE_PLUGINS=./plugins/cache
|
||||||
|
|
||||||
|
# Use old redis for fact caching
|
||||||
|
count=$(ansible-playbook test_fact_gathering.yml -vvv 2>&1 "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
|
||||||
|
failed_dep_version=$(grep out.txt -ce "'redis' python module (version 2.4.5 or newer) is required" || true)
|
||||||
|
cat out.txt
|
||||||
|
if [ "${failed_dep_version}" -eq 1 ] ; then
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "${count}" -ne 1 ] ; then
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Attempt to use old redis for inventory caching; should not work
|
||||||
|
export ANSIBLE_INVENTORY_CACHE=True
|
||||||
|
export ANSIBLE_INVENTORY_CACHE_PLUGIN=redis
|
||||||
|
export ANSIBLE_INVENTORY_ENABLED=test
|
||||||
|
export ANSIBLE_INVENTORY_PLUGINS=./plugins/inventory
|
||||||
|
|
||||||
|
ansible-inventory -i inventory_config --graph 2>&1 "$@" | tee out.txt | grep 'Cache options were provided but may not reconcile correctly unless set via set_options'
|
||||||
|
res=$?
|
||||||
|
cat out.txt
|
||||||
|
if [ "${res}" -eq 1 ] ; then
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use new style redis for fact caching
|
||||||
|
unset ANSIBLE_CACHE_PLUGINS
|
||||||
|
count=$(ansible-playbook test_fact_gathering.yml -vvv "$@" | tee out.txt | grep -c 'Gathering Facts' || true)
|
||||||
|
cat out.txt
|
||||||
|
if [ "${count}" -ne 1 ] ; then
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use new redis for inventory caching
|
||||||
|
ansible-inventory -i inventory_config --graph "$@" 2>&1 | tee out.txt | grep 'host2'
|
||||||
|
res=$?
|
||||||
|
cat out.txt
|
||||||
|
|
||||||
|
STOP_REDIS
|
||||||
|
CLEANUP_REDIS
|
||||||
|
|
||||||
|
exit $res
|
@ -0,0 +1,8 @@
|
|||||||
|
---
|
||||||
|
- hosts: localhost
|
||||||
|
connection: local
|
||||||
|
gather_facts: no
|
||||||
|
|
||||||
|
- hosts: localhost
|
||||||
|
connection: local
|
||||||
|
gather_facts: yes
|
Loading…
Reference in New Issue