From 1df7d95cec36ac22929c8559f89be14346930138 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 2 Feb 2017 17:48:53 -0800 Subject: [PATCH] Module utils default path (#20913) * Make the module_utils path configurable * Add a config value to define the path site module_utils files * Handle module_utils that do not have source as an error * Make an integration test for module_utils envvar working * Add documentation for the ANSIBLE_MODULE_UTILS config option/envvar * Add it to the sample ansible.cfg * Add it to intro_configuration. * Also modify intro_configuration to place envvars on equal footing with the config options (will need to document the envvar names in the future) * Also add the ANSIBLE_LIBRARY use case from https://github.com/ansible/ansible/issues/15432 so we can close out that bug. --- docs/docsite/rst/intro_configuration.rst | 77 ++++++++++++++----- examples/ansible.cfg | 1 + lib/ansible/constants.py | 5 +- lib/ansible/executor/module_common.py | 16 +++- lib/ansible/plugins/__init__.py | 2 +- .../module_utils/library/test_env_override.py | 8 ++ .../module_utils/module_utils/service.py | 1 + .../module_utils/module_utils_envvar.yml | 51 ++++++++++++ .../module_utils/module_utils_test.yml | 3 +- .../module_utils/other_mu_dir/__init__.py | 0 .../module_utils/other_mu_dir/a/__init__.py | 0 .../module_utils/other_mu_dir/a/b/__init__.py | 0 .../other_mu_dir/a/b/c/__init__.py | 0 .../other_mu_dir/a/b/c/d/__init__.py | 0 .../other_mu_dir/a/b/c/d/e/__init__.py | 0 .../other_mu_dir/a/b/c/d/e/f/__init__.py | 0 .../other_mu_dir/a/b/c/d/e/f/g/__init__.py | 0 .../other_mu_dir/a/b/c/d/e/f/g/h/__init__.py | 1 + .../module_utils/other_mu_dir/facts.py | 1 + .../module_utils/other_mu_dir/json_utils.py | 1 + .../targets/module_utils/other_mu_dir/mork.py | 1 + .../integration/targets/module_utils/runme.sh | 1 + 22 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 test/integration/targets/module_utils/library/test_env_override.py create mode 100644 test/integration/targets/module_utils/module_utils/service.py create mode 100644 test/integration/targets/module_utils/module_utils_envvar.yml create mode 100644 test/integration/targets/module_utils/other_mu_dir/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/d/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/h/__init__.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/facts.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/json_utils.py create mode 100644 test/integration/targets/module_utils/other_mu_dir/mork.py diff --git a/docs/docsite/rst/intro_configuration.rst b/docs/docsite/rst/intro_configuration.rst index 4c9555742a1..c1760b184fd 100644 --- a/docs/docsite/rst/intro_configuration.rst +++ b/docs/docsite/rst/intro_configuration.rst @@ -52,8 +52,9 @@ You may wish to consult the `ansible.cfg in source control `_ in the source tree if you want to use these. They are mostly considered to be a legacy system as compared to the config file, but are equally valid. +Ansible also allows configuration of settings via environment variables. If +these environment variables are set, they will override any setting loaded +from the configuration file. These variables are defined in `constants.py `_. .. _config_values_by_section: @@ -521,8 +522,25 @@ This is the default location Ansible looks to find modules:: library = /usr/share/ansible -Ansible knows how to look in multiple locations if you feed it a colon separated path, and it also will look for modules in the -"./library" directory alongside a playbook. +Ansible can look in multiple locations if you feed it a colon +separated path, and it also will look for modules in the :file:`./library` +directory alongside a playbook. + +This can be used to manage modules pulled from several different locations. +For instance, a site wishing to checkout modules from several different git +repositories might handle it like this: + +.. code-block:: shell-session + + $ mkdir -p /srv/modules + $ cd /srv/modules + $ git checkout https://vendor_modules . + $ git checkout ssh://custom_modules . + $ export ANSIBLE_LIBRARY=/srv/modules/custom_modules:/srv/modules/vendor_modules + $ ansible [...] + +In case of modules with the same name, the library paths are searched in order +and the first module found with that name is used. .. _local_tmp: @@ -586,23 +604,8 @@ together. The same holds true for --skip-tags. default value will be True. After 2.4, the option is going away. Multiple --tags and multiple --skip-tags will always be merged together. -.. _module_set_locale: - -module_set_locale -================= - -This boolean value controls whether or not Ansible will prepend locale-specific environment variables (as specified -via the :ref:`module_lang` configuration option). If enabled, it results in the LANG, LC_MESSAGES, and LC_ALL -being set when the module is executed on the given remote system. By default this is disabled. - -.. note:: - - The module_set_locale option was added in Ansible-2.1 and defaulted to - True. The default was changed to False in Ansible-2.2 - .. _module_lang: - module_lang =========== @@ -611,6 +614,10 @@ By default, the value is value `LANG` on the controller or, if unset, `en_US.UTF module_lang = en_US.UTF-8 +.. note:: + + This is only used if :ref:`module_set_locale` is set to True. + .. _module_name: module_name @@ -622,6 +629,38 @@ it to 'shell':: module_name = command +.. _module_set_locale: + +module_set_locale +================= + +This boolean value controls whether or not Ansible will prepend locale-specific environment variables (as specified +via the :ref:`module_lang` configuration option). If enabled, it results in the LANG, LC_MESSAGES, and LC_ALL +being set when the module is executed on the given remote system. By default this is disabled. + +.. note:: + + The module_set_locale option was added in Ansible-2.1 and defaulted to + True. The default was changed to False in Ansible-2.2 + +.. _module_utils: + +module_utils +============ + +This is the default location Ansible looks to find module_utils:: + + module_utils = /usr/share/ansible/my_module_utils + +module_utils are python modules that Ansible is able to combine with Ansible +modules when sending them to the remote machine. Having custom module_utils +is useful for extracting common code when developing a set of site-specific +modules. + +Ansible can look in multiple locations if you feed it a colon +separated path, and it also will look for modules in the +:file:`./module_utils` directory alongside a playbook. + .. _nocolor: nocolor diff --git a/examples/ansible.cfg b/examples/ansible.cfg index ed03f60ed3e..416b48b3d96 100644 --- a/examples/ansible.cfg +++ b/examples/ansible.cfg @@ -13,6 +13,7 @@ #inventory = /etc/ansible/hosts #library = /usr/share/my_modules/ +#module_utils = /usr/share/my_module_utils/ #remote_tmp = ~/.ansible/tmp #local_tmp = ~/.ansible/tmp #forks = 5 diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 7556be44761..2f5708d7bdd 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -196,7 +196,6 @@ MERGE_MULTIPLE_CLI_TAGS = get_config(p, DEFAULTS, 'merge_multiple_cli_tags', 'AN #### GENERALLY CONFIGURABLE THINGS #### DEFAULT_DEBUG = get_config(p, DEFAULTS, 'debug', 'ANSIBLE_DEBUG', False, value_type='boolean') DEFAULT_HOST_LIST = get_config(p, DEFAULTS,'inventory', 'ANSIBLE_INVENTORY', DEPRECATED_HOST_LIST, value_type='path') -DEFAULT_MODULE_PATH = get_config(p, DEFAULTS, 'library', 'ANSIBLE_LIBRARY', None, value_type='pathlist') DEFAULT_ROLES_PATH = get_config(p, DEFAULTS, 'roles_path', 'ANSIBLE_ROLES_PATH', '/etc/ansible/roles', value_type='pathlist', expand_relative_paths=True) DEFAULT_REMOTE_TMP = get_config(p, DEFAULTS, 'remote_tmp', 'ANSIBLE_REMOTE_TEMP', '~/.ansible/tmp') DEFAULT_LOCAL_TMP = get_config(p, DEFAULTS, 'local_tmp', 'ANSIBLE_LOCAL_TEMP', '~/.ansible/tmp', value_type='tmppath') @@ -285,16 +284,20 @@ DEFAULT_BECOME_ASK_PASS = get_config(p, 'privilege_escalation', 'become_ask_pa # (mapping of param: squash field) DEFAULT_SQUASH_ACTIONS = get_config(p, DEFAULTS, 'squash_actions', 'ANSIBLE_SQUASH_ACTIONS', "apk, apt, dnf, homebrew, openbsd_pkg, pacman, pkgng, yum, zypper", value_type='list') # paths + DEFAULT_ACTION_PLUGIN_PATH = get_config(p, DEFAULTS, 'action_plugins', 'ANSIBLE_ACTION_PLUGINS', '~/.ansible/plugins/action:/usr/share/ansible/plugins/action', value_type='pathlist') DEFAULT_CACHE_PLUGIN_PATH = get_config(p, DEFAULTS, 'cache_plugins', 'ANSIBLE_CACHE_PLUGINS', '~/.ansible/plugins/cache:/usr/share/ansible/plugins/cache', value_type='pathlist') DEFAULT_CALLBACK_PLUGIN_PATH = get_config(p, DEFAULTS, 'callback_plugins', 'ANSIBLE_CALLBACK_PLUGINS', '~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback', value_type='pathlist') DEFAULT_CONNECTION_PLUGIN_PATH = get_config(p, DEFAULTS, 'connection_plugins', 'ANSIBLE_CONNECTION_PLUGINS', '~/.ansible/plugins/connection:/usr/share/ansible/plugins/connection', value_type='pathlist') DEFAULT_LOOKUP_PLUGIN_PATH = get_config(p, DEFAULTS, 'lookup_plugins', 'ANSIBLE_LOOKUP_PLUGINS', '~/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup', value_type='pathlist') +DEFAULT_MODULE_PATH = get_config(p, DEFAULTS, 'library', 'ANSIBLE_LIBRARY', None, value_type='pathlist') +DEFAULT_MODULE_UTILS_PATH = get_config(p, DEFAULTS, 'module_utils', 'ANSIBLE_MODULE_UTILS', None, value_type='pathlist') DEFAULT_INVENTORY_PLUGIN_PATH = get_config(p, DEFAULTS, 'inventory_plugins', 'ANSIBLE_INVENTORY_PLUGINS', '~/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory', value_type='pathlist') DEFAULT_VARS_PLUGIN_PATH = get_config(p, DEFAULTS, 'vars_plugins', 'ANSIBLE_VARS_PLUGINS', '~/.ansible/plugins/vars:/usr/share/ansible/plugins/vars', value_type='pathlist') DEFAULT_FILTER_PLUGIN_PATH = get_config(p, DEFAULTS, 'filter_plugins', 'ANSIBLE_FILTER_PLUGINS', '~/.ansible/plugins/filter:/usr/share/ansible/plugins/filter', value_type='pathlist') DEFAULT_TEST_PLUGIN_PATH = get_config(p, DEFAULTS, 'test_plugins', 'ANSIBLE_TEST_PLUGINS', '~/.ansible/plugins/test:/usr/share/ansible/plugins/test', value_type='pathlist') DEFAULT_STRATEGY_PLUGIN_PATH = get_config(p, DEFAULTS, 'strategy_plugins', 'ANSIBLE_STRATEGY_PLUGINS', '~/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy', value_type='pathlist') + DEFAULT_STRATEGY = get_config(p, DEFAULTS, 'strategy', 'ANSIBLE_STRATEGY', 'linear') DEFAULT_STDOUT_CALLBACK = get_config(p, DEFAULTS, 'stdout_callback', 'ANSIBLE_STDOUT_CALLBACK', 'default') # cache diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index eb28fffeff3..1d1c826f880 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -498,7 +498,20 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): if module_info is None: msg = ['Could not find imported module support code for %s. Looked for' % name] if idx == 2: - msg.append('either %s or %s' % (py_module_name[-1], py_module_name[-2])) + msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) + else: + msg.append(py_module_name[-1]) + raise AnsibleError(' '.join(msg)) + + # Found a byte compiled file rather than source. We cannot send byte + # compiled over the wire as the python version might be different. + # imp.find_module seems to prefer to return source packages so we just + # error out if imp.find_module returns byte compiled files (This is + # fragile as it depends on undocumented imp.find_module behaviour) + if module_info[2][2] not in (imp.PY_SOURCE, imp.PKG_DIRECTORY): + msg = ['Could not find python source for imported module support code for %s. Looked for' % name] + if idx == 2: + msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) else: msg.append(py_module_name[-1]) raise AnsibleError(' '.join(msg)) @@ -571,7 +584,6 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas Given the source of the module, convert it to a Jinja2 template to insert module code and return whether it's a new or old style module. """ - module_substyle = module_style = 'old' # module_style is something important to calling code (ActionBase). It diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index f0d610d744d..2e6666bca45 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -485,7 +485,7 @@ module_loader = PluginLoader( module_utils_loader = PluginLoader( '', 'ansible.module_utils', - 'module_utils', + C.DEFAULT_MODULE_UTILS_PATH, 'module_utils', ) diff --git a/test/integration/targets/module_utils/library/test_env_override.py b/test/integration/targets/module_utils/library/test_env_override.py new file mode 100644 index 00000000000..e46f8b141ce --- /dev/null +++ b/test/integration/targets/module_utils/library/test_env_override.py @@ -0,0 +1,8 @@ +#!/usr/bin/python +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.json_utils import data +from ansible.module_utils.mork import data as mork_data + +results = {"json_utils": data, "mork": mork_data} + +AnsibleModule(argument_spec=dict()).exit_json(**results) diff --git a/test/integration/targets/module_utils/module_utils/service.py b/test/integration/targets/module_utils/module_utils/service.py new file mode 100644 index 00000000000..1492f46865d --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/service.py @@ -0,0 +1 @@ +sysv_is_enabled = 'sysv_is_enabled' diff --git a/test/integration/targets/module_utils/module_utils_envvar.yml b/test/integration/targets/module_utils/module_utils_envvar.yml new file mode 100644 index 00000000000..edd55d53b46 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils_envvar.yml @@ -0,0 +1,51 @@ +- hosts: localhost + gather_facts: no + tasks: + - name: Use a specially crafted module to see if things were imported correctly + test: + register: result + + - name: Check that these are all loaded from playbook dir's module_utils + assert: + that: + - 'result["abcdefgh"] == "abcdefgh"' + - 'result["bar0"] == "bar0"' + - 'result["bar1"] == "bar1"' + - 'result["bar2"] == "bar2"' + - 'result["baz1"] == "baz1"' + - 'result["baz2"] == "baz2"' + - 'result["foo0"] == "foo0"' + - 'result["foo1"] == "foo1"' + - 'result["foo2"] == "foo2"' + - 'result["qux1"] == "qux1"' + - 'result["qux2"] == ["qux2:quux", "qux2:quuz"]' + - 'result["spam1"] == "spam1"' + - 'result["spam2"] == "spam2"' + - 'result["spam3"] == "spam3"' + - 'result["spam4"] == "spam4"' + - 'result["spam5"] == ["spam5:bacon", "spam5:eggs"]' + - 'result["spam6"] == ["spam6:bacon", "spam6:eggs"]' + - 'result["spam7"] == ["spam7:bacon", "spam7:eggs"]' + - 'result["spam8"] == ["spam8:bacon", "spam8:eggs"]' + + # Test that overriding something in module_utils with something in the local library works + - name: Test that playbook dir's module_utils overrides facts.py + test_override: + register: result + + - name: Make sure the we used the local facts.py, not the one shipped with ansible + assert: + that: + - 'result["data"] == "overridden facts.py"' + + - name: Test that importing something from the module_utils in the env_vars works + test_env_override: + register: result + + - name: Make sure we used the module_utils from the env_var for these + assert: + that: + # Override of shipped module_utils + - 'result["json_utils"] == "overridden json_utils"' + # Only i nthe env vars directory + - 'result["mork"] == "mork"' diff --git a/test/integration/targets/module_utils/module_utils_test.yml b/test/integration/targets/module_utils/module_utils_test.yml index 8a5cccce472..a1317270840 100644 --- a/test/integration/targets/module_utils/module_utils_test.yml +++ b/test/integration/targets/module_utils/module_utils_test.yml @@ -43,8 +43,9 @@ ignore_errors: True register: result + - debug: var=result - name: Make sure we failed in AnsiBallZ assert: that: - 'result["failed"] == True' - - '"Could not find imported module support code for test_failure. Looked for either foo or zebra" == result["msg"]' + - '"Could not find imported module support code for test_failure. Looked for either foo.py or zebra.py" == result["msg"]' diff --git a/test/integration/targets/module_utils/other_mu_dir/__init__.py b/test/integration/targets/module_utils/other_mu_dir/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/h/__init__.py b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/h/__init__.py new file mode 100644 index 00000000000..796fed385d1 --- /dev/null +++ b/test/integration/targets/module_utils/other_mu_dir/a/b/c/d/e/f/g/h/__init__.py @@ -0,0 +1 @@ +data = 'should not be visible abcdefgh' diff --git a/test/integration/targets/module_utils/other_mu_dir/facts.py b/test/integration/targets/module_utils/other_mu_dir/facts.py new file mode 100644 index 00000000000..dbfab2718e2 --- /dev/null +++ b/test/integration/targets/module_utils/other_mu_dir/facts.py @@ -0,0 +1 @@ +data = 'should not be visible facts.py' diff --git a/test/integration/targets/module_utils/other_mu_dir/json_utils.py b/test/integration/targets/module_utils/other_mu_dir/json_utils.py new file mode 100644 index 00000000000..59757e405cf --- /dev/null +++ b/test/integration/targets/module_utils/other_mu_dir/json_utils.py @@ -0,0 +1 @@ +data = 'overridden json_utils' diff --git a/test/integration/targets/module_utils/other_mu_dir/mork.py b/test/integration/targets/module_utils/other_mu_dir/mork.py new file mode 100644 index 00000000000..3b700fca41a --- /dev/null +++ b/test/integration/targets/module_utils/other_mu_dir/mork.py @@ -0,0 +1 @@ +data = 'mork' diff --git a/test/integration/targets/module_utils/runme.sh b/test/integration/targets/module_utils/runme.sh index ddde89f10ae..4c8aab7ffe2 100755 --- a/test/integration/targets/module_utils/runme.sh +++ b/test/integration/targets/module_utils/runme.sh @@ -3,3 +3,4 @@ set -eux ansible-playbook module_utils_test.yml -i ../../inventory -v "$@" +ANSIBLE_MODULE_UTILS=$(pwd)/other_mu_dir ansible-playbook module_utils_envvar.yml -i ../../inventory -v "$@"