mirror of https://github.com/ansible/ansible.git
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1434 lines
64 KiB
Python
1434 lines
64 KiB
Python
# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>
|
|
# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
|
|
#
|
|
# This file is part of Ansible
|
|
#
|
|
# Ansible is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Ansible is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import base64
|
|
import datetime
|
|
import json
|
|
import os
|
|
import shlex
|
|
import time
|
|
import zipfile
|
|
import re
|
|
import pkgutil
|
|
|
|
from ast import AST, Import, ImportFrom
|
|
from io import BytesIO
|
|
|
|
from ansible.release import __version__, __author__
|
|
from ansible import constants as C
|
|
from ansible.errors import AnsibleError
|
|
from ansible.executor.interpreter_discovery import InterpreterDiscoveryRequiredError
|
|
from ansible.executor.powershell import module_manifest as ps_manifest
|
|
from ansible.module_utils.common.json import AnsibleJSONEncoder
|
|
from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native
|
|
from ansible.plugins.loader import module_utils_loader
|
|
from ansible.utils.collection_loader._collection_finder import _get_collection_metadata, _nested_dict_get
|
|
|
|
# Must import strategy and use write_locks from there
|
|
# If we import write_locks directly then we end up binding a
|
|
# variable to the object and then it never gets updated.
|
|
from ansible.executor import action_write_locks
|
|
|
|
from ansible.utils.display import Display
|
|
from collections import namedtuple
|
|
|
|
import importlib.util
|
|
import importlib.machinery
|
|
|
|
display = Display()
|
|
|
|
ModuleUtilsProcessEntry = namedtuple('ModuleUtilsProcessEntry', ['name_parts', 'is_ambiguous', 'has_redirected_child', 'is_optional'])
|
|
|
|
REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
|
|
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
|
|
REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
|
|
REPLACER_WINDOWS = b"# POWERSHELL_COMMON"
|
|
REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
|
|
REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
|
|
|
|
# We could end up writing out parameters with unicode characters so we need to
|
|
# specify an encoding for the python source file
|
|
ENCODING_STRING = u'# -*- coding: utf-8 -*-'
|
|
b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
|
|
|
|
# module_common is relative to module_utils, so fix the path
|
|
_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
|
|
|
|
# ******************************************************************************
|
|
|
|
ANSIBALLZ_TEMPLATE = u'''%(shebang)s
|
|
%(coding)s
|
|
_ANSIBALLZ_WRAPPER = True # For test-module.py script to tell this is a ANSIBALLZ_WRAPPER
|
|
# This code is part of Ansible, but is an independent component.
|
|
# The code in this particular templatable string, and this templatable string
|
|
# only, is BSD licensed. Modules which end up using this snippet, which is
|
|
# dynamically combined together by Ansible still belong to the author of the
|
|
# module, and they may assign their own license to the complete work.
|
|
#
|
|
# Copyright (c), James Cammarata, 2016
|
|
# Copyright (c), Toshio Kuratomi, 2016
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without modification,
|
|
# are permitted provided that the following conditions are met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
|
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
def _ansiballz_main():
|
|
import os
|
|
import os.path
|
|
|
|
# Access to the working directory is required by Python when using pipelining, as well as for the coverage module.
|
|
# Some platforms, such as macOS, may not allow querying the working directory when using become to drop privileges.
|
|
try:
|
|
os.getcwd()
|
|
except OSError:
|
|
try:
|
|
os.chdir(os.path.expanduser('~'))
|
|
except OSError:
|
|
os.chdir('/')
|
|
|
|
%(rlimit)s
|
|
|
|
import sys
|
|
import __main__
|
|
|
|
# For some distros and python versions we pick up this script in the temporary
|
|
# directory. This leads to problems when the ansible module masks a python
|
|
# library that another import needs. We have not figured out what about the
|
|
# specific distros and python versions causes this to behave differently.
|
|
#
|
|
# Tested distros:
|
|
# Fedora23 with python3.4 Works
|
|
# Ubuntu15.10 with python2.7 Works
|
|
# Ubuntu15.10 with python3.4 Fails without this
|
|
# Ubuntu16.04.1 with python3.5 Fails without this
|
|
# To test on another platform:
|
|
# * use the copy module (since this shadows the stdlib copy module)
|
|
# * Turn off pipelining
|
|
# * Make sure that the destination file does not exist
|
|
# * ansible ubuntu16-test -m copy -a 'src=/etc/motd dest=/var/tmp/m'
|
|
# This will traceback in shutil. Looking at the complete traceback will show
|
|
# that shutil is importing copy which finds the ansible module instead of the
|
|
# stdlib module
|
|
scriptdir = None
|
|
try:
|
|
scriptdir = os.path.dirname(os.path.realpath(__main__.__file__))
|
|
except (AttributeError, OSError):
|
|
# Some platforms don't set __file__ when reading from stdin
|
|
# OSX raises OSError if using abspath() in a directory we don't have
|
|
# permission to read (realpath calls abspath)
|
|
pass
|
|
|
|
# Strip cwd from sys.path to avoid potential permissions issues
|
|
excludes = set(('', '.', scriptdir))
|
|
sys.path = [p for p in sys.path if p not in excludes]
|
|
|
|
import base64
|
|
import runpy
|
|
import shutil
|
|
import tempfile
|
|
import zipfile
|
|
|
|
if sys.version_info < (3,):
|
|
PY3 = False
|
|
else:
|
|
PY3 = True
|
|
|
|
ZIPDATA = %(zipdata)r
|
|
|
|
# Note: temp_path isn't needed once we switch to zipimport
|
|
def invoke_module(modlib_path, temp_path, json_params):
|
|
# When installed via setuptools (including python setup.py install),
|
|
# ansible may be installed with an easy-install.pth file. That file
|
|
# may load the system-wide install of ansible rather than the one in
|
|
# the module. sitecustomize is the only way to override that setting.
|
|
z = zipfile.ZipFile(modlib_path, mode='a')
|
|
|
|
# py3: modlib_path will be text, py2: it's bytes. Need bytes at the end
|
|
sitecustomize = u'import sys\\nsys.path.insert(0,"%%s")\\n' %% modlib_path
|
|
sitecustomize = sitecustomize.encode('utf-8')
|
|
# Use a ZipInfo to work around zipfile limitation on hosts with
|
|
# clocks set to a pre-1980 year (for instance, Raspberry Pi)
|
|
zinfo = zipfile.ZipInfo()
|
|
zinfo.filename = 'sitecustomize.py'
|
|
zinfo.date_time = %(date_time)s
|
|
z.writestr(zinfo, sitecustomize)
|
|
z.close()
|
|
|
|
# Put the zipped up module_utils we got from the controller first in the python path so that we
|
|
# can monkeypatch the right basic
|
|
sys.path.insert(0, modlib_path)
|
|
|
|
# Monkeypatch the parameters into basic
|
|
from ansible.module_utils import basic
|
|
basic._ANSIBLE_ARGS = json_params
|
|
%(coverage)s
|
|
# Run the module! By importing it as '__main__', it thinks it is executing as a script
|
|
runpy.run_module(mod_name=%(module_fqn)r, init_globals=dict(_module_fqn=%(module_fqn)r, _modlib_path=modlib_path),
|
|
run_name='__main__', alter_sys=True)
|
|
|
|
# Ansible modules must exit themselves
|
|
print('{"msg": "New-style module did not handle its own exit", "failed": true}')
|
|
sys.exit(1)
|
|
|
|
def debug(command, zipped_mod, json_params):
|
|
# The code here normally doesn't run. It's only used for debugging on the
|
|
# remote machine.
|
|
#
|
|
# The subcommands in this function make it easier to debug ansiballz
|
|
# modules. Here's the basic steps:
|
|
#
|
|
# Run ansible with the environment variable: ANSIBLE_KEEP_REMOTE_FILES=1 and -vvv
|
|
# to save the module file remotely::
|
|
# $ ANSIBLE_KEEP_REMOTE_FILES=1 ansible host1 -m ping -a 'data=october' -vvv
|
|
#
|
|
# Part of the verbose output will tell you where on the remote machine the
|
|
# module was written to::
|
|
# [...]
|
|
# <host1> SSH: EXEC ssh -C -q -o ControlMaster=auto -o ControlPersist=60s -o KbdInteractiveAuthentication=no -o
|
|
# PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey -o PasswordAuthentication=no -o ConnectTimeout=10 -o
|
|
# ControlPath=/home/badger/.ansible/cp/ansible-ssh-%%h-%%p-%%r -tt rhel7 '/bin/sh -c '"'"'LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
|
|
# LC_MESSAGES=en_US.UTF-8 /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping'"'"''
|
|
# [...]
|
|
#
|
|
# Login to the remote machine and run the module file via from the previous
|
|
# step with the explode subcommand to extract the module payload into
|
|
# source files::
|
|
# $ ssh host1
|
|
# $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping explode
|
|
# Module expanded into:
|
|
# /home/badger/.ansible/tmp/ansible-tmp-1461173408.08-279692652635227/ansible
|
|
#
|
|
# You can now edit the source files to instrument the code or experiment with
|
|
# different parameter values. When you're ready to run the code you've modified
|
|
# (instead of the code from the actual zipped module), use the execute subcommand like this::
|
|
# $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping execute
|
|
|
|
# Okay to use __file__ here because we're running from a kept file
|
|
basedir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'debug_dir')
|
|
args_path = os.path.join(basedir, 'args')
|
|
|
|
if command == 'explode':
|
|
# transform the ZIPDATA into an exploded directory of code and then
|
|
# print the path to the code. This is an easy way for people to look
|
|
# at the code on the remote machine for debugging it in that
|
|
# environment
|
|
z = zipfile.ZipFile(zipped_mod)
|
|
for filename in z.namelist():
|
|
if filename.startswith('/'):
|
|
raise Exception('Something wrong with this module zip file: should not contain absolute paths')
|
|
|
|
dest_filename = os.path.join(basedir, filename)
|
|
if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename):
|
|
os.makedirs(dest_filename)
|
|
else:
|
|
directory = os.path.dirname(dest_filename)
|
|
if not os.path.exists(directory):
|
|
os.makedirs(directory)
|
|
f = open(dest_filename, 'wb')
|
|
f.write(z.read(filename))
|
|
f.close()
|
|
|
|
# write the args file
|
|
f = open(args_path, 'wb')
|
|
f.write(json_params)
|
|
f.close()
|
|
|
|
print('Module expanded into:')
|
|
print('%%s' %% basedir)
|
|
exitcode = 0
|
|
|
|
elif command == 'execute':
|
|
# Execute the exploded code instead of executing the module from the
|
|
# embedded ZIPDATA. This allows people to easily run their modified
|
|
# code on the remote machine to see how changes will affect it.
|
|
|
|
# Set pythonpath to the debug dir
|
|
sys.path.insert(0, basedir)
|
|
|
|
# read in the args file which the user may have modified
|
|
with open(args_path, 'rb') as f:
|
|
json_params = f.read()
|
|
|
|
# Monkeypatch the parameters into basic
|
|
from ansible.module_utils import basic
|
|
basic._ANSIBLE_ARGS = json_params
|
|
|
|
# Run the module! By importing it as '__main__', it thinks it is executing as a script
|
|
runpy.run_module(mod_name=%(module_fqn)r, init_globals=None, run_name='__main__', alter_sys=True)
|
|
|
|
# Ansible modules must exit themselves
|
|
print('{"msg": "New-style module did not handle its own exit", "failed": true}')
|
|
sys.exit(1)
|
|
|
|
else:
|
|
print('WARNING: Unknown debug command. Doing nothing.')
|
|
exitcode = 0
|
|
|
|
return exitcode
|
|
|
|
#
|
|
# See comments in the debug() method for information on debugging
|
|
#
|
|
|
|
ANSIBALLZ_PARAMS = %(params)s
|
|
if PY3:
|
|
ANSIBALLZ_PARAMS = ANSIBALLZ_PARAMS.encode('utf-8')
|
|
try:
|
|
# There's a race condition with the controller removing the
|
|
# remote_tmpdir and this module executing under async. So we cannot
|
|
# store this in remote_tmpdir (use system tempdir instead)
|
|
# Only need to use [ansible_module]_payload_ in the temp_path until we move to zipimport
|
|
# (this helps ansible-test produce coverage stats)
|
|
temp_path = tempfile.mkdtemp(prefix='ansible_' + %(ansible_module)r + '_payload_')
|
|
|
|
zipped_mod = os.path.join(temp_path, 'ansible_' + %(ansible_module)r + '_payload.zip')
|
|
|
|
with open(zipped_mod, 'wb') as modlib:
|
|
modlib.write(base64.b64decode(ZIPDATA))
|
|
|
|
if len(sys.argv) == 2:
|
|
exitcode = debug(sys.argv[1], zipped_mod, ANSIBALLZ_PARAMS)
|
|
else:
|
|
# Note: temp_path isn't needed once we switch to zipimport
|
|
invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)
|
|
finally:
|
|
try:
|
|
shutil.rmtree(temp_path)
|
|
except (NameError, OSError):
|
|
# tempdir creation probably failed
|
|
pass
|
|
sys.exit(exitcode)
|
|
|
|
if __name__ == '__main__':
|
|
_ansiballz_main()
|
|
'''
|
|
|
|
ANSIBALLZ_COVERAGE_TEMPLATE = '''
|
|
os.environ['COVERAGE_FILE'] = %(coverage_output)r + '=python-%%s=coverage' %% '.'.join(str(v) for v in sys.version_info[:2])
|
|
|
|
import atexit
|
|
|
|
try:
|
|
import coverage
|
|
except ImportError:
|
|
print('{"msg": "Could not import `coverage` module.", "failed": true}')
|
|
sys.exit(1)
|
|
|
|
cov = coverage.Coverage(config_file=%(coverage_config)r)
|
|
|
|
def atexit_coverage():
|
|
cov.stop()
|
|
cov.save()
|
|
|
|
atexit.register(atexit_coverage)
|
|
|
|
cov.start()
|
|
'''
|
|
|
|
ANSIBALLZ_COVERAGE_CHECK_TEMPLATE = '''
|
|
try:
|
|
if PY3:
|
|
import importlib.util
|
|
if importlib.util.find_spec('coverage') is None:
|
|
raise ImportError
|
|
else:
|
|
import imp
|
|
imp.find_module('coverage')
|
|
except ImportError:
|
|
print('{"msg": "Could not find `coverage` module.", "failed": true}')
|
|
sys.exit(1)
|
|
'''
|
|
|
|
ANSIBALLZ_RLIMIT_TEMPLATE = '''
|
|
import resource
|
|
|
|
existing_soft, existing_hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
|
|
# adjust soft limit subject to existing hard limit
|
|
requested_soft = min(existing_hard, %(rlimit_nofile)d)
|
|
|
|
if requested_soft != existing_soft:
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_NOFILE, (requested_soft, existing_hard))
|
|
except ValueError:
|
|
# some platforms (eg macOS) lie about their hard limit
|
|
pass
|
|
'''
|
|
|
|
|
|
def _strip_comments(source):
|
|
# Strip comments and blank lines from the wrapper
|
|
buf = []
|
|
for line in source.splitlines():
|
|
l = line.strip()
|
|
if not l or l.startswith(u'#'):
|
|
continue
|
|
buf.append(line)
|
|
return u'\n'.join(buf)
|
|
|
|
|
|
if C.DEFAULT_KEEP_REMOTE_FILES:
|
|
# Keep comments when KEEP_REMOTE_FILES is set. That way users will see
|
|
# the comments with some nice usage instructions
|
|
ACTIVE_ANSIBALLZ_TEMPLATE = ANSIBALLZ_TEMPLATE
|
|
else:
|
|
# ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size
|
|
ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE)
|
|
|
|
# dirname(dirname(dirname(site-packages/ansible/executor/module_common.py) == site-packages
|
|
# Do this instead of getting site-packages from distutils.sysconfig so we work when we
|
|
# haven't been installed
|
|
site_packages = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
|
CORE_LIBRARY_PATH_RE = re.compile(r'%s/(?P<path>ansible/modules/.*)\.(py|ps1)$' % re.escape(site_packages))
|
|
COLLECTION_PATH_RE = re.compile(r'/(?P<path>ansible_collections/[^/]+/[^/]+/plugins/modules/.*)\.(py|ps1)$')
|
|
|
|
# Detect new-style Python modules by looking for required imports:
|
|
# import ansible_collections.[my_ns.my_col.plugins.module_utils.my_module_util]
|
|
# from ansible_collections.[my_ns.my_col.plugins.module_utils import my_module_util]
|
|
# import ansible.module_utils[.basic]
|
|
# from ansible.module_utils[ import basic]
|
|
# from ansible.module_utils[.basic import AnsibleModule]
|
|
# from ..module_utils[ import basic]
|
|
# from ..module_utils[.basic import AnsibleModule]
|
|
NEW_STYLE_PYTHON_MODULE_RE = re.compile(
|
|
# Relative imports
|
|
br'(?:from +\.{2,} *module_utils.* +import |'
|
|
# Collection absolute imports:
|
|
br'from +ansible_collections\.[^.]+\.[^.]+\.plugins\.module_utils.* +import |'
|
|
br'import +ansible_collections\.[^.]+\.[^.]+\.plugins\.module_utils.*|'
|
|
# Core absolute imports
|
|
br'from +ansible\.module_utils.* +import |'
|
|
br'import +ansible\.module_utils\.)'
|
|
)
|
|
|
|
|
|
class ModuleDepFinder(ast.NodeVisitor):
|
|
def __init__(self, module_fqn, tree, is_pkg_init=False, *args, **kwargs):
|
|
"""
|
|
Walk the ast tree for the python module.
|
|
:arg module_fqn: The fully qualified name to reach this module in dotted notation.
|
|
example: ansible.module_utils.basic
|
|
:arg is_pkg_init: Inform the finder it's looking at a package init (eg __init__.py) to allow
|
|
relative import expansion to use the proper package level without having imported it locally first.
|
|
|
|
Save submodule[.submoduleN][.identifier] into self.submodules
|
|
when they are from ansible.module_utils or ansible_collections packages
|
|
|
|
self.submodules will end up with tuples like:
|
|
- ('ansible', 'module_utils', 'basic',)
|
|
- ('ansible', 'module_utils', 'urls', 'fetch_url')
|
|
- ('ansible', 'module_utils', 'database', 'postgres')
|
|
- ('ansible', 'module_utils', 'database', 'postgres', 'quote')
|
|
- ('ansible', 'module_utils', 'database', 'postgres', 'quote')
|
|
- ('ansible_collections', 'my_ns', 'my_col', 'plugins', 'module_utils', 'foo')
|
|
|
|
It's up to calling code to determine whether the final element of the
|
|
tuple are module names or something else (function, class, or variable names)
|
|
.. seealso:: :python3:class:`ast.NodeVisitor`
|
|
"""
|
|
super(ModuleDepFinder, self).__init__(*args, **kwargs)
|
|
self._tree = tree # squirrel this away so we can compare node parents to it
|
|
self.submodules = set()
|
|
self.optional_imports = set()
|
|
self.module_fqn = module_fqn
|
|
self.is_pkg_init = is_pkg_init
|
|
|
|
self._visit_map = {
|
|
Import: self.visit_Import,
|
|
ImportFrom: self.visit_ImportFrom,
|
|
}
|
|
|
|
self.visit(tree)
|
|
|
|
def generic_visit(self, node):
|
|
"""Overridden ``generic_visit`` that makes some assumptions about our
|
|
use case, and improves performance by calling visitors directly instead
|
|
of calling ``visit`` to offload calling visitors.
|
|
"""
|
|
generic_visit = self.generic_visit
|
|
visit_map = self._visit_map
|
|
for field, value in ast.iter_fields(node):
|
|
if isinstance(value, list):
|
|
for item in value:
|
|
if isinstance(item, (Import, ImportFrom)):
|
|
item.parent = node
|
|
visit_map[item.__class__](item)
|
|
elif isinstance(item, AST):
|
|
generic_visit(item)
|
|
|
|
visit = generic_visit
|
|
|
|
def visit_Import(self, node):
|
|
"""
|
|
Handle import ansible.module_utils.MODLIB[.MODLIBn] [as asname]
|
|
|
|
We save these as interesting submodules when the imported library is in ansible.module_utils
|
|
or ansible.collections
|
|
"""
|
|
for alias in node.names:
|
|
if (alias.name.startswith('ansible.module_utils.') or
|
|
alias.name.startswith('ansible_collections.')):
|
|
py_mod = tuple(alias.name.split('.'))
|
|
self.submodules.add(py_mod)
|
|
# if the import's parent is the root document, it's a required import, otherwise it's optional
|
|
if node.parent != self._tree:
|
|
self.optional_imports.add(py_mod)
|
|
self.generic_visit(node)
|
|
|
|
def visit_ImportFrom(self, node):
|
|
"""
|
|
Handle from ansible.module_utils.MODLIB import [.MODLIBn] [as asname]
|
|
|
|
Also has to handle relative imports
|
|
|
|
We save these as interesting submodules when the imported library is in ansible.module_utils
|
|
or ansible.collections
|
|
"""
|
|
|
|
# FIXME: These should all get skipped:
|
|
# from ansible.executor import module_common
|
|
# from ...executor import module_common
|
|
# from ... import executor (Currently it gives a non-helpful error)
|
|
if node.level > 0:
|
|
# if we're in a package init, we have to add one to the node level (and make it none if 0 to preserve the right slicing behavior)
|
|
level_slice_offset = -node.level + 1 or None if self.is_pkg_init else -node.level
|
|
if self.module_fqn:
|
|
parts = tuple(self.module_fqn.split('.'))
|
|
if node.module:
|
|
# relative import: from .module import x
|
|
node_module = '.'.join(parts[:level_slice_offset] + (node.module,))
|
|
else:
|
|
# relative import: from . import x
|
|
node_module = '.'.join(parts[:level_slice_offset])
|
|
else:
|
|
# fall back to an absolute import
|
|
node_module = node.module
|
|
else:
|
|
# absolute import: from module import x
|
|
node_module = node.module
|
|
|
|
# Specialcase: six is a special case because of its
|
|
# import logic
|
|
py_mod = None
|
|
if node.names[0].name == '_six':
|
|
self.submodules.add(('_six',))
|
|
elif node_module.startswith('ansible.module_utils'):
|
|
# from ansible.module_utils.MODULE1[.MODULEn] import IDENTIFIER [as asname]
|
|
# from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [as asname]
|
|
# from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [,IDENTIFIER] [as asname]
|
|
# from ansible.module_utils import MODULE1 [,MODULEn] [as asname]
|
|
py_mod = tuple(node_module.split('.'))
|
|
|
|
elif node_module.startswith('ansible_collections.'):
|
|
if node_module.endswith('plugins.module_utils') or '.plugins.module_utils.' in node_module:
|
|
# from ansible_collections.ns.coll.plugins.module_utils import MODULE [as aname] [,MODULE2] [as aname]
|
|
# from ansible_collections.ns.coll.plugins.module_utils.MODULE import IDENTIFIER [as aname]
|
|
# FIXME: Unhandled cornercase (needs to be ignored):
|
|
# from ansible_collections.ns.coll.plugins.[!module_utils].[FOO].plugins.module_utils import IDENTIFIER
|
|
py_mod = tuple(node_module.split('.'))
|
|
else:
|
|
# Not from module_utils so ignore. for instance:
|
|
# from ansible_collections.ns.coll.plugins.lookup import IDENTIFIER
|
|
pass
|
|
|
|
if py_mod:
|
|
for alias in node.names:
|
|
self.submodules.add(py_mod + (alias.name,))
|
|
# if the import's parent is the root document, it's a required import, otherwise it's optional
|
|
if node.parent != self._tree:
|
|
self.optional_imports.add(py_mod + (alias.name,))
|
|
|
|
self.generic_visit(node)
|
|
|
|
|
|
def _slurp(path):
|
|
if not os.path.exists(path):
|
|
raise AnsibleError("imported module support code does not exist at %s" % os.path.abspath(path))
|
|
with open(path, 'rb') as fd:
|
|
data = fd.read()
|
|
return data
|
|
|
|
|
|
def _get_shebang(interpreter, task_vars, templar, args=tuple(), remote_is_local=False):
|
|
"""
|
|
Handles the different ways ansible allows overriding the shebang target for a module.
|
|
"""
|
|
# FUTURE: add logical equivalence for python3 in the case of py3-only modules
|
|
|
|
interpreter_name = os.path.basename(interpreter).strip()
|
|
|
|
# name for interpreter var
|
|
interpreter_config = u'ansible_%s_interpreter' % interpreter_name
|
|
# key for config
|
|
interpreter_config_key = "INTERPRETER_%s" % interpreter_name.upper()
|
|
|
|
interpreter_out = None
|
|
|
|
# looking for python, rest rely on matching vars
|
|
if interpreter_name == 'python':
|
|
# skip detection for network os execution, use playbook supplied one if possible
|
|
if remote_is_local:
|
|
interpreter_out = task_vars['ansible_playbook_python']
|
|
|
|
# a config def exists for this interpreter type; consult config for the value
|
|
elif C.config.get_configuration_definition(interpreter_config_key):
|
|
|
|
interpreter_from_config = C.config.get_config_value(interpreter_config_key, variables=task_vars)
|
|
interpreter_out = templar.template(interpreter_from_config.strip())
|
|
|
|
# handle interpreter discovery if requested or empty interpreter was provided
|
|
if not interpreter_out or interpreter_out in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']:
|
|
|
|
discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name
|
|
facts_from_task_vars = task_vars.get('ansible_facts', {})
|
|
|
|
if discovered_interpreter_config not in facts_from_task_vars:
|
|
# interpreter discovery is desired, but has not been run for this host
|
|
raise InterpreterDiscoveryRequiredError("interpreter discovery needed", interpreter_name=interpreter_name, discovery_mode=interpreter_out)
|
|
else:
|
|
interpreter_out = facts_from_task_vars[discovered_interpreter_config]
|
|
else:
|
|
raise InterpreterDiscoveryRequiredError("interpreter discovery required", interpreter_name=interpreter_name, discovery_mode='auto_legacy')
|
|
|
|
elif interpreter_config in task_vars:
|
|
# for non python we consult vars for a possible direct override
|
|
interpreter_out = templar.template(task_vars.get(interpreter_config).strip())
|
|
|
|
if not interpreter_out:
|
|
# nothing matched(None) or in case someone configures empty string or empty intepreter
|
|
interpreter_out = interpreter
|
|
|
|
# set shebang
|
|
shebang = u'#!{0}'.format(interpreter_out)
|
|
if args:
|
|
shebang = shebang + u' ' + u' '.join(args)
|
|
|
|
return shebang, interpreter_out
|
|
|
|
|
|
class ModuleUtilLocatorBase:
|
|
def __init__(self, fq_name_parts, is_ambiguous=False, child_is_redirected=False, is_optional=False):
|
|
self._is_ambiguous = is_ambiguous
|
|
# a child package redirection could cause intermediate package levels to be missing, eg
|
|
# from ansible.module_utils.x.y.z import foo; if x.y.z.foo is redirected, we may not have packages on disk for
|
|
# the intermediate packages x.y.z, so we'll need to supply empty packages for those
|
|
self._child_is_redirected = child_is_redirected
|
|
self._is_optional = is_optional
|
|
self.found = False
|
|
self.redirected = False
|
|
self.fq_name_parts = fq_name_parts
|
|
self.source_code = ''
|
|
self.output_path = ''
|
|
self.is_package = False
|
|
self._collection_name = None
|
|
# for ambiguous imports, we should only test for things more than one level below module_utils
|
|
# this lets us detect erroneous imports and redirections earlier
|
|
if is_ambiguous and len(self._get_module_utils_remainder_parts(fq_name_parts)) > 1:
|
|
self.candidate_names = [fq_name_parts, fq_name_parts[:-1]]
|
|
else:
|
|
self.candidate_names = [fq_name_parts]
|
|
|
|
@property
|
|
def candidate_names_joined(self):
|
|
return ['.'.join(n) for n in self.candidate_names]
|
|
|
|
def _handle_redirect(self, name_parts):
|
|
module_utils_relative_parts = self._get_module_utils_remainder_parts(name_parts)
|
|
|
|
# only allow redirects from below module_utils- if above that, bail out (eg, parent package names)
|
|
if not module_utils_relative_parts:
|
|
return False
|
|
|
|
try:
|
|
collection_metadata = _get_collection_metadata(self._collection_name)
|
|
except ValueError as ve: # collection not found or some other error related to collection load
|
|
if self._is_optional:
|
|
return False
|
|
raise AnsibleError('error processing module_util {0} loading redirected collection {1}: {2}'
|
|
.format('.'.join(name_parts), self._collection_name, to_native(ve)))
|
|
|
|
routing_entry = _nested_dict_get(collection_metadata, ['plugin_routing', 'module_utils', '.'.join(module_utils_relative_parts)])
|
|
if not routing_entry:
|
|
return False
|
|
# FIXME: add deprecation warning support
|
|
|
|
dep_or_ts = routing_entry.get('tombstone')
|
|
removed = dep_or_ts is not None
|
|
if not removed:
|
|
dep_or_ts = routing_entry.get('deprecation')
|
|
|
|
if dep_or_ts:
|
|
removal_date = dep_or_ts.get('removal_date')
|
|
removal_version = dep_or_ts.get('removal_version')
|
|
warning_text = dep_or_ts.get('warning_text')
|
|
|
|
msg = 'module_util {0} has been removed'.format('.'.join(name_parts))
|
|
if warning_text:
|
|
msg += ' ({0})'.format(warning_text)
|
|
else:
|
|
msg += '.'
|
|
|
|
display.deprecated(msg, removal_version, removed, removal_date, self._collection_name)
|
|
if 'redirect' in routing_entry:
|
|
self.redirected = True
|
|
source_pkg = '.'.join(name_parts)
|
|
self.is_package = True # treat all redirects as packages
|
|
redirect_target_pkg = routing_entry['redirect']
|
|
|
|
# expand FQCN redirects
|
|
if not redirect_target_pkg.startswith('ansible_collections'):
|
|
split_fqcn = redirect_target_pkg.split('.')
|
|
if len(split_fqcn) < 3:
|
|
raise Exception('invalid redirect for {0}: {1}'.format(source_pkg, redirect_target_pkg))
|
|
# assume it's an FQCN, expand it
|
|
redirect_target_pkg = 'ansible_collections.{0}.{1}.plugins.module_utils.{2}'.format(
|
|
split_fqcn[0], # ns
|
|
split_fqcn[1], # coll
|
|
'.'.join(split_fqcn[2:]) # sub-module_utils remainder
|
|
)
|
|
display.vvv('redirecting module_util {0} to {1}'.format(source_pkg, redirect_target_pkg))
|
|
self.source_code = self._generate_redirect_shim_source(source_pkg, redirect_target_pkg)
|
|
return True
|
|
return False
|
|
|
|
def _get_module_utils_remainder_parts(self, name_parts):
|
|
# subclasses should override to return the name parts after module_utils
|
|
return []
|
|
|
|
def _get_module_utils_remainder(self, name_parts):
|
|
# return the remainder parts as a package string
|
|
return '.'.join(self._get_module_utils_remainder_parts(name_parts))
|
|
|
|
def _find_module(self, name_parts):
|
|
return False
|
|
|
|
def _locate(self, redirect_first=True):
|
|
for candidate_name_parts in self.candidate_names:
|
|
if redirect_first and self._handle_redirect(candidate_name_parts):
|
|
break
|
|
|
|
if self._find_module(candidate_name_parts):
|
|
break
|
|
|
|
if not redirect_first and self._handle_redirect(candidate_name_parts):
|
|
break
|
|
|
|
else: # didn't find what we were looking for- last chance for packages whose parents were redirected
|
|
if self._child_is_redirected: # make fake packages
|
|
self.is_package = True
|
|
self.source_code = ''
|
|
else: # nope, just bail
|
|
return
|
|
|
|
if self.is_package:
|
|
path_parts = candidate_name_parts + ('__init__',)
|
|
else:
|
|
path_parts = candidate_name_parts
|
|
self.found = True
|
|
self.output_path = os.path.join(*path_parts) + '.py'
|
|
self.fq_name_parts = candidate_name_parts
|
|
|
|
def _generate_redirect_shim_source(self, fq_source_module, fq_target_module):
|
|
return """
|
|
import sys
|
|
import {1} as mod
|
|
|
|
sys.modules['{0}'] = mod
|
|
""".format(fq_source_module, fq_target_module)
|
|
|
|
# FIXME: add __repr__ impl
|
|
|
|
|
|
class LegacyModuleUtilLocator(ModuleUtilLocatorBase):
|
|
def __init__(self, fq_name_parts, is_ambiguous=False, mu_paths=None, child_is_redirected=False):
|
|
super(LegacyModuleUtilLocator, self).__init__(fq_name_parts, is_ambiguous, child_is_redirected)
|
|
|
|
if fq_name_parts[0:2] != ('ansible', 'module_utils'):
|
|
raise Exception('this class can only locate from ansible.module_utils, got {0}'.format(fq_name_parts))
|
|
|
|
if fq_name_parts[2] == 'six':
|
|
# FIXME: handle the ansible.module_utils.six._six case with a redirect or an internal _six attr on six itself?
|
|
# six creates its submodules at runtime; convert all these to just 'ansible.module_utils.six'
|
|
fq_name_parts = ('ansible', 'module_utils', 'six')
|
|
self.candidate_names = [fq_name_parts]
|
|
|
|
self._mu_paths = mu_paths
|
|
self._collection_name = 'ansible.builtin' # legacy module utils always look in ansible.builtin for redirects
|
|
self._locate(redirect_first=False) # let local stuff override redirects for legacy
|
|
|
|
def _get_module_utils_remainder_parts(self, name_parts):
|
|
return name_parts[2:] # eg, foo.bar for ansible.module_utils.foo.bar
|
|
|
|
def _find_module(self, name_parts):
|
|
rel_name_parts = self._get_module_utils_remainder_parts(name_parts)
|
|
|
|
# no redirection; try to find the module
|
|
if len(rel_name_parts) == 1: # direct child of module_utils, just search the top-level dirs we were given
|
|
paths = self._mu_paths
|
|
else: # a nested submodule of module_utils, extend the paths given with the intermediate package names
|
|
paths = [os.path.join(p, *rel_name_parts[:-1]) for p in
|
|
self._mu_paths] # extend the MU paths with the relative bit
|
|
|
|
# find_spec needs the full module name
|
|
self._info = info = importlib.machinery.PathFinder.find_spec('.'.join(name_parts), paths)
|
|
if info is not None and os.path.splitext(info.origin)[1] in importlib.machinery.SOURCE_SUFFIXES:
|
|
self.is_package = info.origin.endswith('/__init__.py')
|
|
path = info.origin
|
|
else:
|
|
return False
|
|
self.source_code = _slurp(path)
|
|
|
|
return True
|
|
|
|
|
|
class CollectionModuleUtilLocator(ModuleUtilLocatorBase):
|
|
def __init__(self, fq_name_parts, is_ambiguous=False, child_is_redirected=False, is_optional=False):
|
|
super(CollectionModuleUtilLocator, self).__init__(fq_name_parts, is_ambiguous, child_is_redirected, is_optional)
|
|
|
|
if fq_name_parts[0] != 'ansible_collections':
|
|
raise Exception('CollectionModuleUtilLocator can only locate from ansible_collections, got {0}'.format(fq_name_parts))
|
|
elif len(fq_name_parts) >= 6 and fq_name_parts[3:5] != ('plugins', 'module_utils'):
|
|
raise Exception('CollectionModuleUtilLocator can only locate below ansible_collections.(ns).(coll).plugins.module_utils, got {0}'
|
|
.format(fq_name_parts))
|
|
|
|
self._collection_name = '.'.join(fq_name_parts[1:3])
|
|
|
|
self._locate()
|
|
|
|
def _find_module(self, name_parts):
|
|
# synthesize empty inits for packages down through module_utils- we don't want to allow those to be shipped over, but the
|
|
# package hierarchy needs to exist
|
|
if len(name_parts) < 6:
|
|
self.source_code = ''
|
|
self.is_package = True
|
|
return True
|
|
|
|
# NB: we can't use pkgutil.get_data safely here, since we don't want to import/execute package/module code on
|
|
# the controller while analyzing/assembling the module, so we'll have to manually import the collection's
|
|
# Python package to locate it (import root collection, reassemble resource path beneath, fetch source)
|
|
|
|
collection_pkg_name = '.'.join(name_parts[0:3])
|
|
resource_base_path = os.path.join(*name_parts[3:])
|
|
|
|
src = None
|
|
# look for package_dir first, then module
|
|
try:
|
|
src = pkgutil.get_data(collection_pkg_name, to_native(os.path.join(resource_base_path, '__init__.py')))
|
|
except ImportError:
|
|
pass
|
|
|
|
# TODO: we might want to synthesize fake inits for py3-style packages, for now they're required beneath module_utils
|
|
|
|
if src is not None: # empty string is OK
|
|
self.is_package = True
|
|
else:
|
|
try:
|
|
src = pkgutil.get_data(collection_pkg_name, to_native(resource_base_path + '.py'))
|
|
except ImportError:
|
|
pass
|
|
|
|
if src is None: # empty string is OK
|
|
return False
|
|
|
|
self.source_code = src
|
|
return True
|
|
|
|
def _get_module_utils_remainder_parts(self, name_parts):
|
|
return name_parts[5:] # eg, foo.bar for ansible_collections.ns.coll.plugins.module_utils.foo.bar
|
|
|
|
|
|
def _make_zinfo(filename, date_time, zf=None):
|
|
zinfo = zipfile.ZipInfo(
|
|
filename=filename,
|
|
date_time=date_time
|
|
)
|
|
if zf:
|
|
zinfo.compress_type = zf.compression
|
|
return zinfo
|
|
|
|
|
|
def recursive_finder(name, module_fqn, module_data, zf, date_time=None):
|
|
"""
|
|
Using ModuleDepFinder, make sure we have all of the module_utils files that
|
|
the module and its module_utils files needs. (no longer actually recursive)
|
|
:arg name: Name of the python module we're examining
|
|
:arg module_fqn: Fully qualified name of the python module we're scanning
|
|
:arg module_data: string Python code of the module we're scanning
|
|
:arg zf: An open :python:class:`zipfile.ZipFile` object that holds the Ansible module payload
|
|
which we're assembling
|
|
"""
|
|
if date_time is None:
|
|
date_time = time.gmtime()[:6]
|
|
|
|
# py_module_cache maps python module names to a tuple of the code in the module
|
|
# and the pathname to the module.
|
|
# Here we pre-load it with modules which we create without bothering to
|
|
# read from actual files (In some cases, these need to differ from what ansible
|
|
# ships because they're namespace packages in the module)
|
|
# FIXME: do we actually want ns pkg behavior for these? Seems like they should just be forced to emptyish pkg stubs
|
|
py_module_cache = {
|
|
('ansible',): (
|
|
b'from pkgutil import extend_path\n'
|
|
b'__path__=extend_path(__path__,__name__)\n'
|
|
b'__version__="' + to_bytes(__version__) +
|
|
b'"\n__author__="' + to_bytes(__author__) + b'"\n',
|
|
'ansible/__init__.py'),
|
|
('ansible', 'module_utils'): (
|
|
b'from pkgutil import extend_path\n'
|
|
b'__path__=extend_path(__path__,__name__)\n',
|
|
'ansible/module_utils/__init__.py')}
|
|
|
|
module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)]
|
|
module_utils_paths.append(_MODULE_UTILS_PATH)
|
|
|
|
# Parse the module code and find the imports of ansible.module_utils
|
|
try:
|
|
tree = compile(module_data, '<unknown>', 'exec', ast.PyCF_ONLY_AST)
|
|
except (SyntaxError, IndentationError) as e:
|
|
raise AnsibleError("Unable to import %s due to %s" % (name, e.msg))
|
|
|
|
finder = ModuleDepFinder(module_fqn, tree)
|
|
|
|
# the format of this set is a tuple of the module name and whether or not the import is ambiguous as a module name
|
|
# or an attribute of a module (eg from x.y import z <-- is z a module or an attribute of x.y?)
|
|
modules_to_process = [ModuleUtilsProcessEntry(m, True, False, is_optional=m in finder.optional_imports) for m in finder.submodules]
|
|
|
|
# HACK: basic is currently always required since module global init is currently tied up with AnsiballZ arg input
|
|
modules_to_process.append(ModuleUtilsProcessEntry(('ansible', 'module_utils', 'basic'), False, False, is_optional=False))
|
|
|
|
# we'll be adding new modules inline as we discover them, so just keep going til we've processed them all
|
|
while modules_to_process:
|
|
modules_to_process.sort() # not strictly necessary, but nice to process things in predictable and repeatable order
|
|
py_module_name, is_ambiguous, child_is_redirected, is_optional = modules_to_process.pop(0)
|
|
|
|
if py_module_name in py_module_cache:
|
|
# this is normal; we'll often see the same module imported many times, but we only need to process it once
|
|
continue
|
|
|
|
if py_module_name[0:2] == ('ansible', 'module_utils'):
|
|
module_info = LegacyModuleUtilLocator(py_module_name, is_ambiguous=is_ambiguous,
|
|
mu_paths=module_utils_paths, child_is_redirected=child_is_redirected)
|
|
elif py_module_name[0] == 'ansible_collections':
|
|
module_info = CollectionModuleUtilLocator(py_module_name, is_ambiguous=is_ambiguous,
|
|
child_is_redirected=child_is_redirected, is_optional=is_optional)
|
|
else:
|
|
# FIXME: dot-joined result
|
|
display.warning('ModuleDepFinder improperly found a non-module_utils import %s'
|
|
% [py_module_name])
|
|
continue
|
|
|
|
# Could not find the module. Construct a helpful error message.
|
|
if not module_info.found:
|
|
if is_optional:
|
|
# this was a best-effort optional import that we couldn't find, oh well, move along...
|
|
continue
|
|
# FIXME: use dot-joined candidate names
|
|
msg = 'Could not find imported module support code for {0}. Looked for ({1})'.format(module_fqn, module_info.candidate_names_joined)
|
|
raise AnsibleError(msg)
|
|
|
|
# check the cache one more time with the module we actually found, since the name could be different than the input
|
|
# eg, imported name vs module
|
|
if module_info.fq_name_parts in py_module_cache:
|
|
continue
|
|
|
|
# compile the source, process all relevant imported modules
|
|
try:
|
|
tree = compile(module_info.source_code, '<unknown>', 'exec', ast.PyCF_ONLY_AST)
|
|
except (SyntaxError, IndentationError) as e:
|
|
raise AnsibleError("Unable to import %s due to %s" % (module_info.fq_name_parts, e.msg))
|
|
|
|
finder = ModuleDepFinder('.'.join(module_info.fq_name_parts), tree, module_info.is_package)
|
|
modules_to_process.extend(ModuleUtilsProcessEntry(m, True, False, is_optional=m in finder.optional_imports)
|
|
for m in finder.submodules if m not in py_module_cache)
|
|
|
|
# we've processed this item, add it to the output list
|
|
py_module_cache[module_info.fq_name_parts] = (module_info.source_code, module_info.output_path)
|
|
|
|
# ensure we process all ancestor package inits
|
|
accumulated_pkg_name = []
|
|
for pkg in module_info.fq_name_parts[:-1]:
|
|
accumulated_pkg_name.append(pkg) # we're accumulating this across iterations
|
|
normalized_name = tuple(accumulated_pkg_name) # extra machinations to get a hashable type (list is not)
|
|
if normalized_name not in py_module_cache:
|
|
modules_to_process.append(ModuleUtilsProcessEntry(normalized_name, False, module_info.redirected, is_optional=is_optional))
|
|
|
|
for py_module_name in py_module_cache:
|
|
py_module_file_name = py_module_cache[py_module_name][1]
|
|
|
|
zf.writestr(
|
|
_make_zinfo(py_module_file_name, date_time, zf=zf),
|
|
py_module_cache[py_module_name][0]
|
|
)
|
|
mu_file = to_text(py_module_file_name, errors='surrogate_or_strict')
|
|
display.vvvvv("Including module_utils file %s" % mu_file)
|
|
|
|
|
|
def _is_binary(b_module_data):
|
|
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
|
|
start = b_module_data[:1024]
|
|
return bool(start.translate(None, textchars))
|
|
|
|
|
|
def _get_ansible_module_fqn(module_path):
|
|
"""
|
|
Get the fully qualified name for an ansible module based on its pathname
|
|
|
|
remote_module_fqn is the fully qualified name. Like ansible.modules.system.ping
|
|
Or ansible_collections.Namespace.Collection_name.plugins.modules.ping
|
|
.. warning:: This function is for ansible modules only. It won't work for other things
|
|
(non-module plugins, etc)
|
|
"""
|
|
remote_module_fqn = None
|
|
|
|
# Is this a core module?
|
|
match = CORE_LIBRARY_PATH_RE.search(module_path)
|
|
if not match:
|
|
# Is this a module in a collection?
|
|
match = COLLECTION_PATH_RE.search(module_path)
|
|
|
|
# We can tell the FQN for core modules and collection modules
|
|
if match:
|
|
path = match.group('path')
|
|
if '.' in path:
|
|
# FQNs must be valid as python identifiers. This sanity check has failed.
|
|
# we could check other things as well
|
|
raise ValueError('Module name (or path) was not a valid python identifier')
|
|
|
|
remote_module_fqn = '.'.join(path.split('/'))
|
|
else:
|
|
# Currently we do not handle modules in roles so we can end up here for that reason
|
|
raise ValueError("Unable to determine module's fully qualified name")
|
|
|
|
return remote_module_fqn
|
|
|
|
|
|
def _add_module_to_zip(zf, date_time, remote_module_fqn, b_module_data):
|
|
"""Add a module from ansible or from an ansible collection into the module zip"""
|
|
module_path_parts = remote_module_fqn.split('.')
|
|
|
|
# Write the module
|
|
module_path = '/'.join(module_path_parts) + '.py'
|
|
zf.writestr(
|
|
_make_zinfo(module_path, date_time, zf=zf),
|
|
b_module_data
|
|
)
|
|
|
|
# Write the __init__.py's necessary to get there
|
|
if module_path_parts[0] == 'ansible':
|
|
# The ansible namespace is setup as part of the module_utils setup...
|
|
start = 2
|
|
existing_paths = frozenset()
|
|
else:
|
|
# ... but ansible_collections and other toplevels are not
|
|
start = 1
|
|
existing_paths = frozenset(zf.namelist())
|
|
|
|
for idx in range(start, len(module_path_parts)):
|
|
package_path = '/'.join(module_path_parts[:idx]) + '/__init__.py'
|
|
# If a collections module uses module_utils from a collection then most packages will have already been added by recursive_finder.
|
|
if package_path in existing_paths:
|
|
continue
|
|
# Note: We don't want to include more than one ansible module in a payload at this time
|
|
# so no need to fill the __init__.py with namespace code
|
|
zf.writestr(
|
|
_make_zinfo(package_path, date_time, zf=zf),
|
|
b''
|
|
)
|
|
|
|
|
|
def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, async_timeout, become,
|
|
become_method, become_user, become_password, become_flags, environment, remote_is_local=False):
|
|
"""
|
|
Given the source of the module, convert it to a Jinja2 template to insert
|
|
module code and return whether it's a new or old style module.
|
|
"""
|
|
module_substyle = module_style = 'old'
|
|
|
|
# module_style is something important to calling code (ActionBase). It
|
|
# determines how arguments are formatted (json vs k=v) and whether
|
|
# a separate arguments file needs to be sent over the wire.
|
|
# module_substyle is extra information that's useful internally. It tells
|
|
# us what we have to look to substitute in the module files and whether
|
|
# we're using module replacer or ansiballz to format the module itself.
|
|
if _is_binary(b_module_data):
|
|
module_substyle = module_style = 'binary'
|
|
elif REPLACER in b_module_data:
|
|
# Do REPLACER before from ansible.module_utils because we need make sure
|
|
# we substitute "from ansible.module_utils basic" for REPLACER
|
|
module_style = 'new'
|
|
module_substyle = 'python'
|
|
b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
|
|
elif NEW_STYLE_PYTHON_MODULE_RE.search(b_module_data):
|
|
module_style = 'new'
|
|
module_substyle = 'python'
|
|
elif REPLACER_WINDOWS in b_module_data:
|
|
module_style = 'new'
|
|
module_substyle = 'powershell'
|
|
b_module_data = b_module_data.replace(REPLACER_WINDOWS, b'#Requires -Module Ansible.ModuleUtils.Legacy')
|
|
elif re.search(b'#Requires -Module', b_module_data, re.IGNORECASE) \
|
|
or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE)\
|
|
or re.search(b'#AnsibleRequires -OSVersion', b_module_data, re.IGNORECASE) \
|
|
or re.search(b'#AnsibleRequires -Powershell', b_module_data, re.IGNORECASE) \
|
|
or re.search(b'#AnsibleRequires -CSharpUtil', b_module_data, re.IGNORECASE):
|
|
module_style = 'new'
|
|
module_substyle = 'powershell'
|
|
elif REPLACER_JSONARGS in b_module_data:
|
|
module_style = 'new'
|
|
module_substyle = 'jsonargs'
|
|
elif b'WANT_JSON' in b_module_data:
|
|
module_substyle = module_style = 'non_native_want_json'
|
|
|
|
shebang = None
|
|
# Neither old-style, non_native_want_json nor binary modules should be modified
|
|
# except for the shebang line (Done by modify_module)
|
|
if module_style in ('old', 'non_native_want_json', 'binary'):
|
|
return b_module_data, module_style, shebang
|
|
|
|
output = BytesIO()
|
|
|
|
try:
|
|
remote_module_fqn = _get_ansible_module_fqn(module_path)
|
|
except ValueError:
|
|
# Modules in roles currently are not found by the fqn heuristic so we
|
|
# fallback to this. This means that relative imports inside a module from
|
|
# a role may fail. Absolute imports should be used for future-proofness.
|
|
# People should start writing collections instead of modules in roles so we
|
|
# may never fix this
|
|
display.debug('ANSIBALLZ: Could not determine module FQN')
|
|
remote_module_fqn = 'ansible.modules.%s' % module_name
|
|
|
|
if module_substyle == 'python':
|
|
date_time = time.gmtime()[:6]
|
|
if date_time[0] < 1980:
|
|
date_string = datetime.datetime(*date_time, tzinfo=datetime.timezone.utc).strftime('%c')
|
|
raise AnsibleError(f'Cannot create zipfile due to pre-1980 configured date: {date_string}')
|
|
params = dict(ANSIBLE_MODULE_ARGS=module_args,)
|
|
try:
|
|
python_repred_params = repr(json.dumps(params, cls=AnsibleJSONEncoder, vault_to_text=True))
|
|
except TypeError as e:
|
|
raise AnsibleError("Unable to pass options to module, they must be JSON serializable: %s" % to_native(e))
|
|
|
|
try:
|
|
compression_method = getattr(zipfile, module_compression)
|
|
except AttributeError:
|
|
display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression)
|
|
compression_method = zipfile.ZIP_STORED
|
|
|
|
lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache')
|
|
cached_module_filename = os.path.join(lookup_path, "%s-%s" % (remote_module_fqn, module_compression))
|
|
|
|
zipdata = None
|
|
# Optimization -- don't lock if the module has already been cached
|
|
if os.path.exists(cached_module_filename):
|
|
display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename)
|
|
with open(cached_module_filename, 'rb') as module_data:
|
|
zipdata = module_data.read()
|
|
else:
|
|
if module_name in action_write_locks.action_write_locks:
|
|
display.debug('ANSIBALLZ: Using lock for %s' % module_name)
|
|
lock = action_write_locks.action_write_locks[module_name]
|
|
else:
|
|
# If the action plugin directly invokes the module (instead of
|
|
# going through a strategy) then we don't have a cross-process
|
|
# Lock specifically for this module. Use the "unexpected
|
|
# module" lock instead
|
|
display.debug('ANSIBALLZ: Using generic lock for %s' % module_name)
|
|
lock = action_write_locks.action_write_locks[None]
|
|
|
|
display.debug('ANSIBALLZ: Acquiring lock')
|
|
with lock:
|
|
display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock))
|
|
# Check that no other process has created this while we were
|
|
# waiting for the lock
|
|
if not os.path.exists(cached_module_filename):
|
|
display.debug('ANSIBALLZ: Creating module')
|
|
# Create the module zip data
|
|
zipoutput = BytesIO()
|
|
zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
|
|
|
|
# walk the module imports, looking for module_utils to send- they'll be added to the zipfile
|
|
recursive_finder(module_name, remote_module_fqn, b_module_data, zf, date_time)
|
|
|
|
display.debug('ANSIBALLZ: Writing module into payload')
|
|
_add_module_to_zip(zf, date_time, remote_module_fqn, b_module_data)
|
|
|
|
zf.close()
|
|
zipdata = base64.b64encode(zipoutput.getvalue())
|
|
|
|
# Write the assembled module to a temp file (write to temp
|
|
# so that no one looking for the file reads a partially
|
|
# written file)
|
|
#
|
|
# FIXME: Once split controller/remote is merged, this can be simplified to
|
|
# os.makedirs(lookup_path, exist_ok=True)
|
|
if not os.path.exists(lookup_path):
|
|
try:
|
|
# Note -- if we have a global function to setup, that would
|
|
# be a better place to run this
|
|
os.makedirs(lookup_path)
|
|
except OSError:
|
|
# Multiple processes tried to create the directory. If it still does not
|
|
# exist, raise the original exception.
|
|
if not os.path.exists(lookup_path):
|
|
raise
|
|
display.debug('ANSIBALLZ: Writing module')
|
|
with open(cached_module_filename + '-part', 'wb') as f:
|
|
f.write(zipdata)
|
|
|
|
# Rename the file into its final position in the cache so
|
|
# future users of this module can read it off the
|
|
# filesystem instead of constructing from scratch.
|
|
display.debug('ANSIBALLZ: Renaming module')
|
|
os.rename(cached_module_filename + '-part', cached_module_filename)
|
|
display.debug('ANSIBALLZ: Done creating module')
|
|
|
|
if zipdata is None:
|
|
display.debug('ANSIBALLZ: Reading module after lock')
|
|
# Another process wrote the file while we were waiting for
|
|
# the write lock. Go ahead and read the data from disk
|
|
# instead of re-creating it.
|
|
try:
|
|
with open(cached_module_filename, 'rb') as f:
|
|
zipdata = f.read()
|
|
except IOError:
|
|
raise AnsibleError('A different worker process failed to create module file. '
|
|
'Look at traceback for that process for debugging information.')
|
|
zipdata = to_text(zipdata, errors='surrogate_or_strict')
|
|
|
|
o_interpreter, o_args = _extract_interpreter(b_module_data)
|
|
if o_interpreter is None:
|
|
o_interpreter = u'/usr/bin/python'
|
|
|
|
shebang, interpreter = _get_shebang(o_interpreter, task_vars, templar, o_args, remote_is_local=remote_is_local)
|
|
|
|
# FUTURE: the module cache entry should be invalidated if we got this value from a host-dependent source
|
|
rlimit_nofile = C.config.get_config_value('PYTHON_MODULE_RLIMIT_NOFILE', variables=task_vars)
|
|
|
|
if not isinstance(rlimit_nofile, int):
|
|
rlimit_nofile = int(templar.template(rlimit_nofile))
|
|
|
|
if rlimit_nofile:
|
|
rlimit = ANSIBALLZ_RLIMIT_TEMPLATE % dict(
|
|
rlimit_nofile=rlimit_nofile,
|
|
)
|
|
else:
|
|
rlimit = ''
|
|
|
|
coverage_config = os.environ.get('_ANSIBLE_COVERAGE_CONFIG')
|
|
|
|
if coverage_config:
|
|
coverage_output = os.environ['_ANSIBLE_COVERAGE_OUTPUT']
|
|
|
|
if coverage_output:
|
|
# Enable code coverage analysis of the module.
|
|
# This feature is for internal testing and may change without notice.
|
|
coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict(
|
|
coverage_config=coverage_config,
|
|
coverage_output=coverage_output,
|
|
)
|
|
else:
|
|
# Verify coverage is available without importing it.
|
|
# This will detect when a module would fail with coverage enabled with minimal overhead.
|
|
coverage = ANSIBALLZ_COVERAGE_CHECK_TEMPLATE
|
|
else:
|
|
coverage = ''
|
|
|
|
output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict(
|
|
zipdata=zipdata,
|
|
ansible_module=module_name,
|
|
module_fqn=remote_module_fqn,
|
|
params=python_repred_params,
|
|
shebang=shebang,
|
|
coding=ENCODING_STRING,
|
|
date_time=date_time,
|
|
coverage=coverage,
|
|
rlimit=rlimit,
|
|
)))
|
|
b_module_data = output.getvalue()
|
|
|
|
elif module_substyle == 'powershell':
|
|
# Powershell/winrm don't actually make use of shebang so we can
|
|
# safely set this here. If we let the fallback code handle this
|
|
# it can fail in the presence of the UTF8 BOM commonly added by
|
|
# Windows text editors
|
|
shebang = u'#!powershell'
|
|
# create the common exec wrapper payload and set that as the module_data
|
|
# bytes
|
|
b_module_data = ps_manifest._create_powershell_wrapper(
|
|
b_module_data, module_path, module_args, environment,
|
|
async_timeout, become, become_method, become_user, become_password,
|
|
become_flags, module_substyle, task_vars, remote_module_fqn
|
|
)
|
|
|
|
elif module_substyle == 'jsonargs':
|
|
module_args_json = to_bytes(json.dumps(module_args, cls=AnsibleJSONEncoder, vault_to_text=True))
|
|
|
|
# these strings could be included in a third-party module but
|
|
# officially they were included in the 'basic' snippet for new-style
|
|
# python modules (which has been replaced with something else in
|
|
# ansiballz) If we remove them from jsonargs-style module replacer
|
|
# then we can remove them everywhere.
|
|
python_repred_args = to_bytes(repr(module_args_json))
|
|
b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
|
|
b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args)
|
|
b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
|
|
|
|
# The main event -- substitute the JSON args string into the module
|
|
b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
|
|
|
|
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
|
|
b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility)
|
|
|
|
return (b_module_data, module_style, shebang)
|
|
|
|
|
|
def _extract_interpreter(b_module_data):
|
|
"""
|
|
Used to extract shebang expression from binary module data and return a text
|
|
string with the shebang, or None if no shebang is detected.
|
|
"""
|
|
|
|
interpreter = None
|
|
args = []
|
|
b_lines = b_module_data.split(b"\n", 1)
|
|
if b_lines[0].startswith(b"#!"):
|
|
b_shebang = b_lines[0].strip()
|
|
|
|
# shlex.split needs text on Python 3
|
|
cli_split = shlex.split(to_text(b_shebang[2:], errors='surrogate_or_strict'))
|
|
|
|
# convert args to text
|
|
cli_split = [to_text(a, errors='surrogate_or_strict') for a in cli_split]
|
|
interpreter = cli_split[0]
|
|
args = cli_split[1:]
|
|
|
|
return interpreter, args
|
|
|
|
|
|
def modify_module(module_name, module_path, module_args, templar, task_vars=None, module_compression='ZIP_STORED', async_timeout=0, become=False,
|
|
become_method=None, become_user=None, become_password=None, become_flags=None, environment=None, remote_is_local=False):
|
|
"""
|
|
Used to insert chunks of code into modules before transfer rather than
|
|
doing regular python imports. This allows for more efficient transfer in
|
|
a non-bootstrapping scenario by not moving extra files over the wire and
|
|
also takes care of embedding arguments in the transferred modules.
|
|
|
|
This version is done in such a way that local imports can still be
|
|
used in the module code, so IDEs don't have to be aware of what is going on.
|
|
|
|
Example:
|
|
|
|
from ansible.module_utils.basic import *
|
|
|
|
... will result in the insertion of basic.py into the module
|
|
from the module_utils/ directory in the source tree.
|
|
|
|
For powershell, this code effectively no-ops, as the exec wrapper requires access to a number of
|
|
properties not available here.
|
|
|
|
"""
|
|
task_vars = {} if task_vars is None else task_vars
|
|
environment = {} if environment is None else environment
|
|
|
|
with open(module_path, 'rb') as f:
|
|
|
|
# read in the module source
|
|
b_module_data = f.read()
|
|
|
|
(b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression,
|
|
async_timeout=async_timeout, become=become, become_method=become_method,
|
|
become_user=become_user, become_password=become_password, become_flags=become_flags,
|
|
environment=environment, remote_is_local=remote_is_local)
|
|
|
|
if module_style == 'binary':
|
|
return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))
|
|
elif shebang is None:
|
|
interpreter, args = _extract_interpreter(b_module_data)
|
|
# No interpreter/shebang, assume a binary module?
|
|
if interpreter is not None:
|
|
|
|
shebang, new_interpreter = _get_shebang(interpreter, task_vars, templar, args, remote_is_local=remote_is_local)
|
|
|
|
# update shebang
|
|
b_lines = b_module_data.split(b"\n", 1)
|
|
|
|
if interpreter != new_interpreter:
|
|
b_lines[0] = to_bytes(shebang, errors='surrogate_or_strict', nonstring='passthru')
|
|
|
|
if os.path.basename(interpreter).startswith(u'python'):
|
|
b_lines.insert(1, b_ENCODING_STRING)
|
|
|
|
b_module_data = b"\n".join(b_lines)
|
|
|
|
return (b_module_data, module_style, shebang)
|
|
|
|
|
|
def get_action_args_with_defaults(action, args, defaults, templar, action_groups=None):
|
|
# Get the list of groups that contain this action
|
|
if action_groups is None:
|
|
msg = (
|
|
"Finding module_defaults for action %s. "
|
|
"The caller has not passed the action_groups, so any "
|
|
"that may include this action will be ignored."
|
|
)
|
|
display.warning(msg=msg)
|
|
group_names = []
|
|
else:
|
|
group_names = action_groups.get(action, [])
|
|
|
|
tmp_args = {}
|
|
module_defaults = {}
|
|
|
|
# Merge latest defaults into dict, since they are a list of dicts
|
|
if isinstance(defaults, list):
|
|
for default in defaults:
|
|
module_defaults.update(default)
|
|
|
|
# module_defaults keys are static, but the values may be templated
|
|
module_defaults = templar.template(module_defaults)
|
|
for default in module_defaults:
|
|
if default.startswith('group/'):
|
|
group_name = default.split('group/')[-1]
|
|
if group_name in group_names:
|
|
tmp_args.update((module_defaults.get('group/%s' % group_name) or {}).copy())
|
|
|
|
# handle specific action defaults
|
|
tmp_args.update(module_defaults.get(action, {}).copy())
|
|
|
|
# direct args override all
|
|
tmp_args.update(args)
|
|
|
|
return tmp_args
|