mirror of https://github.com/ansible/ansible.git
Python interpreter discovery (#50163)
* Python interpreter discovery * No longer blindly default to only `/usr/bin/python` * `ansible_python_interpreter` defaults to `auto_legacy`, which will discover the platform Python interpreter on some platforms (but still favor `/usr/bin/python` if present for backward compatibility). Use `auto` to always use the discovered interpreter, append `_silent` to either value to suppress warnings. * includes new doc utility method `get_versioned_doclink` to generate a major.minor versioned doclink against docs.ansible.com (or some other config-overridden URL) * docs revisions for python interpreter discovery (cherry picked from commit 5b53c0012ab7212304c28fdd24cb33fd8ff755c2) * verify output on some distros, cleanuppull/53099/head
Matt Davis
5 years ago
committed by
GitHub
parent
b8a82f5930
commit
4d3a6123d5
@ -0,0 +1,5 @@
|
||||
major_changes:
|
||||
- Python interpreter discovery - The first time a Python module runs on a target, Ansible will attempt to discover the
|
||||
proper default Python interpreter to use for the target platform/version (instead of immediately defaulting to
|
||||
``/usr/bin/python``). You can override this behavior by setting ``ansible_python_interpreter`` or via config. (see
|
||||
https://github.com/ansible/ansible/pull/50163)
|
||||
@ -0,0 +1,51 @@
|
||||
.. _interpreter_discovery:
|
||||
|
||||
Interpreter Discovery
|
||||
=====================
|
||||
|
||||
Most Ansible modules that execute under a POSIX environment require a Python
|
||||
interpreter on the target host. Unless configured otherwise, Ansible will
|
||||
attempt to discover a suitable Python interpreter on each target host
|
||||
the first time a Python module is executed for that host.
|
||||
|
||||
To control the discovery behavior:
|
||||
|
||||
* for individual hosts and groups, use the ``ansible_python_interpreter`` inventory variable
|
||||
* globally, use the ``interpreter_python`` key in the ``[defaults]`` section of ``ansible.cfg``
|
||||
|
||||
Use one of the following values:
|
||||
|
||||
auto_legacy : (default in 2.8)
|
||||
Detects the target OS platform, distribution, and version, then consults a
|
||||
table listing the correct Python interpreter and path for each
|
||||
platform/distribution/version. If an entry is found, and ``/usr/bin/python`` is absent, uses the discovered interpreter (and path). If an entry
|
||||
is found, and ``/usr/bin/python`` is present, uses ``/usr/bin/python``
|
||||
and issues a warning.
|
||||
This exception provides temporary compatibility with previous versions of
|
||||
Ansible that always defaulted to ``/usr/bin/python``, so if you have
|
||||
installed Python and other dependencies at ``usr/bin/python`` on some hosts,
|
||||
Ansible will find and use them with this setting.
|
||||
If no entry is found, or the listed Python is not present on the
|
||||
target host, searches a list of common Python interpreter
|
||||
paths and uses the first one found; also issues a warning that future
|
||||
installation of another Python interpreter could alter the one chosen.
|
||||
|
||||
auto : (future default in 2.12)
|
||||
Detects the target OS platform, distribution, and version, then consults a
|
||||
table listing the correct Python interpreter and path for each
|
||||
platform/distribution/version. If an entry is found, uses the discovered
|
||||
interpreter.
|
||||
If no entry is found, or the listed Python is not present on the
|
||||
target host, searches a list of common Python interpreter
|
||||
paths and uses the first one found; also issues a warning that future
|
||||
installation of another Python interpreter could alter the one chosen.
|
||||
|
||||
auto_legacy_silent
|
||||
Same as ``auto_legacy``, but does not issue warnings.
|
||||
|
||||
auto_silent
|
||||
Same as ``auto``, but does not issue warnings.
|
||||
|
||||
You can still set ``ansible_python_interpreter`` to a specific path at any
|
||||
variable level (for example, in host_vars, in vars files, in playbooks, etc.).
|
||||
Setting a specific path completely disables automatic interpreter discovery; Ansible always uses the path specified.
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright: (c) 2018 Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# FUTURE: this could be swapped out for our bundled version of distro to move more complete platform
|
||||
# logic to the targets, so long as we maintain Py2.6 compat and don't need to do any kind of script assembly
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import json
|
||||
import platform
|
||||
import io
|
||||
import os
|
||||
|
||||
|
||||
def read_utf8_file(path, encoding='utf-8'):
|
||||
if not os.access(path, os.R_OK):
|
||||
return None
|
||||
with io.open(path, 'r', encoding=encoding) as fd:
|
||||
content = fd.read()
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def get_platform_info():
|
||||
result = dict(platform_dist_result=[])
|
||||
|
||||
if hasattr(platform, 'dist'):
|
||||
result['platform_dist_result'] = platform.dist()
|
||||
|
||||
osrelease_content = read_utf8_file('/etc/os-release')
|
||||
# try to fall back to /usr/lib/os-release
|
||||
if not osrelease_content:
|
||||
osrelease_content = read_utf8_file('/usr/lib/os-release')
|
||||
|
||||
result['osrelease_content'] = osrelease_content
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
info = get_platform_info()
|
||||
|
||||
print(json.dumps(info))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,203 @@
|
||||
# Copyright: (c) 2018 Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import bisect
|
||||
import json
|
||||
import pkgutil
|
||||
import re
|
||||
|
||||
from ansible import constants as C
|
||||
from ansible.module_utils._text import to_text
|
||||
from ansible.module_utils.distro import LinuxDistribution
|
||||
from ansible.utils.display import Display
|
||||
from ansible.utils.plugin_docs import get_versioned_doclink
|
||||
from distutils.version import LooseVersion
|
||||
from traceback import format_exc
|
||||
|
||||
display = Display()
|
||||
foundre = re.compile(r'(?s)PLATFORM[\r\n]+(.*)FOUND(.*)ENDFOUND')
|
||||
|
||||
|
||||
class InterpreterDiscoveryRequiredError(Exception):
|
||||
def __init__(self, message, interpreter_name, discovery_mode):
|
||||
super(InterpreterDiscoveryRequiredError, self).__init__(message)
|
||||
self.interpreter_name = interpreter_name
|
||||
self.discovery_mode = discovery_mode
|
||||
|
||||
def __str__(self):
|
||||
return self.message
|
||||
|
||||
def __repr__(self):
|
||||
# TODO: proper repr impl
|
||||
return self.message
|
||||
|
||||
|
||||
def discover_interpreter(action, interpreter_name, discovery_mode, task_vars):
|
||||
# interpreter discovery is a 2-step process with the target. First, we use a simple shell-agnostic bootstrap to
|
||||
# get the system type from uname, and find any random Python that can get us the info we need. For supported
|
||||
# target OS types, we'll dispatch a Python script that calls plaform.dist() (for older platforms, where available)
|
||||
# and brings back /etc/os-release (if present). The proper Python path is looked up in a table of known
|
||||
# distros/versions with included Pythons; if nothing is found, depending on the discovery mode, either the
|
||||
# default fallback of /usr/bin/python is used (if we know it's there), or discovery fails.
|
||||
|
||||
# FUTURE: add logical equivalence for "python3" in the case of py3-only modules?
|
||||
if interpreter_name != 'python':
|
||||
raise ValueError('Interpreter discovery not supported for {0}'.format(interpreter_name))
|
||||
|
||||
host = task_vars.get('inventory_hostname', 'unknown')
|
||||
res = None
|
||||
platform_type = 'unknown'
|
||||
found_interpreters = ['/usr/bin/python'] # fallback value
|
||||
is_auto_legacy = discovery_mode.startswith('auto_legacy')
|
||||
is_silent = discovery_mode.endswith('_silent')
|
||||
|
||||
try:
|
||||
platform_python_map = C.config.get_config_value('INTERPRETER_PYTHON_DISTRO_MAP', variables=task_vars)
|
||||
bootstrap_python_list = C.config.get_config_value('INTERPRETER_PYTHON_FALLBACK', variables=task_vars)
|
||||
|
||||
display.vvv(msg="Attempting {0} interpreter discovery".format(interpreter_name), host=host)
|
||||
|
||||
# not all command -v impls accept a list of commands, so we have to call it once per python
|
||||
command_list = ["command -v '%s'" % py for py in bootstrap_python_list]
|
||||
shell_bootstrap = "echo PLATFORM; uname; echo FOUND; {0}; echo ENDFOUND".format('; '.join(command_list))
|
||||
|
||||
# FUTURE: in most cases we probably don't want to use become, but maybe sometimes we do?
|
||||
res = action._low_level_execute_command(shell_bootstrap, sudoable=False)
|
||||
|
||||
raw_stdout = res.get('stdout', '')
|
||||
|
||||
match = foundre.match(raw_stdout)
|
||||
|
||||
if not match:
|
||||
display.debug('raw interpreter discovery output: {0}'.format(raw_stdout), host=host)
|
||||
raise ValueError('unexpected output from Python interpreter discovery')
|
||||
|
||||
platform_type = match.groups()[0].lower().strip()
|
||||
|
||||
found_interpreters = [interp.strip() for interp in match.groups()[1].splitlines() if interp.startswith('/')]
|
||||
|
||||
display.debug("found interpreters: {0}".format(found_interpreters), host=host)
|
||||
|
||||
if not found_interpreters:
|
||||
action._discovery_warnings.append('No python interpreters found for host {0} (tried {1})'.format(host, bootstrap_python_list))
|
||||
# this is lame, but returning None or throwing an exception is uglier
|
||||
return '/usr/bin/python'
|
||||
|
||||
if platform_type != 'linux':
|
||||
raise NotImplementedError('unsupported platform for extended discovery: {0}'.format(platform_type))
|
||||
|
||||
platform_script = pkgutil.get_data('ansible.executor.discovery', 'python_target.py')
|
||||
|
||||
# FUTURE: respect pipelining setting instead of just if the connection supports it?
|
||||
if action._connection.has_pipelining:
|
||||
res = action._low_level_execute_command(found_interpreters[0], sudoable=False, in_data=platform_script)
|
||||
else:
|
||||
# FUTURE: implement on-disk case (via script action or ?)
|
||||
raise NotImplementedError('pipelining support required for extended interpreter discovery')
|
||||
|
||||
platform_info = json.loads(res.get('stdout'))
|
||||
|
||||
distro, version = _get_linux_distro(platform_info)
|
||||
|
||||
if not distro or not version:
|
||||
raise NotImplementedError('unable to get Linux distribution/version info')
|
||||
|
||||
version_map = platform_python_map.get(distro.lower().strip())
|
||||
if not version_map:
|
||||
raise NotImplementedError('unsupported Linux distribution: {0}'.format(distro))
|
||||
|
||||
platform_interpreter = _version_fuzzy_match(version, version_map)
|
||||
|
||||
# provide a transition period for hosts that were using /usr/bin/python previously (but shouldn't have been)
|
||||
if is_auto_legacy:
|
||||
if platform_interpreter != '/usr/bin/python' and '/usr/bin/python' in found_interpreters:
|
||||
# FIXME: support comments in sivel's deprecation scanner so we can get reminded on this
|
||||
if not is_silent:
|
||||
action._discovery_deprecation_warnings.append(dict(
|
||||
msg="Distribution {0} {1} should use {2}, but is using "
|
||||
"/usr/bin/python for backward compatibility with prior Ansible releases. "
|
||||
"A future Ansible release will default to using the discovered platform "
|
||||
"python for this host. See {3} for more information"
|
||||
.format(distro, version, platform_interpreter,
|
||||
get_versioned_doclink('reference_appendices/interpreter_discovery.html')),
|
||||
version='2.12'))
|
||||
return '/usr/bin/python'
|
||||
|
||||
if platform_interpreter not in found_interpreters:
|
||||
if platform_interpreter not in bootstrap_python_list:
|
||||
# sanity check to make sure we looked for it
|
||||
if not is_silent:
|
||||
action._discovery_warnings \
|
||||
.append("Platform interpreter {0} is missing from bootstrap list"
|
||||
.format(platform_interpreter))
|
||||
|
||||
if not is_silent:
|
||||
action._discovery_warnings \
|
||||
.append("Distribution {0} {1} should use {2}, but is using {3}, since the "
|
||||
"discovered platform python interpreter was not present. See {4} "
|
||||
"for more information."
|
||||
.format(distro, version, platform_interpreter, found_interpreters[0],
|
||||
get_versioned_doclink('reference_appendices/interpreter_discovery.html')))
|
||||
return found_interpreters[0]
|
||||
|
||||
return platform_interpreter
|
||||
except NotImplementedError as ex:
|
||||
display.vvv(msg='Python interpreter discovery fallback ({0})'.format(to_text(ex)), host=host)
|
||||
except Exception as ex:
|
||||
if not is_silent:
|
||||
display.warning(msg='Unhandled error in Python interpreter discovery for host {0}: {1}'.format(host, to_text(ex)))
|
||||
display.debug(msg='Interpreter discovery traceback:\n{0}'.format(to_text(format_exc())), host=host)
|
||||
if res and res.get('stderr'):
|
||||
display.vvv(msg='Interpreter discovery remote stderr:\n{0}'.format(to_text(res.get('stderr'))), host=host)
|
||||
|
||||
if not is_silent:
|
||||
action._discovery_warnings \
|
||||
.append("Platform {0} is using the discovered Python interpreter at {1}, but future installation of "
|
||||
"another Python interpreter could change this. See {2} "
|
||||
"for more information."
|
||||
.format(platform_type, found_interpreters[0],
|
||||
get_versioned_doclink('reference_appendices/interpreter_discovery.html')))
|
||||
return found_interpreters[0]
|
||||
|
||||
|
||||
def _get_linux_distro(platform_info):
|
||||
dist_result = platform_info.get('platform_dist_result', [])
|
||||
|
||||
if len(dist_result) == 3 and any(dist_result):
|
||||
return dist_result[0], dist_result[1]
|
||||
|
||||
osrelease_content = platform_info.get('osrelease_content')
|
||||
|
||||
if not osrelease_content:
|
||||
return '', ''
|
||||
|
||||
osr = LinuxDistribution._parse_os_release_content(osrelease_content)
|
||||
|
||||
return osr.get('id', ''), osr.get('version_id', '')
|
||||
|
||||
|
||||
def _version_fuzzy_match(version, version_map):
|
||||
# try exact match first
|
||||
res = version_map.get(version)
|
||||
if res:
|
||||
return res
|
||||
|
||||
sorted_looseversions = sorted([LooseVersion(v) for v in version_map.keys()])
|
||||
|
||||
find_looseversion = LooseVersion(version)
|
||||
|
||||
# slot match; return nearest previous version we're newer than
|
||||
kpos = bisect.bisect(sorted_looseversions, find_looseversion)
|
||||
|
||||
if kpos == 0:
|
||||
# older than everything in the list, return the oldest version
|
||||
# TODO: warning-worthy?
|
||||
return version_map.get(sorted_looseversions[0].vstring)
|
||||
|
||||
# TODO: is "past the end of the list" warning-worthy too (at least if it's not a major version match)?
|
||||
|
||||
# return the next-oldest entry that we're newer than...
|
||||
return version_map.get(sorted_looseversions[kpos - 1].vstring)
|
||||
@ -0,0 +1,2 @@
|
||||
shippable/posix/group1
|
||||
non_local # workaround to allow override of ansible_python_interpreter; disables coverage on this integration target
|
||||
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
|
||||
# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import sys
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
|
||||
|
||||
def main():
|
||||
result = dict(changed=False)
|
||||
|
||||
module = AnsibleModule(argument_spec=dict(
|
||||
facts=dict(type=dict, default={})
|
||||
))
|
||||
|
||||
result['ansible_facts'] = module.params['facts']
|
||||
result['running_python_interpreter'] = sys.executable
|
||||
|
||||
module.exit_json(**result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,145 @@
|
||||
- name: ensure we can override ansible_python_interpreter
|
||||
vars:
|
||||
ansible_python_interpreter: overriddenpython
|
||||
assert:
|
||||
that:
|
||||
- ansible_python_interpreter == 'overriddenpython'
|
||||
fail_msg: "'ansible_python_interpreter' appears to be set at a high precedence to {{ ansible_python_interpreter }},
|
||||
which breaks this test."
|
||||
|
||||
- name: snag some facts to validate for later
|
||||
set_fact:
|
||||
distro: '{{ ansible_distribution | default("unknown") | lower }}'
|
||||
distro_version: '{{ ansible_distribution_version | default("unknown") }}'
|
||||
os_family: '{{ ansible_os_family | default("unknown") }}'
|
||||
|
||||
- name: test that python discovery is working and that fact persistence makes it only run once
|
||||
block:
|
||||
- name: clear facts to force interpreter discovery to run
|
||||
meta: clear_facts
|
||||
|
||||
- name: trigger discovery with auto
|
||||
vars:
|
||||
ansible_python_interpreter: auto
|
||||
ping:
|
||||
register: auto_out
|
||||
|
||||
- name: get the interpreter being used on the target to execute modules
|
||||
vars:
|
||||
# keep this set so we can verify we didn't repeat discovery
|
||||
ansible_python_interpreter: auto
|
||||
test_echo_module:
|
||||
register: echoout
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- auto_out.ansible_facts.discovered_interpreter_python is defined
|
||||
- echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python
|
||||
# verify that discovery didn't run again (if it did, we'd have the fact in the result)
|
||||
- echoout.ansible_facts is not defined or echoout.ansible_facts.discovered_interpreter_python is not defined
|
||||
|
||||
|
||||
- name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result
|
||||
block:
|
||||
- name: clear facts to force interpreter discovery to run
|
||||
meta: clear_facts
|
||||
|
||||
- name: trigger discovery with auto_legacy
|
||||
vars:
|
||||
ansible_python_interpreter: auto_legacy
|
||||
ping:
|
||||
register: legacy
|
||||
|
||||
- name: check for dep warning (only on platforms where auto result is not /usr/bin/python and legacy is)
|
||||
assert:
|
||||
that:
|
||||
- legacy.deprecations | default([]) | length > 0
|
||||
# only check for a dep warning if legacy returned /usr/bin/python and auto didn't
|
||||
when: legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and
|
||||
auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python'
|
||||
|
||||
|
||||
- name: test that auto_silent never warns and got the same answer as auto
|
||||
block:
|
||||
- name: clear facts to force interpreter discovery to run
|
||||
meta: clear_facts
|
||||
|
||||
- name: initial task to trigger discovery
|
||||
vars:
|
||||
ansible_python_interpreter: auto_silent
|
||||
ping:
|
||||
register: auto_silent_out
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- auto_silent_out.warnings is not defined
|
||||
- auto_silent_out.ansible_facts.discovered_interpreter_python == auto_out.ansible_facts.discovered_interpreter_python
|
||||
|
||||
|
||||
- name: test that auto_legacy_silent never warns and got the same answer as auto_legacy
|
||||
block:
|
||||
- name: clear facts to force interpreter discovery to run
|
||||
meta: clear_facts
|
||||
|
||||
- name: trigger discovery with auto_legacy_silent
|
||||
vars:
|
||||
ansible_python_interpreter: auto_legacy_silent
|
||||
ping:
|
||||
register: legacy_silent
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- legacy_silent.warnings is not defined
|
||||
- legacy_silent.ansible_facts.discovered_interpreter_python == legacy.ansible_facts.discovered_interpreter_python
|
||||
|
||||
- name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter
|
||||
block:
|
||||
- test_echo_module:
|
||||
facts:
|
||||
ansible_discovered_interpreter_bogus: from module
|
||||
discovered_interpreter_bogus: from_module
|
||||
ansible_bogus_interpreter: from_module
|
||||
test_fact: from_module
|
||||
register: echoout
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- test_fact == 'from_module'
|
||||
- discovered_interpreter_bogus | default('nope') == 'nope'
|
||||
- ansible_bogus_interpreter | default('nope') == 'nope'
|
||||
# this one will exist in facts, but with its prefix removed
|
||||
- ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope'
|
||||
- ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope'
|
||||
|
||||
- name: fedora assertions
|
||||
assert:
|
||||
that:
|
||||
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
|
||||
when: distro == 'fedora' and distro_version is version('23', '>=')
|
||||
|
||||
- name: rhel assertions
|
||||
assert:
|
||||
that:
|
||||
# rhel 6/7
|
||||
- (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('8','<')) or distro_version is version('8','>=')
|
||||
# rhel 8+
|
||||
- (auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' and distro_version is version('8','>=')) or distro_version is version('8','<')
|
||||
when: distro == 'redhat'
|
||||
|
||||
- name: ubuntu assertions
|
||||
assert:
|
||||
that:
|
||||
# ubuntu < 16
|
||||
- (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('16.04','<')) or distro_version is version('16.04','>=')
|
||||
# ubuntu >= 16
|
||||
- (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' and distro_version is version('16.04','>=')) or distro_version is version('16.04','<')
|
||||
when: distro == 'ubuntu'
|
||||
|
||||
- name: mac assertions
|
||||
assert:
|
||||
that:
|
||||
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
|
||||
when: os_family == 'darwin'
|
||||
|
||||
always:
|
||||
- meta: clear_facts
|
||||
Loading…
Reference in New Issue