diff --git a/.travis.yml b/.travis.yml index 4aed8d0a..1f8c5c6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,21 @@ install: - pip install -r dev_requirements.txt script: -- ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh +- | + if [ -f "${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh" ]; then + ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh; + else + ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.py; + fi + services: - docker + +# To avoid matrix explosion, just test against oldest->newest and +# newest->oldest in various configuartions. + matrix: include: # Mitogen tests. @@ -34,85 +44,32 @@ matrix: # 2.6 -> 2.7 - python: "2.6" env: MODE=mitogen DISTRO=centos7 - # 2.6 -> 2.6 - - python: "2.6" - env: MODE=mitogen DISTRO=centos6 - # 3.6 -> 2.7 + # 3.6 -> 2.6 - python: "3.6" - env: MODE=mitogen DISTRO=debian + env: MODE=mitogen DISTRO=centos6 # Debops tests. - # 2.4.3.0; 2.7 -> 2.7 - - python: "2.7" - env: MODE=debops_common VER=2.4.3.0 - # 2.5.5; 2.7 -> 2.7 + # 2.4.6.0; 2.7 -> 2.7 - python: "2.7" - env: MODE=debops_common VER=2.6.1 - # 2.5.5; 3.6 -> 2.7 + env: MODE=debops_common VER=2.4.6.0 + # 2.5.7; 3.6 -> 2.7 - python: "3.6" - env: MODE=debops_common VER=2.6.1 + env: MODE=debops_common VER=2.6.2 # ansible_mitogen tests. - # 2.4.3.0; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.4.3.0 DISTRO=debian - # 2.5.5; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=debian - # 2.6.0; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=debian - # 2.6.1; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=debian - # Centos 7 Python2 - # Latest + # 2.6 -> {debian, centos6, centos7} - python: "2.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - # Backward Compatiability - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 - - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=centos7 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - - # Centos 7 Python3 - - python: "3.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 - - python: "3.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos7 - - python: "3.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - - - # Centos 6 Python2 - # Latest - - python: "2.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 - # Backward Compatiability + env: MODE=ansible VER=2.4.6.0 - python: "2.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 - - python: "2.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos6 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 + env: MODE=ansible VER=2.6.2 - # Centos 6 Python3 + # 3.6 -> {debian, centos6, centos7} - python: "3.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 + env: MODE=ansible VER=2.4.6.0 - python: "3.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos6 - - python: "3.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 + env: MODE=ansible VER=2.6.2 - # Sanity check our tests against vanilla Ansible, they should pass. - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear + # Sanity check against vanilla Ansible. One job suffices. - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=debian STRATEGY=linear - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=debian STRATEGY=linear - - + env: MODE=ansible VER=2.6.2 DISTRO=debian STRATEGY=linear diff --git a/.travis/ansible_tests.py b/.travis/ansible_tests.py new file mode 100755 index 00000000..0c47ab27 --- /dev/null +++ b/.travis/ansible_tests.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen + +import os +import sys + +import ci_lib +from ci_lib import run + + +BASE_PORT = 2201 +TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') +HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') + + +with ci_lib.Fold('docker_setup'): + for i, distro in enumerate(ci_lib.DISTROS): + try: + run("docker rm -f target-%s", distro) + except: pass + + run(""" + docker run + --rm + --detach + --publish 0.0.0.0:%s:22/tcp + --name=target-%s + mitogen/%s-test + """, BASE_PORT + i, distro, distro,) + + +with ci_lib.Fold('job_setup'): + os.chdir(TESTS_DIR) + os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) + + # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. + run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + + run("mkdir %s", HOSTS_DIR) + run("ln -s %s/common-hosts %s", TESTS_DIR, HOSTS_DIR) + + with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: + fp.write('[test-targets]\n') + for i, distro in enumerate(ci_lib.DISTROS): + fp.write("target-%s " + "ansible_host=%s " + "ansible_port=%s " + "ansible_user=mitogen__has_sudo_nopw " + "ansible_password=has_sudo_nopw_password" + "\n" % ( + distro, + ci_lib.DOCKER_HOSTNAME, + BASE_PORT + i, + )) + + # Build the binaries. + run("make -C %s", TESTS_DIR) + if not ci_lib.exists_in_path('sshpass'): + run("sudo apt-get update") + run("sudo apt-get install -y sshpass") + + +with ci_lib.Fold('ansible'): + run('/usr/bin/time ./run_ansible_playbook.sh all.yml -i "%s" %s', + HOSTS_DIR, ' '.join(sys.argv[1:])) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh deleted file mode 100755 index e6441343..00000000 --- a/.travis/ansible_tests.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -ex -# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen - -TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" -TMPDIR="/tmp/ansible-tests-$$" -ANSIBLE_VERSION="${VER:-2.6.1}" -export ANSIBLE_STRATEGY="${STRATEGY:-mitogen_linear}" -DISTRO="${DISTRO:-debian}" - -export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}" - -# SSH passes these through to the container when run interactively, causing -# stdout to get messed up with libc warnings. -unset LANG LC_ALL - -function on_exit() -{ - rm -rf "$TMPDIR" - docker kill target || true -} - -trap on_exit EXIT -mkdir "$TMPDIR" - - -echo travis_fold:start:docker_setup -DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)" - -docker run \ - --rm \ - --detach \ - --publish 0.0.0.0:2201:22/tcp \ - --name=target \ - mitogen/${DISTRO}-test -echo travis_fold:end:docker_setup - - -echo travis_fold:start:job_setup -pip install ansible=="${ANSIBLE_VERSION}" -cd ${TRAVIS_BUILD_DIR}/tests/ansible - -chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key -echo '[test-targets]' > ${TMPDIR}/hosts -echo \ - target \ - ansible_host=$DOCKER_HOSTNAME \ - ansible_port=2201 \ - ansible_user=mitogen__has_sudo_nopw \ - ansible_password=has_sudo_nopw_password \ - >> ${TMPDIR}/hosts - -# Build the binaries. -make -C ${TRAVIS_BUILD_DIR}/tests/ansible - -[ ! "$(type -p sshpass)" ] && sudo apt install -y sshpass - -echo travis_fold:end:job_setup - - -echo travis_fold:start:ansible -/usr/bin/time ./run_ansible_playbook.sh \ - all.yml \ - -i "${TMPDIR}/hosts" "$@" -echo travis_fold:end:ansible diff --git a/.travis/ci_lib.py b/.travis/ci_lib.py new file mode 100644 index 00000000..828cae39 --- /dev/null +++ b/.travis/ci_lib.py @@ -0,0 +1,100 @@ + +from __future__ import absolute_import +from __future__ import print_function + +import atexit +import os +import subprocess +import sys +import shlex +import shutil +import tempfile + + +# +# check_output() monkeypatch cutpasted from testlib.py +# + +def subprocess__check_output(*popenargs, **kwargs): + # Missing from 2.6. + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, _ = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output + +if not hasattr(subprocess, 'check_output'): + subprocess.check_output = subprocess__check_output + +# ----------------- + +def _argv(s, *args): + if args: + s %= args + return shlex.split(s) + + +def run(s, *args, **kwargs): + argv = _argv(s, *args) + print('Running: %s' % (argv,)) + return subprocess.check_call(argv, **kwargs) + + +def get_output(s, *args, **kwargs): + argv = _argv(s, *args) + print('Running: %s' % (argv,)) + return subprocess.check_output(argv, **kwargs) + + +def exists_in_path(progname): + return any(os.path.exists(os.path.join(dirname, progname)) + for dirname in os.environ['PATH'].split(os.pathsep)) + + +class TempDir(object): + def __init__(self): + self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib') + atexit.register(self.destroy) + + def destroy(self, rmtree=shutil.rmtree): + rmtree(self.path) + + +class Fold(object): + def __init__(self, name): + self.name = name + + def __enter__(self): + print('travis_fold:start:%s' % (self.name)) + + def __exit__(self, _1, _2, _3): + print('') + print('travis_fold:end:%s' % (self.name)) + + +os.environ.setdefault('ANSIBLE_STRATEGY', + os.environ.get('STRATEGY', 'mitogen_linear')) +ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') +GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() +TMP = TempDir().path + +os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' +os.environ['PYTHONPATH'] = '%s:%s' % ( + os.environ.get('PYTHONPATH', ''), + GIT_ROOT +) + +DOCKER_HOSTNAME = subprocess.check_output([ + sys.executable, + os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), +]).decode().strip() + +# SSH passes these through to the container when run interactively, causing +# stdout to get messed up with libc warnings. +os.environ.pop('LANG', None) +os.environ.pop('LC_ALL', None) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c45a8aa7..9b6a36a7 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -53,7 +53,28 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) +def optional_secret(value): + """ + Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`, + otherwise return :data:`None`. + """ + if value is not None: + return mitogen.core.Secret(value) + + +def parse_python_path(s): + """ + Given the string set for ansible_python_interpeter, parse it using shell + syntax and return an appropriate argument vector. + """ + if s: + return ansible.utils.shlex.shlex_split(s) + + def _connect_local(spec): + """ + Return ContextService arguments for a local connection. + """ return { 'method': 'local', 'kwargs': { @@ -62,12 +83,10 @@ def _connect_local(spec): } -def wrap_or_none(klass, value): - if value is not None: - return klass(value) - - def _connect_ssh(spec): + """ + Return ContextService arguments for an SSH connection. + """ if C.HOST_KEY_CHECKING: check_host_keys = 'enforce' else: @@ -79,7 +98,7 @@ def _connect_ssh(spec): 'check_host_keys': check_host_keys, 'hostname': spec['remote_addr'], 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'port': spec['port'], 'python_path': spec['python_path'], 'identity_file': spec['private_key_file'], @@ -92,6 +111,9 @@ def _connect_ssh(spec): def _connect_docker(spec): + """ + Return ContextService arguments for a Docker connection. + """ return { 'method': 'docker', 'kwargs': { @@ -104,6 +126,9 @@ def _connect_docker(spec): def _connect_jail(spec): + """ + Return ContextService arguments for a FreeBSD jail connection. + """ return { 'method': 'jail', 'kwargs': { @@ -116,6 +141,9 @@ def _connect_jail(spec): def _connect_lxc(spec): + """ + Return ContextService arguments for an LXC Classic container connection. + """ return { 'method': 'lxc', 'kwargs': { @@ -126,11 +154,31 @@ def _connect_lxc(spec): } +def _connect_lxd(spec): + """ + Return ContextService arguments for an LXD container connection. + """ + return { + 'method': 'lxd', + 'kwargs': { + 'container': spec['remote_addr'], + 'python_path': spec['python_path'], + 'connect_timeout': spec['ansible_ssh_timeout'] or spec['timeout'], + } + } + + def _connect_machinectl(spec): + """ + Return ContextService arguments for a machinectl connection. + """ return _connect_setns(dict(spec, mitogen_kind='machinectl')) def _connect_setns(spec): + """ + Return ContextService arguments for a mitogen_setns connection. + """ return { 'method': 'setns', 'kwargs': { @@ -146,12 +194,15 @@ def _connect_setns(spec): def _connect_su(spec): + """ + Return ContextService arguments for su as a become method. + """ return { 'method': 'su', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'su_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -160,12 +211,15 @@ def _connect_su(spec): def _connect_sudo(spec): + """ + Return ContextService arguments for sudo as a become method. + """ return { 'method': 'sudo', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'sudo_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -175,12 +229,15 @@ def _connect_sudo(spec): def _connect_doas(spec): + """ + Return ContextService arguments for doas as a become method. + """ return { 'method': 'doas', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'doas_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -189,12 +246,14 @@ def _connect_doas(spec): def _connect_mitogen_su(spec): - # su as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for su as a first class connection. + """ return { 'method': 'su', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'su_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -203,12 +262,14 @@ def _connect_mitogen_su(spec): def _connect_mitogen_sudo(spec): - # sudo as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for sudo as a first class connection. + """ return { 'method': 'sudo', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'sudo_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -218,12 +279,14 @@ def _connect_mitogen_sudo(spec): def _connect_mitogen_doas(spec): - # doas as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for doas as a first class connection. + """ return { 'method': 'doas', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'doas_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -231,12 +294,15 @@ def _connect_mitogen_doas(spec): } +#: Mapping of connection method names to functions invoked as `func(spec)` +#: generating ContextService keyword arguments matching a connection +#: specification. CONNECTION_METHOD = { 'docker': _connect_docker, 'jail': _connect_jail, 'local': _connect_local, 'lxc': _connect_lxc, - 'lxd': _connect_lxc, + 'lxd': _connect_lxd, 'machinectl': _connect_machinectl, 'setns': _connect_setns, 'ssh': _connect_ssh, @@ -249,17 +315,6 @@ CONNECTION_METHOD = { } -def parse_python_path(s): - """ - Given the string set for ansible_python_interpeter, parse it using shell - syntax and return an appropriate argument vector. - """ - if not s: - return None - - return ansible.utils.shlex.shlex_split(s) - - def config_from_play_context(transport, inventory_name, connection): """ Return a dict representing all important connection configuration, allowing @@ -318,7 +373,7 @@ def config_from_hostvars(transport, inventory_name, connection, config = config_from_play_context(transport, inventory_name, connection) hostvars = dict(hostvars) return dict(config, **{ - 'remote_addr': hostvars.get('ansible_hostname', inventory_name), + 'remote_addr': hostvars.get('ansible_host', inventory_name), 'become': bool(become_user), 'become_user': become_user, 'become_pass': None, @@ -393,12 +448,17 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Set to 'hostvars' by on_action_run() host_vars = None - #: Set to '_loader.get_basedir()' by on_action_run(). + #: Set to '_loader.get_basedir()' by on_action_run(). Used by mitogen_local + #: to change the working directory to that of the current playbook, + #: matching vanilla Ansible behaviour. loader_basedir = None #: Set after connection to the target context's home directory. home_dir = None + #: Set after connection to the target context's home directory. + _temp_dir = None + def __init__(self, play_context, new_stdin, **kwargs): assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( 'Mitogen connection types may only be instantiated ' @@ -415,11 +475,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): # https://github.com/dw/mitogen/issues/140 self.close() - def on_action_run(self, task_vars, loader_basedir): + def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir): """ Invoked by ActionModuleMixin to indicate a new task is about to start executing. We use the opportunity to grab relevant bits from the task-specific data. + + :param dict task_vars: + Task variable dictionary. + :param str delegate_to_hostname: + :data:`None`, or the template-expanded inventory hostname this task + is being delegated to. A similar variable exists on PlayContext + when ``delegate_to:`` is active, however it is unexpanded. + :param str loader_basedir: + Loader base directory; see :attr:`loader_basedir`. """ self.ansible_ssh_timeout = task_vars.get('ansible_ssh_timeout', C.DEFAULT_TIMEOUT) @@ -433,6 +502,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.mitogen_ssh_debug_level = task_vars.get('mitogen_ssh_debug_level') self.inventory_hostname = task_vars['inventory_hostname'] self.host_vars = task_vars['hostvars'] + self.delegate_to_hostname = delegate_to_hostname self.loader_basedir = loader_basedir self.close(new_task=True) @@ -446,6 +516,10 @@ class Connection(ansible.plugins.connection.ConnectionBase): return self.context is not None def _config_from_via(self, via_spec): + """ + Produce a dict connection specifiction given a string `via_spec`, of + the form `[become_user@]inventory_hostname`. + """ become_user, _, inventory_name = via_spec.rpartition('@') via_vars = self.host_vars[inventory_name] if isinstance(via_vars, jinja2.runtime.Undefined): @@ -492,20 +566,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): return stack, seen_names - def _connect(self): + def _connect_broker(self): """ - Establish a connection to the master process's UNIX listener socket, - constructing a mitogen.master.Router to communicate with the master, - and a mitogen.parent.Context to represent it. - - Depending on the original transport we should emulate, trigger one of - the _connect_*() service calls defined above to cause the master - process to establish the real connection on our behalf, or return a - reference to the existing one. + Establish a reference to the Broker, Router and parent context used for + connections. """ - if self.connected: - return - if not self.broker: self.broker = mitogen.master.Broker() self.router, self.parent = mitogen.unix.connect( @@ -513,14 +578,47 @@ class Connection(ansible.plugins.connection.ConnectionBase): broker=self.broker, ) - stack, _ = self._stack_from_config( - config_from_play_context( - transport=self.transport, - inventory_name=self.inventory_hostname, - connection=self - ) + def _config_from_direct_connection(self): + """ + """ + return config_from_play_context( + transport=self.transport, + inventory_name=self.inventory_hostname, + connection=self + ) + + def _config_from_delegate_to(self): + return config_from_hostvars( + transport=self._play_context.connection, + inventory_name=self.delegate_to_hostname, + connection=self, + hostvars=self.host_vars[self._play_context.delegate_to], + become_user=(self._play_context.become_user + if self._play_context.become + else None), ) + def _build_stack(self): + """ + Construct a list of dictionaries representing the connection + configuration between the controller and the target. This is + additionally used by the integration tests "mitogen_get_stack" action + to fetch the would-be connection configuration. + """ + if self.delegate_to_hostname is not None: + target_config = self._config_from_delegate_to() + else: + target_config = self._config_from_direct_connection() + + stack, _ = self._stack_from_config(target_config) + return stack + + def _connect_stack(self, stack): + """ + Pass `stack` to ContextService, requesting a copy of the context object + representing the target. If no connection exists yet, ContextService + will establish it before returning it or throwing an error. + """ dct = self.parent.call_service( service_name='ansible_mitogen.services.ContextService', method_name='get', @@ -540,6 +638,29 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.fork_context = dct['init_child_result']['fork_context'] self.home_dir = dct['init_child_result']['home_dir'] + self._temp_dir = dct['init_child_result']['temp_dir'] + + def get_temp_dir(self): + self._connect() + return self._temp_dir + + def _connect(self): + """ + Establish a connection to the master process's UNIX listener socket, + constructing a mitogen.master.Router to communicate with the master, + and a mitogen.parent.Context to represent it. + + Depending on the original transport we should emulate, trigger one of + the _connect_*() service calls defined above to cause the master + process to establish the real connection on our behalf, or return a + reference to the existing one. + """ + if self.connected: + return + + self._connect_broker() + stack = self._build_stack() + self._connect_stack(stack) def close(self, new_task=False): """ diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 441e8113..08c59278 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -37,10 +37,12 @@ try: from ansible.plugins.loader import connection_loader from ansible.plugins.loader import module_loader from ansible.plugins.loader import module_utils_loader + from ansible.plugins.loader import shell_loader from ansible.plugins.loader import strategy_loader except ImportError: # Ansible <2.4 from ansible.plugins import action_loader from ansible.plugins import connection_loader from ansible.plugins import module_loader from ansible.plugins import module_utils_loader + from ansible.plugins import shell_loader from ansible.plugins import strategy_loader diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2a9fdac8..7d8f5518 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -110,6 +110,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ self._connection.on_action_run( task_vars=task_vars, + delegate_to_hostname=self._task.delegate_to, loader_basedir=self._loader.get_basedir(), ) return super(ActionModuleMixin, self).run(tmp, task_vars) @@ -179,48 +180,26 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ assert False, "_is_pipelining_enabled() should never be called." - def _get_remote_tmp(self): - """ - Mitogen-only: return the 'remote_tmp' setting. - """ - try: - s = self._connection._shell.get_option('remote_tmp') - except AttributeError: - s = ansible.constants.DEFAULT_REMOTE_TMP # <=2.4.x - - return self._remote_expand_user(s, sudoable=False) - def _make_tmp_path(self, remote_user=None): """ - Replace the base implementation's use of shell to implement mkdtemp() - with an actual call to mkdtemp(). Like vanilla, the directory is always - created in the login account context. + Return the temporary directory created by the persistent interpreter at + startup. """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) - # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. - # The copy action plugin violates layering and grabs this attribute - # directly. - self._connection._shell.tmpdir = self._connection.call( - ansible_mitogen.target.make_temp_directory, - base_dir=self._get_remote_tmp(), - use_login_context=True, - ) + self._connection._shell.tmpdir = self._connection.get_temp_dir() LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True return self._connection._shell.tmpdir def _remove_tmp_path(self, tmp_path): """ - Replace the base implementation's invocation of rm -rf with a call to - shutil.rmtree(). + Stub out the base implementation's invocation of rm -rf, replacing it + with nothing, as the persistent interpreter automatically cleans up + after itself without introducing roundtrips. """ LOG.debug('_remove_tmp_path(%r)', tmp_path) - if tmp_path is None: - tmp_path = self._connection._shell.tmpdir - if self._should_remove_tmp_path(tmp_path): - self.call(shutil.rmtree, tmp_path) - self._connection._shell.tmpdir = None + self._connection._shell.tmpdir = None def _transfer_data(self, remote_path, data): """ @@ -331,7 +310,15 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): env = {} self._compute_environment_string(env) + # Always set _ansible_tmpdir regardless of whether _make_remote_tmp() + # has ever been called. This short-circuits all the .tmpdir logic in + # module_common and ensures no second temporary directory or atexit + # handler is installed. self._connection._connect() + + if ansible.__version__ > '2.5': + module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() + return ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( action=self, diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index c297ad8f..8ebf4f67 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -55,6 +55,7 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' +NO_MODULE_MSG = 'The module %s was not found in configured module paths.' class Invocation(object): @@ -393,6 +394,9 @@ _planners = [ def get_module_data(name): path = ansible_mitogen.loaders.module_loader.find_plugin(name, '') + if path is None: + raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,)) + with open(path, 'rb') as fp: source = fp.read() return mitogen.core.to_text(path), source diff --git a/ansible_mitogen/plugins/connection/mitogen_doas.py b/ansible_mitogen/plugins/connection/mitogen_doas.py index 7d60b482..873b0d9d 100644 --- a/ansible_mitogen/plugins/connection/mitogen_doas.py +++ b/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index a98273e0..8af42711 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index 1c57bb38..fb7bce54 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index 35504d4d..fcd9c030 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index 2195aa3c..ce394102 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 5d1391b9..77efe6c1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index e71496a3..9b332a3f 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 5f131655..23f62135 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index c0c577c3..d2af109c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_su.py b/ansible_mitogen/plugins/connection/mitogen_su.py index fd09d0f0..104a7190 100644 --- a/ansible_mitogen/plugins/connection/mitogen_su.py +++ b/ansible_mitogen/plugins/connection/mitogen_su.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py index a6cb8bd2..367dd61b 100644 --- a/ansible_mitogen/plugins/connection/mitogen_sudo.py +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 4f595161..f8608745 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index 8dfaa16e..d3b1cdc6 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index d995b67b..51b03096 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 4724ca93..5ce1b8be 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -27,6 +27,7 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +import atexit import errno import logging import os @@ -53,6 +54,22 @@ from mitogen.core import b LOG = logging.getLogger(__name__) +def clean_shutdown(sock): + """ + Shut the write end of `sock`, causing `recv` in the worker process to wake + up with a 0-byte read and initiate mux process exit, then wait for a 0-byte + read from the read end, which will occur after the the child closes the + descriptor on exit. + + This is done using :mod:`atexit` since Ansible lacks any more sensible hook + to run code during exit, and unless some synchronization exists with + MuxProcess, debug logs may appear on the user's terminal *after* the prompt + has been printed. + """ + sock.shutdown(socket.SHUT_WR) + sock.recv(1) + + class MuxProcess(object): """ Implement a subprocess forked from the Ansible top-level, as a safe place @@ -112,6 +129,7 @@ class MuxProcess(object): cls.unix_listener_path = mitogen.unix.make_socket_path() cls.worker_sock, cls.child_sock = socket.socketpair() + atexit.register(lambda: clean_shutdown(cls.worker_sock)) mitogen.core.set_cloexec(cls.worker_sock.fileno()) mitogen.core.set_cloexec(cls.child_sock.fileno()) @@ -143,7 +161,6 @@ class MuxProcess(object): # Let the parent know our listening socket is ready. mitogen.core.io_op(self.child_sock.send, b('1')) - self.child_sock.send(b('1')) # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.child_sock.recv, 1) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 86f7b329..97a3cb36 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -44,6 +44,7 @@ import imp import json import logging import os +import shlex import sys import tempfile import types @@ -65,6 +66,9 @@ except ImportError: # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +ansible.module_utils.basic.get_module_path = lambda: ( + ansible_mitogen.target.temp_dir +) # For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache # resolv.conf at startup and never implicitly reload it. Cope with that via an @@ -82,6 +86,110 @@ iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) +class EnvironmentFileWatcher(object): + """ + Usually Ansible edits to /etc/environment and ~/.pam_environment are + reflected in subsequent tasks if become:true or SSH multiplexing is + disabled, due to sudo and/or SSH reinvoking pam_env. Rather than emulate + existing semantics, do our best to ensure edits are always reflected. + + This can't perfectly replicate the existing behaviour, but it can safely + update and remove keys that appear to originate in `path`, and that do not + conflict with any existing environment key inherited from elsewhere. + + A more robust future approach may simply be to arrange for the persistent + interpreter to restart when a change is detected. + """ + def __init__(self, path): + self.path = os.path.expanduser(path) + #: Inode data at time of last check. + self._st = self._stat() + #: List of inherited keys appearing to originated from this file. + self._keys = [key for key, value in self._load() + if value == os.environ.get(key)] + LOG.debug('%r installed; existing keys: %r', self, self._keys) + + def __repr__(self): + return 'EnvironmentFileWatcher(%r)' % (self.path,) + + def _stat(self): + try: + return os.stat(self.path) + except OSError: + return None + + def _load(self): + try: + with open(self.path, 'r') as fp: + return list(self._parse(fp)) + except IOError: + return [] + + def _parse(self, fp): + """ + linux-pam-1.3.1/modules/pam_env/pam_env.c#L207 + """ + for line in fp: + # ' #export foo=some var ' -> ['#export', 'foo=some var '] + bits = shlex.split(line, comments=True) + if (not bits) or bits[0].startswith('#'): + continue + + if bits[0] == 'export': + bits.pop(0) + + key, sep, value = (' '.join(bits)).partition('=') + if key and sep: + yield key, value + + def _on_file_changed(self): + LOG.debug('%r: file changed, reloading', self) + for key, value in self._load(): + if key in os.environ: + LOG.debug('%r: existing key %r=%r exists, not setting %r', + self, key, os.environ[key], value) + else: + LOG.debug('%r: setting key %r to %r', self, key, value) + self._keys.append(key) + os.environ[key] = value + + def _remove_existing(self): + """ + When a change is detected, remove keys that existed in the old file. + """ + for key in self._keys: + if key in os.environ: + LOG.debug('%r: removing old key %r', self, key) + del os.environ[key] + self._keys = [] + + def check(self): + """ + Compare the :func:`os.stat` for the pam_env style environmnt file + `path` with the previous result `old_st`, which may be :data:`None` if + the previous stat attempt failed. Reload its contents if the file has + changed or appeared since last attempt. + + :returns: + New :func:`os.stat` result. The new call to :func:`reload_env` should + pass it as the value of `old_st`. + """ + st = self._stat() + if self._st == st: + return + + self._st = st + self._remove_existing() + + if st is None: + LOG.debug('%r: file has disappeared', self) + else: + self._on_file_changed() + +_pam_env_watcher = EnvironmentFileWatcher('~/.pam_environment') +_etc_env_watcher = EnvironmentFileWatcher('/etc/environment') + + def utf8(s): """ Coerce an object to bytes if it is Unicode. @@ -154,12 +262,25 @@ class Runner(object): from the parent, as :meth:`run` may detach prior to beginning execution. The base implementation simply prepares the environment. """ + self._setup_cwd() + self._setup_environ() + + def _setup_cwd(self): + """ + For situations like sudo to a non-privileged account, CWD could be + $HOME of the old account, which could have mode go=, which means it is + impossible to restore the old directory, so don't even try. + """ if self.cwd: - # For situations like sudo to another non-privileged account, the - # CWD could be $HOME of the old account, which could have mode go=, - # which means it is impossible to restore the old directory, so - # don't even bother. os.chdir(self.cwd) + + def _setup_environ(self): + """ + Apply changes from /etc/environment files before creating a + TemporaryEnvironment to snapshot environment state prior to module run. + """ + _pam_env_watcher.check() + _etc_env_watcher.check() env = dict(self.extra_env or {}) if self.env: env.update(self.env) @@ -548,6 +669,14 @@ class NewStyleRunner(ScriptRunner): for fullname in self.module_map['builtin']: mitogen.core.import_module(fullname) + def _setup_excepthook(self): + """ + Starting with Ansible 2.6, some modules (file.py) install a + sys.excepthook and never clean it up. So we must preserve the original + excepthook and restore it after the run completes. + """ + self.original_excepthook = sys.excepthook + def setup(self): super(NewStyleRunner, self).setup() @@ -561,12 +690,17 @@ class NewStyleRunner(ScriptRunner): module_utils=self.module_map['custom'], ) self._setup_imports() + self._setup_excepthook() if libc__res_init: libc__res_init() + def _revert_excepthook(self): + sys.excepthook = self.original_excepthook + def revert(self): self._argv.revert() self._stdio.revert() + self._revert_excepthook() super(NewStyleRunner, self).revert() def _get_program_filename(self): @@ -600,6 +734,20 @@ class NewStyleRunner(ScriptRunner): else: main_module_name = b'__main__' + def _handle_magic_exception(self, mod, exc): + """ + Beginning with Ansible >2.6, some modules (file.py) install a + sys.excepthook which is a closure over AnsibleModule, redirecting the + magical exception to AnsibleModule.fail_json(). + + For extra special needs bonus points, the class is not defined in + module_utils, but is defined in the module itself, meaning there is no + type for isinstance() that outlasts the invocation. + """ + klass = getattr(mod, 'AnsibleModuleError', None) + if klass and isinstance(exc, klass): + mod.module.fail_json(**exc.results) + def _run(self): code = self._get_code() @@ -616,10 +764,14 @@ class NewStyleRunner(ScriptRunner): exc = None try: - if mitogen.core.PY3: - exec(code, vars(mod)) - else: - exec('exec code in vars(mod)') + try: + if mitogen.core.PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + except Exception as e: + self._handle_magic_exception(mod, e) + raise except SystemExit as e: exc = e diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index e95fc226..952e991a 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -46,14 +46,23 @@ import os.path import sys import threading +import ansible.constants + import mitogen import mitogen.service +import mitogen.utils +import ansible_mitogen.loaders import ansible_mitogen.module_finder import ansible_mitogen.target LOG = logging.getLogger(__name__) +# Force load of plugin to ensure ConfigManager has definitions loaded. Done +# during module import to ensure a single-threaded environment; PluginLoader +# is not thread-safe. +ansible_mitogen.loaders.shell_loader.get('sh') + if sys.version_info[0] == 3: def reraise(tp, value, tb): @@ -69,6 +78,17 @@ else: ) +def _get_candidate_temp_dirs(): + options = ansible.constants.config.get_plugin_options('shell', 'sh') + + # Pre 2.5 this came from ansible.constants. + remote_tmp = (options.get('remote_tmp') or + ansible.constants.DEFAULT_REMOTE_TMP) + dirs = list(options.get('system_tmpdirs', ('/var/tmp', '/tmp'))) + dirs.insert(0, remote_tmp) + return mitogen.utils.cast(dirs) + + class Error(Exception): pass @@ -252,6 +272,18 @@ class ContextService(mitogen.service.Service): for fullname in self.ALWAYS_PRELOAD: self.router.responder.forward_module(context, fullname) + _candidate_temp_dirs = None + + def _get_candidate_temp_dirs(self): + """ + Return a list of locations to try to create the single temporary + directory used by the run. This simply caches the (expensive) plugin + load of :func:`_get_candidate_temp_dirs`. + """ + if self._candidate_temp_dirs is None: + self._candidate_temp_dirs = _get_candidate_temp_dirs() + return self._candidate_temp_dirs + def _connect(self, key, spec, via=None): """ Actual connect implementation. Arranges for the Mitogen connection to @@ -298,8 +330,11 @@ class ContextService(mitogen.service.Service): lambda: self._on_stream_disconnect(stream)) self._send_module_forwards(context) - init_child_result = context.call(ansible_mitogen.target.init_child, - log_level=LOG.getEffectiveLevel()) + init_child_result = context.call( + ansible_mitogen.target.init_child, + log_level=LOG.getEffectiveLevel(), + candidate_temp_dirs=self._get_candidate_temp_dirs(), + ) if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'): from mitogen import debug diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 582bf85e..35863cb2 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -69,14 +69,30 @@ import ansible_mitogen.runner LOG = logging.getLogger(__name__) -#: Set by init_child() to the single temporary directory that will exist for -#: the duration of the process. -temp_dir = None +MAKE_TEMP_FAILED_MSG = ( + "Unable to find a useable temporary directory. This likely means no\n" + "system-supplied TMP directory can be written to, or all directories\n" + "were mounted on 'noexec' filesystems.\n" + "\n" + "The following paths were tried:\n" + " %(namelist)s\n" + "\n" + "Please check '-vvv' output for a log of individual path errors." +) + #: Initialized to an econtext.parent.Context pointing at a pristine fork of #: the target Python interpreter before it executes any code or imports. _fork_parent = None +#: Set by init_child() to a list of candidate $variable-expanded and +#: tilde-expanded directory paths that may be usable as a temporary directory. +_candidate_temp_dirs = None + +#: Set by reset_temp_dir() to the single temporary directory that will exist +#: for the duration of the process. +temp_dir = None + def get_small_file(context, path): """ @@ -190,6 +206,53 @@ def _on_broker_shutdown(): prune_tree(temp_dir) +def find_good_temp_dir(): + """ + Given a list of candidate temp directories extracted from ``ansible.cfg`` + and stored in _candidate_temp_dirs, combine it with the Python-builtin list + of candidate directories used by :mod:`tempfile`, then iteratively try each + in turn until one is found that is both writeable and executable. + """ + paths = [os.path.expandvars(os.path.expanduser(p)) + for p in _candidate_temp_dirs] + paths.extend(tempfile._candidate_tempdir_list()) + + for path in paths: + try: + tmp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen_find_good_temp_dir', + dir=path, + ) + except (OSError, IOError) as e: + LOG.debug('temp dir %r unusable: %s', path, e) + continue + + try: + try: + os.chmod(tmp.name, int('0700', 8)) + except OSError as e: + LOG.debug('temp dir %r unusable: %s: chmod failed: %s', + path, e) + continue + + try: + # access(.., X_OK) is sufficient to detect noexec. + if not os.access(tmp.name, os.X_OK): + raise OSError('filesystem appears to be mounted noexec') + except OSError as e: + LOG.debug('temp dir %r unusable: %s: %s', path, e) + continue + + LOG.debug('Selected temp directory: %r (from %r)', path, paths) + return path + finally: + tmp.close() + + raise IOError(MAKE_TEMP_FAILED_MSG % { + 'paths': '\n '.join(paths), + }) + + @mitogen.core.takes_econtext def reset_temp_dir(econtext): """ @@ -204,7 +267,9 @@ def reset_temp_dir(econtext): """ global temp_dir # https://github.com/dw/mitogen/issues/239 - temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') + + basedir = find_good_temp_dir() + temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_', dir=basedir) # This must be reinstalled in forked children too, since the Broker # instance from the parent process does not carry over to the new child. @@ -212,7 +277,7 @@ def reset_temp_dir(econtext): @mitogen.core.takes_econtext -def init_child(econtext, log_level): +def init_child(econtext, log_level, candidate_temp_dirs): """ Called by ContextService immediately after connection; arranges for the (presently) spotless Python interpreter to be forked, where the newly @@ -225,6 +290,9 @@ def init_child(econtext, log_level): :param int log_level: Logging package level active in the master. + :param list[str] candidate_temp_dirs: + List of $variable-expanded and tilde-expanded directory names to add to + candidate list of temporary directories. :returns: Dict like:: @@ -238,6 +306,9 @@ def init_child(econtext, log_level): the controller will use to start forked jobs, and `home_dir` is the home directory for the active user account. """ + global _candidate_temp_dirs + _candidate_temp_dirs = candidate_temp_dirs + global _fork_parent mitogen.parent.upgrade_router(econtext) _fork_parent = econtext.router.fork() @@ -252,6 +323,7 @@ def init_child(econtext, log_level): return { 'fork_context': _fork_parent, 'home_dir': mitogen.core.to_text(os.path.expanduser('~')), + 'temp_dir': temp_dir, } @@ -416,27 +488,6 @@ def run_module_async(kwargs, job_id, timeout_secs, econtext): arunner.run() -def make_temp_directory(base_dir): - """ - Handle creation of `base_dir` if it is absent, in addition to a unique - temporary directory within `base_dir`. This is the temporary directory that - becomes 'remote_tmp', not the one used by Ansiballz. It always uses the - system temporary directory. - - :returns: - Newly created temporary directory. - """ - # issue #301: remote_tmp may contain $vars. - base_dir = os.path.expandvars(base_dir) - - if not os.path.exists(base_dir): - os.makedirs(base_dir, mode=int('0700', 8)) - return tempfile.mkdtemp( - dir=base_dir, - prefix='ansible-mitogen-tmp-', - ) - - def get_user_shell(): """ For commands executed directly via an SSH command-line, SSH looks up the diff --git a/docs/ansible.rst b/docs/ansible.rst index 5bec89e7..bdd05307 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -271,8 +271,8 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally `sftp `_ or -`scp `_ are used to copy files by the +Normally `sftp(1) `_ or +`scp(1) `_ are used to copy files by the `assemble `_, `copy `_, `patch `_, @@ -302,7 +302,7 @@ to rename over any existing file. This ensures the file remains consistent at all times, in the event of a crash, or when overlapping `ansible-playbook` runs deploy differing file contents. -The `sftp `_ and `scp +The `sftp(1) `_ and `scp(1) `_ tools may cause undetected data corruption in the form of truncated files, or files containing intermingled data segments from overlapping runs. As part of normal operation, both tools expose a window @@ -401,6 +401,129 @@ this precisely, to avoid breaking playbooks that expect text to appear in specific variables with a particular linefeed style. +.. _ansible_tempfiles: + +Temporary Files +~~~~~~~~~~~~~~~ + +Ansible creates a variety of temporary files and directories depending on its +operating mode. + +In the best case when pipelining is enabled and no temporary uploads are +required, for each task Ansible will create one directory below a +system-supplied temporary directory returned by :func:`tempfile.mkdtemp`, owned +by the target account a new-style module will execute in. + +In other cases depending on the task type, whether become is active, whether +the target become user is privileged, whether the associated action plugin +needs to upload files, and whether the associated module needs to store files, +Ansible may: + +* Create a directory owned by the SSH user either under ``remote_tmp``, or a + system-default directory, +* Upload action dependencies such as non-new style modules or rendered + templates to that directory via `sftp(1) `_ + or `scp(1) `_. +* Attempt to modify the directory's access control list to grant access to the + target user using `setfacl(1) `_, + requiring that tool to be installed and a supported filesystem to be in use, + or for the ``allow_world_readable_tmpfiles`` setting to be :data:`True`. +* Create a directory owned by the target user either under ``remote_tmp``, or + a system-default directory, if a new-style module needs a temporary directory + and one was not previously created for a supporting file earlier in the + invocation. + +In summary, for each task Ansible may create one or more of: + +* ``~ssh_user//...`` owned by the login user, +* ``$TMPDIR/ansible-tmp-...`` owned by the login user, +* ``$TMPDIR/ansible-tmp-...`` owned by the login user with ACLs permitting + write access by the become user, +* ``~become_user//...`` owned by the become user, +* ``$TMPDIR/ansible__payload_.../`` owned by the become user, +* ``$TMPDIR/ansible-module-tmp-.../`` owned by the become user. + +A directory must exist to maintain compatibility with Ansible, as many modules +introspect :data:`sys.argv` to find a directory where they may write files, +however only one directory exists for the lifetime of each interpreter, its +location is consistent for each target account, and it is always privately +owned by that account. + +The paths below are tried until one is found that is writeable and lives on a +filesystem with ``noexec`` disabled: + +1. ``$variable`` and tilde-expanded ``remote_tmp`` setting from + ``ansible.cfg`` +2. ``$variable`` and tilde-expanded ``system_tmpdirs`` setting from + ``ansible.cfg`` +3. ``TMPDIR`` environment variable +4. ``TEMP`` environment variable +5. ``TMP`` environment variable +6. ``/tmp`` +7. ``/var/tmp`` +8. ``/usr/tmp`` +9. Current working directory + +As the directory is created once at startup, and its content is managed by code +running remotely, no additional network roundtrips are required to manage it +for each task requiring temporary storage. + + +.. _ansible_process_env: + +Process Environment Emulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Ansible discards processes after each module invocation, follow-up tasks +often (but not always) receive a new environment that will usually include +changes made by previous tasks. As such modifications are common, for +compatibility the extension emulates the existing behaviour as closely as +possible. + +Some scenarios exist where emulation is impossible, for example, applying +``nsswitch.conf`` changes when ``nscd`` is not in use. If future scenarios +appear that cannot be solved through emulation, the extension will be updated +to automatically restart affected interpreters instead. + + +DNS Resolution +^^^^^^^^^^^^^^ + +Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to +be reloaded via `res_init(3) `_. This +isn't necessary on some Linux distributions carrying glibc patches to +automatically check ``/etc/resolv.conf`` periodically, however it is necessary +on at least Debian and BSD derivatives. + + +``/etc/environment`` +^^^^^^^^^^^^^^^^^^^^ + +When ``become: true`` is active or SSH multiplexing is disabled, modifications +by previous tasks to ``/etc/environment`` and ``$HOME/.pam_environment`` are +normally reflected, since the content of those files is reapplied by `PAM +`_ via `pam_env` +on each authentication of ``sudo`` or ``sshd``. + +Both files are monitored for changes, and changes are applied where it appears +safe to do so: + +* New keys are added if they did not otherwise exist in the inherited + environment, or previously had the same value as found in the file before it + changed. + +* Given a key (such as ``http_proxy``) added to the file where no such key + exists in the environment, the key will be added. + +* Given a key (such as ``PATH``) where an existing environment key exists with + a different value, the update or deletion will be ignored, as it is likely + the key was overridden elsewhere after `pam_env` ran, such as by + ``/etc/profile``. + +* Given a key removed from the file that had the same value as the existing + environment key, the key will be removed. + + How Modules Execute ~~~~~~~~~~~~~~~~~~~ @@ -569,10 +692,10 @@ additional differences exist that may break existing playbooks. LXC ~~~ -Like `lxc `_ -and `lxd `_ -except connection delegation is supported, and ``lxc-attach`` is always used -rather than the LXC Python bindings, as is usual with ``lxc``. +Connect to classic LXC containers, like `lxc +`_ except +connection delegation is supported, and ``lxc-attach`` is always used rather +than the LXC Python bindings, as is usual with ``lxc``. The ``lxc-attach`` command must be available on the host machine. @@ -580,6 +703,20 @@ The ``lxc-attach`` command must be available on the host machine. * ``ansible_host``: Name of LXC container (default: inventory hostname). +.. _method-lxd: + +LXD +~~~ + +Connect to modern LXD containers, like `lxd +`_ except +connection delegation is supported. The ``lxc`` command must be available on +the host machine. + +* ``ansible_python_interpreter`` +* ``ansible_host``: Name of LXC container (default: inventory hostname). + + .. _machinectl: Machinectl @@ -602,21 +739,23 @@ Setns ~~~~~ The ``setns`` method connects to Linux containers via `setns(2) -`_. Unlike :ref:`method-docker` and -:ref:`method-lxc` the namespace transition is handled internally, ensuring -optimal throughput to the child. This is necessary for :ref:`machinectl` where -only PTY channels are supported. +`_. Unlike :ref:`method-docker`, +:ref:`method-lxc`, and :ref:`method-lxd` the namespace transition is handled +internally, ensuring optimal throughput to the child. This is necessary for +:ref:`machinectl` where only PTY channels are supported. A utility program must be installed to discover the PID of the container's root process. -* ``mitogen_kind``: one of ``docker``, ``lxc`` or ``machinectl``. +* ``mitogen_kind``: one of ``docker``, ``lxc``, ``lxd`` or ``machinectl``. * ``ansible_host``: Name of container as it is known to the corresponding tool (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. * ``mitogen_docker_path``: path to Docker if not available on the system path. -* ``mitogen_lxc_info_path``: path to ``lxc-info`` command if not available as - ``/usr/bin/lxc-info``. +* ``mitogen_lxc_path``: path to LXD's ``lxc`` command if not available as + ``lxc-info``. +* ``mitogen_lxc_info_path``: path to LXC classic's ``lxc-info`` command if not + available as ``lxc-info``. * ``mitogen_machinectl_path``: path to ``machinectl`` command if not available as ``/bin/machinectl``. diff --git a/docs/api.rst b/docs/api.rst index 92abe1ec..9caf3e13 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -87,10 +87,10 @@ Message Class .. class:: Message - Messages are the fundamental unit of communication, comprising the fields - from in the :ref:`stream-protocol` header, an optional reference to the - receiving :class:`mitogen.core.Router` for ingress messages, and helper - methods for deserialization and generating replies. + Messages are the fundamental unit of communication, comprising fields from + the :ref:`stream-protocol` header, an optional reference to the receiving + :class:`mitogen.core.Router` for ingress messages, and helper methods for + deserialization and generating replies. .. attribute:: router @@ -238,16 +238,16 @@ Router Class .. method:: add_handler (fn, handle=None, persist=True, respondent=None, policy=None) Invoke `fn(msg)` for each Message sent to `handle` from this context. - Unregister after one invocation if `persist` is ``False``. If `handle` - is ``None``, a new handle is allocated and returned. + Unregister after one invocation if `persist` is :data:`False`. If + `handle` is :data:`None`, a new handle is allocated and returned. :param int handle: - If not ``None``, an explicit handle to register, usually one of the - ``mitogen.core.*`` constants. If unspecified, a new unused handle - will be allocated. + If not :data:`None`, an explicit handle to register, usually one of + the ``mitogen.core.*`` constants. If unspecified, a new unused + handle will be allocated. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Context respondent: @@ -281,7 +281,8 @@ Router Class sender indicating refusal occurred. :return: - `handle`, or if `handle` was ``None``, the newly allocated handle. + `handle`, or if `handle` was :data:`None`, the newly allocated + handle. .. method:: del_handler (handle) @@ -300,10 +301,10 @@ Router Class called from the I/O multiplexer thread. :param mitogen.core.Stream stream: - If not ``None``, a reference to the stream the message arrived on. - Used for performing source route verification, to ensure sensitive - messages such as ``CALL_FUNCTION`` arrive only from trusted - contexts. + If not :data:`None`, a reference to the stream the message arrived + on. Used for performing source route verification, to ensure + sensitive messages such as ``CALL_FUNCTION`` arrive only from + trusted contexts. .. method:: route(msg) @@ -515,8 +516,8 @@ Router Class otherwise. :param mitogen.core.Context via: - If not ``None``, arrange for construction to occur via RPCs made to - the context `via`, and for :py:data:`ADD_ROUTE + If not :data:`None`, arrange for construction to occur via RPCs + made to the context `via`, and for :py:data:`ADD_ROUTE ` messages to be generated as appropriate. .. code-block:: python @@ -567,7 +568,7 @@ Router Class :data:`None`, which Docker interprets as ``root``. :param str image: Image tag to use to construct a temporary container. Defaults to - ``None``. + :data:`None`. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. @@ -590,18 +591,31 @@ Router Class .. method:: lxc (container, lxc_attach_path=None, \**kwargs) - Construct a context on the local machine within an LXC container using - the ``lxc-attach`` program. + Construct a context on the local machine within an LXC classic + container using the ``lxc-attach`` program. Accepts all parameters accepted by :py:meth:`local`, in addition to: :param str container: - Existing container to connect to. Defaults to ``None``. + Existing container to connect to. Defaults to :data:`None`. :param str lxc_attach_path: Filename or complete path to the ``lxc-attach`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``lxc-attach``. + .. method:: lxc (container, lxc_attach_path=None, \**kwargs) + + Construct a context on the local machine within a LXD container using + the ``lxc`` program. + + Accepts all parameters accepted by :py:meth:`local`, in addition to: + + :param str container: + Existing container to connect to. Defaults to :data:`None`. + :param str lxc_path: + Filename or complete path to the ``lxc`` binary. ``PATH`` will be + searched if given as a filename. Defaults to ``lxc``. + .. method:: setns (container, kind, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs) Construct a context in the style of :meth:`local`, but change the @@ -609,7 +623,8 @@ Router Class executing Python. The namespaces to use, and the active root file system are taken from - the root PID of a running Docker, LXC, or systemd-nspawn container. + the root PID of a running Docker, LXC, LXD, or systemd-nspawn + container. A program is required only to find the root PID, after which management of the child Python interpreter is handled directly. @@ -617,14 +632,16 @@ Router Class :param str container: Container to connect to. :param str kind: - One of ``docker``, ``lxc`` or ``machinectl``. + One of ``docker``, ``lxc``, ``lxd`` or ``machinectl``. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. + :param str lxc_path: + Filename or complete path to the LXD ``lxc`` binary. ``PATH`` will + be searched if given as a filename. Defaults to ``lxc``. :param str lxc_info_path: - Filename or complete path to the ``lxc-info`` binary. ``PATH`` - will be searched if given as a filename. Defaults to - ``lxc-info``. + Filename or complete path to the LXC ``lxc-info`` binary. ``PATH`` + will be searched if given as a filename. Defaults to ``lxc-info``. :param str machinectl_path: Filename or complete path to the ``machinectl`` binary. ``PATH`` will be searched if given as a filename. Defaults to @@ -774,7 +791,7 @@ Context Class handle which is placed in the message's `reply_to`. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Message msg: @@ -793,7 +810,7 @@ Context Class The message. :param float deadline: - If not ``None``, seconds before timing out waiting for a reply. + If not :data:`None`, seconds before timing out waiting for a reply. :raises mitogen.core.TimeoutError: No message was received and `deadline` passed. @@ -915,8 +932,8 @@ Receiver Class Router to register the handler on. :param int handle: - If not ``None``, an explicit handle to register, otherwise an unused - handle is chosen. + If not :data:`None`, an explicit handle to register, otherwise an + unused handle is chosen. :param bool persist: If :data:`True`, do not unregister the receiver's handler after the @@ -924,13 +941,13 @@ Receiver Class :param mitogen.core.Context respondent: Reference to the context this receiver is receiving from. If not - ``None``, arranges for the receiver to receive a dead message if + :data:`None`, arranges for the receiver to receive a dead message if messages can no longer be routed to the context, due to disconnection or exit. .. attribute:: notify = None - If not ``None``, a reference to a function invoked as + If not :data:`None`, a reference to a function invoked as `notify(receiver)` when a new message is delivered to this receiver. Used by :py:class:`mitogen.select.Select` to implement waiting on multiple receivers. @@ -984,7 +1001,7 @@ Receiver Class Sleep waiting for a message to arrive on this receiver. :param float timeout: - If not ``None``, specifies a timeout in seconds. + If not :data:`None`, specifies a timeout in seconds. :raises mitogen.core.ChannelError: The remote end indicated the channel should be closed, or @@ -1167,10 +1184,10 @@ Select Class message may be posted at any moment between :py:meth:`empty` and :py:meth:`get`. - :py:meth:`empty` may return ``False`` even when :py:meth:`get` would - block if another thread has drained a receiver added to this select. - This can be avoided by only consuming each receiver from a single - thread. + :py:meth:`empty` may return :data:`False` even when :py:meth:`get` + would block if another thread has drained a receiver added to this + select. This can be avoided by only consuming each receiver from a + single thread. .. py:method:: __iter__ (self) @@ -1354,8 +1371,8 @@ A random assortment of utility functions useful on masters and children. variables. See :ref:`logging-env-vars`. :param str path: - If not ``None``, a filesystem path to write logs to. Otherwise, logs - are written to :py:data:`sys.stderr`. + If not :data:`None`, a filesystem path to write logs to. Otherwise, + logs are written to :py:data:`sys.stderr`. :param bool io: If :data:`True`, include extremely verbose IO logs in the output. @@ -1395,29 +1412,9 @@ Exceptions .. currentmodule:: mitogen.core -.. class:: Error (fmt, \*args) - - Base for all exceptions raised by Mitogen. - -.. class:: CallError (e) - - Raised when :py:meth:`Context.call() ` fails. - A copy of the traceback from the external context is appended to the - exception message. - -.. class:: ChannelError (fmt, \*args) - - Raised when a channel dies or has been closed. - -.. class:: LatchError (fmt, \*args) - - Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` that - has been marked closed. - -.. class:: StreamError (fmt, \*args) - - Raised when a stream cannot be established. - -.. class:: TimeoutError (fmt, \*args) - - Raised when a timeout occurs on a stream. +.. autoclass:: Error +.. autoclass:: CallError +.. autoclass:: ChannelError +.. autoclass:: LatchError +.. autoclass:: StreamError +.. autoclass:: TimeoutError diff --git a/docs/changelog.rst b/docs/changelog.rst index fd098e6a..f3e0dab5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,99 @@ Release Notes * Compatible with development versions of Ansible post https://github.com/ansible/ansible/pull/41749 +v0.2.3 (2018-08-??) +------------------- + +Mitogen for Ansible +~~~~~~~~~~~~~~~~~~~ + +* `#251 `_, + `#340 `_: Connection Delegation + could establish connections to the wrong target when ``delegate_to:`` is + present. + +* `#291 `_: when Mitogen had + previously been installed using ``pip`` or ``setuptools``, the globally + installed version could conflict with a newer version bundled with an + extension that had been installed using the documented steps. Now the bundled + library always overrides over any system-installed copy. + +* `#321 `_: temporary file handling + has been simplified and additional network roundtrips have been removed, + undoing earlier damage caused by compatibility fixes, and improving 2.6 + compatibility. One directory is created at startup for each persistent + interpreter. See :ref:`ansible_tempfiles` for a complete description. + +* `#324 `_: plays with a custom + ``module_utils`` would fail due to fallout from the Python 3 port and related + tests being disabled. + +* `#331 `_: fixed known issue: the + connection multiplexer subprocess always exits before the main Ansible + process exits, ensuring logs generated by it do not overwrite the user's + prompt when ``-vvv`` is enabled. + +* `#332 `_: support a new + :data:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. + +* `#338 `_: compatibility: changes to + ``/etc/environment`` and ``~/.pam_environment`` made by a task are reflected + in the runtime environment of subsequent tasks. See + :ref:`ansible_process_env` for a complete description. + +* `#343 `_: the sudo ``--login`` + option is supported. + +* `#344 `_: connections no longer + fail when the parent machine's logged in username contains slashes. + +* Runs with many targets executed the module dependency scanner redundantly + due to missing synchronization, causing significant wasted computation in the + connection multiplexer subprocess. For one real-world playbook the scanner + runtime was reduced by 95%, which may manifest as shorter runs. + +* A missing check caused an exception traceback to appear when using the + ``ansible`` command-line tool with a missing or misspelled module name. + +* Ansible since >2.6 began importing ``__main__`` from + ``ansible.module_utils.basic``, causing an error during execution, due to the + controller being configured to refuse network imports outside the + ``ansible.*`` namespace. Update the target implementation to construct a stub + ``__main__`` module to satisfy the otherwise seemingly vestigial import. + + +Core Library +~~~~~~~~~~~~ + +* `#313 `_: + :meth:`mitogen.parent.Context.call` was documented as capable of accepting + static methods. While possible on Python 2.x the result is very ugly, and in + every case it should be trivially possible to replace with a class method. + The API docs were updated to remove mention of static methods. + +* `#339 `_: the LXD connection method + was erroneously executing LXC Classic commands. + +* Add a :func:`mitogen.fork.on_fork` function to allow non-Mitogen managed + process forks to clean up Mitogen resources in the forked chlid. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +the bug reports in this release contributed by +`Alex Russu `_, +`atoom `_, +`Dan Quackenbush `_, +`Jesse London `_, +`Luca Nunzi `_, +`Pateek Jain `_, +`Pierre-Henry Muller `_, +`Rick Box `_, and +`Timo Beckers `_. + + v0.2.2 (2018-07-26) ------------------- @@ -204,11 +297,16 @@ Mitogen for Ansible for Message(..., 102, ...), my ID is ...* may be visible. These are due to a minor race while initializing logging and can be ignored. -* When running with ``-vvv``, log messages will be printed to the console - *after* the Ansible run completes, as connection multiplexer shutdown only - begins after Ansible exits. This is due to a lack of suitable shutdown hook - in Ansible, and is fairly harmless, albeit cosmetically annoying. A future - release may include a solution. +.. * When running with ``-vvv``, log messages will be printed to the console + *after* the Ansible run completes, as connection multiplexer shutdown only + begins after Ansible exits. This is due to a lack of suitable shutdown hook + in Ansible, and is fairly harmless, albeit cosmetically annoying. A future + release may include a solution. + +.. * Configurations will break that rely on the `hashbang argument splitting + behaviour `_ of the + ``ansible_python_interpreter`` setting, contrary to the Ansible + documentation. This will be addressed in a future 0.2 release. * Performance does not scale linearly with target count. This requires significant additional work, as major bottlenecks exist in the surrounding @@ -237,11 +335,6 @@ Mitogen for Ansible actions, such as the ``synchronize`` module. This will be addressed in the 0.3 series. -* Configurations will break that rely on the `hashbang argument splitting - behaviour `_ of the - ``ansible_python_interpreter`` setting, contrary to the Ansible - documentation. This will be addressed in a future 0.2 release. - Core Library ~~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index 57adf597..abb6e97e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ html_theme_options = { 'head_font_family': "Georgia, serif", } htmlhelp_basename = 'mitogendoc' -intersphinx_mapping = {'python': ('https://docs.python.org/2', None)} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} language = None master_doc = 'toc' project = u'Mitogen' diff --git a/docs/howitworks.rst b/docs/howitworks.rst index b14ceab7..1d45647f 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -332,7 +332,7 @@ Masters listen on the following handles: Receives the name of a module to load `fullname`, locates the source code for `fullname`, and routes one or more :py:data:`LOAD_MODULE` messages back towards the sender of the :py:data:`GET_MODULE` request. If lookup fails, - ``None`` is sent instead. + :data:`None` is sent instead. See :ref:`import-preloading` for a deeper discussion of :py:data:`GET_MODULE`/:py:data:`LOAD_MODULE`. @@ -355,12 +355,13 @@ Children listen on the following handles: Receives `(pkg_present, path, compressed, related)` tuples, composed of: - * **pkg_present**: Either ``None`` for a plain ``.py`` module, or a list of - canonical names of submodules existing witin this package. For example, a - :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would return a - list like: `["mitogen.core", "mitogen.fakessh", "mitogen.master", ..]`. - This list is used by children to avoid generating useless round-trips due - to Python 2.x's :keyword:`import` statement behavior. + * **pkg_present**: Either :data:`None` for a plain ``.py`` module, or a + list of canonical names of submodules existing witin this package. For + example, a :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would + return a list like: `["mitogen.core", "mitogen.fakessh", + "mitogen.master", ..]`. This list is used by children to avoid generating + useless round-trips due to Python 2.x's :keyword:`import` statement + behavior. * **path**: Original filesystem where the module was found on the master. * **compressed**: :py:mod:`zlib`-compressed module source code. * **related**: list of canonical module names on which this module appears diff --git a/docs/internals.rst b/docs/internals.rst index 3d4d4130..03f12e1e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -8,17 +8,27 @@ Internal API Reference signals -mitogen.core -============ +Constants +========= + +.. currentmodule:: mitogen.core +.. autodata:: CHUNK_SIZE Latch Class ------------ +=========== .. currentmodule:: mitogen.core +.. autoclass:: Latch + :members: -.. autoclass:: Latch () +PidfulStreamHandler Class +========================= + +.. currentmodule:: mitogen.core +.. autoclass:: PidfulStreamHandler + :members: Side Class @@ -50,24 +60,24 @@ Side Class .. attribute:: fd - Integer file descriptor to perform IO on, or ``None`` if + Integer file descriptor to perform IO on, or :data:`None` if :py:meth:`close` has been called. .. attribute:: keep_alive - If ``True``, causes presence of this side in :py:class:`Broker`'s + If :data:`True`, causes presence of this side in :py:class:`Broker`'s active reader set to defer shutdown until the side is disconnected. .. method:: fileno - Return :py:attr:`fd` if it is not ``None``, otherwise raise + Return :py:attr:`fd` if it is not :data:`None`, otherwise raise :py:class:`StreamError`. This method is implemented so that :py:class:`Side` can be used directly by :py:func:`select.select`. .. method:: close - Call :py:func:`os.close` on :py:attr:`fd` if it is not ``None``, then - set it to ``None``. + Call :py:func:`os.close` on :py:attr:`fd` if it is not :data:`None`, + then set it to :data:`None`. .. method:: read (n=CHUNK_SIZE) @@ -89,12 +99,9 @@ Side Class wrapping the underlying :py:func:`os.write` call with :py:func:`io_op` to trap common disconnection connditions. - :py:meth:`read` always behaves as if it is writing to a regular UNIX - file; socket, pipe, and TTY disconnection errors are masked and result - in a 0-sized write. - :returns: - Number of bytes written, or ``None`` if disconnection was detected. + Number of bytes written, or :data:`None` if disconnection was + detected. Stream Classes @@ -302,123 +309,47 @@ mitogen.master Blocking I/O Functions ----------------------- +====================== These functions exist to support the blocking phase of setting up a new context. They will eventually be replaced with asynchronous equivalents. -.. currentmodule:: mitogen.master - -.. function:: iter_read(fd, deadline=None) - - Return a generator that arranges for up to 4096-byte chunks to be read at a - time from the file descriptor `fd` until the generator is destroyed. - - :param fd: - File descriptor to read from. - - :param deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - - :raises mitogen.core.TimeoutError: - Attempt to read beyond deadline. - - :raises mitogen.core.StreamError: - Attempt to read past end of file. - - -.. currentmodule:: mitogen.master - -.. function:: write_all (fd, s, deadline=None) - - Arrange for all of bytestring `s` to be written to the file descriptor - `fd`. - - :param int fd: - File descriptor to write to. - - :param bytes s: - Bytestring to write to file descriptor. - - :param float deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - - :raises mitogen.core.TimeoutError: - Bytestring could not be written entirely before deadline was exceeded. - - :raises mitogen.core.StreamError: - File descriptor was disconnected before write could complete. - - -Helper Functions ----------------- - -.. currentmodule:: mitogen.core - -.. function:: io_op (func, \*args) - - Wrap a function that may raise :py:class:`OSError`, trapping common error - codes relating to disconnection events in various subsystems: - - * When performing IO against a TTY, disconnection of the remote end is - signalled by :py:data:`errno.EIO`. - * When performing IO against a socket, disconnection of the remote end is - signalled by :py:data:`errno.ECONNRESET`. - * When performing IO against a pipe, disconnection of the remote end is - signalled by :py:data:`errno.EPIPE`. - - :returns: - Tuple of `(return_value, disconnected)`, where `return_value` is the - return value of `func(\*args)`, and `disconnected` is ``True`` if - disconnection was detected, otherwise ``False``. - - .. currentmodule:: mitogen.parent +.. autofunction:: discard_until +.. autofunction:: iter_read +.. autofunction:: write_all -.. autofunction:: create_child +Subprocess Creation Functions +============================= .. currentmodule:: mitogen.parent - +.. autofunction:: create_child +.. autofunction:: hybrid_tty_create_child .. autofunction:: tty_create_child -.. currentmodule:: mitogen.parent +Helper Functions +================ -.. autofunction:: hybrid_tty_create_child +.. currentmodule:: mitogen.core +.. autofunction:: to_text +.. autofunction:: has_parent_authority +.. autofunction:: set_cloexec +.. autofunction:: set_nonblock +.. autofunction:: set_block +.. autofunction:: io_op +.. currentmodule:: mitogen.parent +.. autofunction:: close_nonstandard_fds +.. autofunction:: create_socketpair .. currentmodule:: mitogen.master - -.. function:: get_child_modules (path) - - Return the suffixes of submodules directly neated beneath of the package - directory at `path`. - - :param str path: - Path to the module's source code on disk, or some PEP-302-recognized - equivalent. Usually this is the module's ``__file__`` attribute, but - is specified explicitly to avoid loading the module. - - :return: - List of submodule name suffixes. - +.. autofunction:: get_child_modules .. currentmodule:: mitogen.minify - -.. autofunction:: minimize_source (source) - - Remove comments and docstrings from Python `source`, preserving line - numbers and syntax of empty blocks. - - :param str source: - The source to minimize. - - :returns str: - The minimized source. +.. autofunction:: minimize_source Signals diff --git a/docs/signals.rst b/docs/signals.rst index 1c41353a..19533bb1 100644 --- a/docs/signals.rst +++ b/docs/signals.rst @@ -19,16 +19,10 @@ functions registered to receive it will be called back. Functions --------- -.. function:: mitogen.core.listen (obj, name, func) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked when the named signal - is fired by `obj`. - -.. function:: mitogen.core.fire (obj, name, \*args, \*\*kwargs) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked for every function - registered for the named signal on `obj`. +.. currentmodule:: mitogen.core +.. autofunction:: listen +.. autofunction:: fire List diff --git a/mitogen/core.py b/mitogen/core.py index cfcd70d2..12983071 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -116,7 +116,34 @@ AnyTextType = (BytesType, UnicodeType) if sys.version_info < (2, 5): next = lambda it: it.next() +#: Default size for calls to :meth:`Side.read` or :meth:`Side.write`, and the +#: size of buffers configured by :func:`mitogen.parent.create_socketpair`. This +#: value has many performance implications, 128KiB seems to be a sweet spot. +#: +#: * When set low, large messages cause many :class:`Broker` IO loop +#: iterations, burning CPU and reducing throughput. +#: * When set high, excessive RAM is reserved by the OS for socket buffers (2x +#: per child), and an identically sized temporary userspace buffer is +#: allocated on each read that requires zeroing, and over a particular size +#: may require two system calls to allocate/deallocate. +#: +#: Care must be taken to ensure the underlying kernel object and receiving +#: program support the desired size. For example, +#: +#: * Most UNIXes have TTYs with fixed 2KiB-4KiB buffers, making them unsuitable +#: for efficient IO. +#: * Different UNIXes have varying presets for pipes, which may not be +#: configurable. On recent Linux the default pipe buffer size is 64KiB, but +#: under memory pressure may be as low as 4KiB for unprivileged processes. +#: * When communication is via an intermediary process, its internal buffers +#: effect the speed OS buffers will drain. For example OpenSSH uses 64KiB +#: reads. +#: +#: An ideal :class:`Message` has a size that is a multiple of +#: :data:`CHUNK_SIZE` inclusive of headers, to avoid wasting IO loop iterations +#: writing small trailer chunks. CHUNK_SIZE = 131072 + _tls = threading.local() @@ -131,6 +158,13 @@ else: class Error(Exception): + """Base for all exceptions raised by Mitogen. + + :param str fmt: + Exception text, or format string if `args` is non-empty. + :param tuple args: + Format string arguments. + """ def __init__(self, fmt=None, *args): if args: fmt %= args @@ -140,10 +174,14 @@ class Error(Exception): class LatchError(Error): + """Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` + that has been marked closed.""" pass class Blob(BytesType): + """A serializable bytes subclass whose content is summarized in repr() + output, making it suitable for logging binary data.""" def __repr__(self): return '[blob: %d bytes]' % len(self) @@ -152,6 +190,8 @@ class Blob(BytesType): class Secret(UnicodeType): + """A serializable unicode subclass whose content is masked in repr() + output, making it suitable for logging passwords.""" def __repr__(self): return '[secret]' @@ -165,6 +205,10 @@ class Secret(UnicodeType): class Kwargs(dict): + """A serializable dict subclass that indicates the contained keys should be + be coerced to Unicode on Python 3 as required. Python 2 produces keyword + argument dicts whose keys are bytestrings, requiring a helper to ensure + compatibility with Python 3.""" if PY3: def __init__(self, dct): for k, v in dct.items(): @@ -181,6 +225,10 @@ class Kwargs(dict): class CallError(Error): + """Serializable :class:`Error` subclass raised when + :py:meth:`Context.call() ` fails. A copy of + the traceback from the external context is appended to the exception + message.""" def __init__(self, fmt=None, *args): if not isinstance(fmt, BaseException): Error.__init__(self, fmt, *args) @@ -207,37 +255,54 @@ def _unpickle_call_error(s): class ChannelError(Error): + """Raised when a channel dies or has been closed.""" remote_msg = 'Channel closed by remote end.' local_msg = 'Channel closed by local end.' class StreamError(Error): + """Raised when a stream cannot be established.""" pass class TimeoutError(Error): + """Raised when a timeout occurs on a stream.""" pass def to_text(o): - if isinstance(o, UnicodeType): - return UnicodeType(o) + """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of + :class:`bytes`, otherwise pass it to the :class:`str` constructor. The + returned object is always a plain :class:`str`, any subclass is removed.""" if isinstance(o, BytesType): return o.decode('utf-8') return UnicodeType(o) def has_parent_authority(msg, _stream=None): + """Policy function for use with :class:`Receiver` and + :meth:`Router.add_handler` that requires incoming messages to originate + from a parent context, or on a :class:`Stream` whose :attr:`auth_id + ` has been set to that of a parent context or the current + context.""" return (msg.auth_id == mitogen.context_id or msg.auth_id in mitogen.parent_ids) def listen(obj, name, func): + """ + Arrange for `func(*args, **kwargs)` to be invoked when the named signal is + fired by `obj`. + """ signals = vars(obj).setdefault('_signals', {}) signals.setdefault(name, []).append(func) def fire(obj, name, *args, **kwargs): + """ + Arrange for `func(*args, **kwargs)` to be invoked for every function + registered for the named signal on `obj`. + """ signals = vars(obj).get('_signals', {}) return [func(*args, **kwargs) for func in signals.get(name, ())] @@ -253,7 +318,8 @@ def takes_router(func): def is_blacklisted_import(importer, fullname): - """Return ``True`` if `fullname` is part of a blacklisted package, or if + """ + Return :data:`True` if `fullname` is part of a blacklisted package, or if any packages have been whitelisted and `fullname` is not part of one. NB: @@ -266,22 +332,51 @@ def is_blacklisted_import(importer, fullname): def set_cloexec(fd): + """Set the file descriptor `fd` to automatically close on + :func:`os.execve`. This has no effect on file descriptors inherited across + :func:`os.fork`, they must be explicitly closed through some other means, + such as :func:`mitogen.fork.on_fork`.""" flags = fcntl.fcntl(fd, fcntl.F_GETFD) assert fd > 2 fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def set_nonblock(fd): + """Set the file descriptor `fd` to non-blocking mode. For most underlying + file types, this causes :func:`os.read` or :func:`os.write` to raise + :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) def set_block(fd): + """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) def io_op(func, *args): + """Wrap `func(*args)` that may raise :class:`select.error`, + :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating + to disconnection and retry events in various subsystems: + + * When a signal is delivered to the process on Python 2, system call retry + is signalled through :data:`errno.EINTR`. The invocation is automatically + restarted. + * When performing IO against a TTY, disconnection of the remote end is + signalled by :data:`errno.EIO`. + * When performing IO against a socket, disconnection of the remote end is + signalled by :data:`errno.ECONNRESET`. + * When performing IO against a pipe, disconnection of the remote end is + signalled by :data:`errno.EPIPE`. + + :returns: + Tuple of `(return_value, disconnected)`, where `return_value` is the + return value of `func(\*args)`, and `disconnected` is :data:`True` if + disconnection was detected, otherwise :data:`False`. + """ while True: try: return func(*args), False @@ -296,7 +391,19 @@ def io_op(func, *args): class PidfulStreamHandler(logging.StreamHandler): + """A :class:`logging.StreamHandler` subclass used when + :meth:`Router.enable_debug() ` has been + called, or the `debug` parameter was specified during context construction. + Verifies the process ID has not changed on each call to :meth:`emit`, + reopening the associated log file when a change is detected. + + This ensures logging to the per-process output files happens correctly even + when uncooperative third party components call :func:`os.fork`. + """ + #: PID that last opened the log file. open_pid = None + + #: Output path template. template = '/tmp/mitogen.%s.%s.log' def _reopen(self): @@ -614,6 +721,7 @@ class Importer(object): 'fork', 'jail', 'lxc', + 'lxd', 'master', 'minify', 'parent', @@ -935,7 +1043,7 @@ class Stream(BasicStream): :py:class:`BasicStream` subclass implementing mitogen's :ref:`stream protocol `. """ - #: If not ``None``, :py:class:`Router` stamps this into + #: If not :data:`None`, :py:class:`Router` stamps this into #: :py:attr:`Message.auth_id` of every message received on this stream. auth_id = None @@ -958,6 +1066,16 @@ class Stream(BasicStream): def construct(self): pass + def _internal_receive(self, broker, buf): + if self._input_buf and self._input_buf_len < 128: + self._input_buf[0] += buf + else: + self._input_buf.append(buf) + + self._input_buf_len += len(buf) + while self._receive_one(broker): + pass + def on_receive(self, broker): """Handle the next complete message on the stream. Raise :py:class:`StreamError` on failure.""" @@ -967,14 +1085,7 @@ class Stream(BasicStream): if not buf: return self.on_disconnect(broker) - if self._input_buf and self._input_buf_len < 128: - self._input_buf[0] += buf - else: - self._input_buf.append(buf) - - self._input_buf_len += len(buf) - while self._receive_one(broker): - pass + self._internal_receive(broker, buf) HEADER_FMT = '>LLLLLL' HEADER_LEN = struct.calcsize(HEADER_FMT) diff --git a/mitogen/docker.py b/mitogen/docker.py index 38ee9d4e..36b0635b 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -43,6 +43,11 @@ class Stream(mitogen.parent.Stream): username = None docker_path = 'docker' + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + def construct(self, container=None, image=None, docker_path=None, username=None, **kwargs): diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 4d6c21db..71b12221 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -52,7 +52,7 @@ class Stream(mitogen.parent.Stream): super(Stream, self).construct(**kwargs) self.container = container if lxc_attach_path: - self.lxc_attach_path = lxc_attach_apth + self.lxc_attach_path = lxc_attach_path def connect(self): super(Stream, self).connect() diff --git a/mitogen/lxd.py b/mitogen/lxd.py new file mode 100644 index 00000000..6e8e8b18 --- /dev/null +++ b/mitogen/lxd.py @@ -0,0 +1,70 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + create_child_args = { + # If lxc finds any of stdin, stdout, stderr connected to a TTY, to + # prevent input injection it creates a proxy pty, forcing all IO to be + # buffered in <4KiB chunks. So ensure stderr is also routed to the + # socketpair. + 'merge_stdio': True + } + + container = None + lxc_path = 'lxc' + python_path = 'python' + + def construct(self, container, lxc_path=None, **kwargs): + super(Stream, self).construct(**kwargs) + self.container = container + if lxc_path: + self.lxc_path = lxc_path + + def connect(self): + super(Stream, self).connect() + self.name = u'lxd.' + self.container + + def get_boot_command(self): + bits = [ + self.lxc_path, + 'exec', + '--force-noninteractive', + self.container, + '--', + ] + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen/master.py b/mitogen/master.py index d057f7f1..671bee85 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -84,6 +84,17 @@ def _stdlib_paths(): def get_child_modules(path): + """Return the suffixes of submodules directly neated beneath of the package + directory at `path`. + + :param str path: + Path to the module's source code on disk, or some PEP-302-recognized + equivalent. Usually this is the module's ``__file__`` attribute, but + is specified explicitly to avoid loading the module. + + :return: + List of submodule name suffixes. + """ it = pkgutil.iter_modules([os.path.dirname(path)]) return [to_text(name) for _, name, _ in it] @@ -276,7 +287,7 @@ def is_stdlib_path(path): def is_stdlib_name(modname): - """Return ``True`` if `modname` appears to come from the standard + """Return :data:`True` if `modname` appears to come from the standard library.""" if imp.is_builtin(modname) != 0: return True @@ -412,8 +423,8 @@ class ModuleFinder(object): source code. :returns: - Tuple of `(module path, source text, is package?)`, or ``None`` if - the source cannot be found. + Tuple of `(module path, source text, is package?)`, or :data:`None` + if the source cannot be found. """ tup = self._found_cache.get(fullname) if tup: diff --git a/mitogen/minify.py b/mitogen/minify.py index 26ecf62f..a261bf6a 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -48,10 +48,16 @@ except ImportError: @lru_cache() def minimize_source(source): - """Remove most comments and docstrings from Python source code. + """Remove comments and docstrings from Python `source`, preserving line + numbers and syntax of empty blocks. + + :param str source: + The source to minimize. + + :returns str: + The minimized source. """ - if not isinstance(source, mitogen.core.UnicodeType): - source = source.decode('utf-8') + source = mitogen.core.to_text(source) tokens = tokenize.generate_tokens(StringIO(source).readline) tokens = strip_comments(tokens) tokens = strip_docstrings(tokens) diff --git a/mitogen/parent.py b/mitogen/parent.py index 4299d3cd..44504258 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -93,6 +93,19 @@ def get_core_source(): return inspect.getsource(mitogen.core) +def get_default_remote_name(): + """ + Return the default name appearing in argv[0] of remote machines. + """ + s = u'%s@%s:%d' + s %= (getpass.getuser(), socket.gethostname(), os.getpid()) + # In mixed UNIX/Windows environments, the username may contain slashes. + return s.translate({ + ord(u'\\'): ord(u'_'), + ord(u'/'): ord(u'_') + }) + + def is_immediate_child(msg, stream): """ Handler policy that requires messages to arrive only from immediately @@ -144,6 +157,14 @@ def close_nonstandard_fds(): def create_socketpair(): + """ + Create a :func:`socket.socketpair` to use for use as a child process's UNIX + stdio channels. As socket pairs are bidirectional, they are economical on + file descriptor usage as the same descriptor can be used for ``stdin`` and + ``stdout``. As they are sockets their buffers are tunable, allowing large + buffers to be configured in order to improve throughput for file transfers + and reduce :class:`mitogen.core.Broker` IO loop iterations. + """ parentfp, childfp = socket.socketpair() parentfp.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, @@ -284,6 +305,22 @@ def hybrid_tty_create_child(args): def write_all(fd, s, deadline=None): + """Arrange for all of bytestring `s` to be written to the file descriptor + `fd`. + + :param int fd: + File descriptor to write to. + :param bytes s: + Bytestring to write to file descriptor. + :param float deadline: + If not :data:`None`, absolute UNIX timestamp after which timeout should + occur. + + :raises mitogen.core.TimeoutError: + Bytestring could not be written entirely before deadline was exceeded. + :raises mitogen.core.StreamError: + File descriptor was disconnected before write could complete. + """ timeout = None written = 0 poller = PREFERRED_POLLER() @@ -312,6 +349,20 @@ def write_all(fd, s, deadline=None): def iter_read(fds, deadline=None): + """Return a generator that arranges for up to 4096-byte chunks to be read + at a time from the file descriptor `fd` until the generator is destroyed. + + :param int fd: + File descriptor to read from. + :param float deadline: + If not :data:`None`, an absolute UNIX timestamp after which timeout + should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ poller = PREFERRED_POLLER() for fd in fds: poller.start_receive(fd) @@ -346,6 +397,24 @@ def iter_read(fds, deadline=None): def discard_until(fd, s, deadline): + """Read chunks from `fd` until one is encountered that ends with `s`. This + is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and + mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to + appear, indicating the first stage is ready to receive the compressed + :mod:`mitogen.core` source. + + :param int fd: + File descriptor to read from. + :param bytes s: + Marker string to discard until encountered. + :param float deadline: + Absolute UNIX timestamp after which timeout should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ for buf in iter_read([fd], deadline): if IOLOG.level == logging.DEBUG: for line in buf.splitlines(): @@ -765,8 +834,7 @@ class Stream(mitogen.core.Stream): if connect_timeout: self.connect_timeout = connect_timeout if remote_name is None: - remote_name = '%s@%s:%d' - remote_name %= (getpass.getuser(), socket.gethostname(), os.getpid()) + remote_name = get_default_remote_name() if '/' in remote_name or '\\' in remote_name: raise ValueError('remote_name= cannot contain slashes') self.remote_name = remote_name @@ -968,7 +1036,9 @@ class Stream(mitogen.core.Stream): self._reap_child() raise - #: For ssh.py, this must be at least max(len('password'), len('debug1:')) + #: Sentinel value emitted by the first stage to indicate it is ready to + #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have + #: length of at least `max(len('password'), len('debug1:'))` EC0_MARKER = mitogen.core.b('MITO000\n') EC1_MARKER = mitogen.core.b('MITO001\n') @@ -1288,6 +1358,9 @@ class Router(mitogen.core.Router): def lxc(self, **kwargs): return self.connect(u'lxc', **kwargs) + def lxd(self, **kwargs): + return self.connect(u'lxd', **kwargs) + def setns(self, **kwargs): return self.connect(u'setns', **kwargs) diff --git a/mitogen/service.py b/mitogen/service.py index 923ec04a..f1ccadde 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -372,8 +372,9 @@ class DeduplicatingInvoker(Invoker): class Service(object): - #: Sentinel object to suppress reply generation, since returning ``None`` - #: will trigger a response message containing the pickled ``None``. + #: Sentinel object to suppress reply generation, since returning + #: :data:`None` will trigger a response message containing the pickled + #: :data:`None`. NO_REPLY = object() invoker_class = Invoker diff --git a/mitogen/setns.py b/mitogen/setns.py index 1779ca77..224550ce 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -94,6 +94,16 @@ def get_lxc_pid(path, name): raise Error("could not find PID from lxc-info output.\n%s", output) +def get_lxd_pid(path, name): + output = _run_command([path, 'info', name]) + for line in output.splitlines(): + bits = line.split() + if bits and bits[0] == 'Pid:': + return int(bits[1]) + + raise Error("could not find PID from lxc output.\n%s", output) + + def get_machinectl_pid(path, name): output = _run_command([path, 'status', name]) for line in output.splitlines(): @@ -110,18 +120,22 @@ class Stream(mitogen.parent.Stream): container = None username = None kind = None + python_path = 'python' docker_path = 'docker' + lxc_path = 'lxc' lxc_info_path = 'lxc-info' machinectl_path = 'machinectl' GET_LEADER_BY_KIND = { 'docker': ('docker_path', get_docker_pid), 'lxc': ('lxc_info_path', get_lxc_pid), + 'lxd': ('lxc_path', get_lxd_pid), 'machinectl': ('machinectl_path', get_machinectl_pid), } def construct(self, container, kind, username=None, docker_path=None, - lxc_info_path=None, machinectl_path=None, **kwargs): + lxc_path=None, lxc_info_path=None, machinectl_path=None, + **kwargs): super(Stream, self).construct(**kwargs) if kind not in self.GET_LEADER_BY_KIND: raise Error('unsupported container kind: %r', kind) @@ -132,6 +146,8 @@ class Stream(mitogen.parent.Stream): self.username = username if docker_path: self.docker_path = docker_path + if lxc_path: + self.lxc_path = lxc_path if lxc_info_path: self.lxc_info_path = lxc_info_path if machinectl_path: diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 1ec5c2f6..402d8549 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -49,7 +49,7 @@ SUDO_OPTIONS = [ #(False, 'str', '--group', '-g') (True, 'bool', '--set-home', '-H'), #(False, 'str', '--host', '-h') - #(False, 'bool', '--login', '-i') + (False, 'bool', '--login', '-i'), #(False, 'bool', '--remove-timestamp', '-K') #(False, 'bool', '--reset-timestamp', '-k') #(False, 'bool', '--list', '-l') @@ -116,10 +116,11 @@ class Stream(mitogen.parent.Stream): password = None preserve_env = False set_home = False + login = False def construct(self, username=None, sudo_path=None, password=None, preserve_env=None, set_home=None, sudo_args=None, - **kwargs): + login=None, **kwargs): super(Stream, self).construct(**kwargs) opts = parse_sudo_flags(sudo_args or []) @@ -133,6 +134,8 @@ class Stream(mitogen.parent.Stream): self.preserve_env = preserve_env or opts.preserve_env if (set_home or opts.set_home) is not None: self.set_home = set_home or opts.set_home + if (login or opts.login) is not None: + self.login = True def connect(self): super(Stream, self).connect() @@ -144,13 +147,16 @@ class Stream(mitogen.parent.Stream): def get_boot_command(self): # Note: sudo did not introduce long-format option processing until July - # 2013, so even though we parse long-format options, we always supply - # short-form to the sudo command. + # 2013, so even though we parse long-format options, supply short-form + # to the sudo command. bits = [self.sudo_path, '-u', self.username] if self.preserve_env: bits += ['-E'] if self.set_home: bits += ['-H'] + if self.login: + bits += ['-i'] + bits = bits + super(Stream, self).get_boot_command() LOG.debug('sudo command line: %r', bits) return bits diff --git a/preamble_size.py b/preamble_size.py index 66d5ccf3..bf3b5950 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -4,6 +4,7 @@ contexts. """ import inspect +import sys import zlib import mitogen.fakessh @@ -24,6 +25,10 @@ print('Preamble size: %s (%.2fKiB)' % ( len(stream.get_preamble()), len(stream.get_preamble()) / 1024.0, )) +if '--dump' in sys.argv: + print(zlib.decompress(stream.get_preamble())) + exit() + print( ' ' diff --git a/tests/README.md b/tests/README.md index f5bbbc41..51464989 100644 --- a/tests/README.md +++ b/tests/README.md @@ -73,13 +73,16 @@ also by Ansible's `osx_setup.yml`. used to target this account, the parent session requires a TTY and the account password must be entered. -`mitogen__user1` .. `mitogen__user21` +`mitogen__user1` .. `mitogen__user5` These accounts do not have passwords set. They exist to test the Ansible interpreter recycling logic. +`mitogen__sudo1` .. `mitogen__sudo4` + May passwordless sudo to any account. + `mitogen__webapp` A plain old account with no sudo access, used as the target for fakessh - tddests. + tests. # Ansible Integration Test Environment diff --git a/tests/ansible/Makefile b/tests/ansible/Makefile index d9bdc521..00d6a8ab 100644 --- a/tests/ansible/Makefile +++ b/tests/ansible/Makefile @@ -1,10 +1,14 @@ -all: \ - lib/modules/custom_binary_producing_junk \ - lib/modules/custom_binary_producing_json +TARGETS+=lib/modules/custom_binary_producing_junk +TARGETS+=lib/modules/custom_binary_producing_json + +all: clean $(TARGETS) lib/modules/custom_binary_producing_junk: lib/modules.src/custom_binary_producing_junk.c $(CC) -o $@ $< lib/modules/custom_binary_producing_json: lib/modules.src/custom_binary_producing_json.c $(CC) -o $@ $< + +clean: + rm -f $(TARGETS) diff --git a/tests/ansible/README.md b/tests/ansible/README.md index a76c7c1f..46320951 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -1,5 +1,5 @@ -# ``tests/ansible`` Directory +# `tests/ansible` Directory This is an an organically growing collection of integration and regression tests used for development and end-user bug reports. @@ -10,10 +10,10 @@ demonstrator for what does and doesn't work. ## Preparation -For OS X, run the ``osx_setup.yml`` script to create a bunch of users. +See `../image_prep/README.md`. -## ``run_ansible_playbook.sh`` +## `run_ansible_playbook.sh` This is necessary to set some environment variables used by future tests, as there appears to be no better way to inject them into the top-level process @@ -22,6 +22,19 @@ environment before the Mitogen connection process forks. ## Running Everything -``` -ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml -``` +`ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml` + + +## `hosts/` and `common-hosts` + +To support running the tests against a dev machine that has the requisite user +accounts, the the default inventory is a directory containing a 'localhost' +file that defines 'localhost' to be named 'target' in Ansible inventory, and a +symlink to 'common-hosts', which defines additional targets that all derive +from 'target'. + +This allows `ansible_tests.sh` to reuse the common-hosts definitions while +replacing localhost as the test target by creating a new directory that +similarly symlinks in common-hosts. + +There may be a better solution for this, but it works fine for now. diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index d9224ab7..539964b8 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -17,10 +17,6 @@ timeout = 10 # On Travis, paramiko check fails due to host key checking enabled. host_key_checking = False -# "mitogen-tests" required by integration/runner/remote_tmp.yml -# "$HOME" required by integration/action/make_tmp_path.yml -remote_tmp = $HOME/.ansible/mitogen-tests/ - [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s pipelining = True diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml new file mode 100644 index 00000000..0feb57c5 --- /dev/null +++ b/tests/ansible/bench/loop-100-items.yml @@ -0,0 +1,10 @@ +# Execute 'hostname' 100 times in a loop. Loops execute within TaskExecutor +# within a single WorkerProcess, each iteration is a fair approximation of the +# non-controller overhead involved in executing a task. +# +# See also: loop-100-tasks.yml +# +- hosts: all + tasks: + - command: hostname + with_sequence: start=1 end=100 diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml new file mode 100644 index 00000000..bf6e31b8 --- /dev/null +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -0,0 +1,112 @@ +# Execute 'hostname' 100 times, using 100 individual tasks. Each task causes a +# new WorkerProcess to be forked, along with get_vars() calculation, and in the +# Mitogen extension, reestablishment of the UNIX socket connectionto the +# multiplexer process. +# +# It does not measure at least module dependency scanning (cached after first +# iteration). +# +# See also: loop-100-items.yml +# +- hosts: all + tasks: + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname diff --git a/tests/ansible/common-hosts b/tests/ansible/common-hosts new file mode 100644 index 00000000..6dafaf47 --- /dev/null +++ b/tests/ansible/common-hosts @@ -0,0 +1,51 @@ +# vim: syntax=dosini + +[connection-delegation-test] +cd-bastion +cd-rack11 mitogen_via=ssh-user@cd-bastion +cd-rack11a mitogen_via=root@cd-rack11 +cd-rack11a-docker mitogen_via=docker-admin@cd-rack11a ansible_connection=docker + +[connection-delegation-cycle] +# Create cycle with Docker container. +cdc-bastion mitogen_via=cdc-rack11a-docker +cdc-rack11 mitogen_via=ssh-user@cdc-bastion +cdc-rack11a mitogen_via=root@cdc-rack11 +cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker + +[conn-delegation] +cd-user1 ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=target + + +# Connection delegation scenarios. It's impossible to connection to them, but +# you can inspect the would-be config via "mitogen_get_stack" action. +[cd-no-connect] +# Normal inventory host, no aliasing. +cd-normal ansible_connection=mitogen_doas ansible_user=normal-user +# Inventory host that is really a different host. +cd-alias ansible_connection=ssh ansible_user=alias-user ansible_host=alias-host + +# Via one normal host. +cd-normal-normal mitogen_via=cd-normal +# Via one aliased host. +cd-normal-alias mitogen_via=cd-alias + +# newuser@host via host with explicit username. +cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-normal-user + +# doas:newuser via host. +cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user + + +# Connection Delegation issue #340 reproduction. +# Path to jails is SSH to H -> mitogen_sudo to root -> jail to J + +[issue340] +# 'target' plays the role of the normal host machine H. +# 'mitogen__sudo1' plays the role of root@H via mitogen_sudo. +# 'mitogen__user1' plays the role of root@J via mitogen__user1. +# 'mitogen__user2' plays the role of E, the delgate_to target for certs. + +i340-root ansible_user=mitogen__sudo1 ansible_connection=mitogen_sudo mitogen_via=target +i340-jail ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=i340-root +i340-certs ansible_user=mitogen__user2 ansible_connection=mitogen_sudo mitogen_via=target diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index b4ce3fcf..f7989ddf 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -1,5 +1,9 @@ - hosts: controller + vars: + git_username: '{{ lookup("pipe", "git config --global user.name") }}' + git_email: '{{ lookup("pipe", "git config --global user.email") }}' + tasks: - lineinfile: line: "net.ipv4.ip_forward=1" @@ -32,6 +36,11 @@ - shell: "rsync -a ~/.ssh {{inventory_hostname}}:" connection: local + - shell: | + git config --global user.email "{{git_username}}" + git config --global user.name "{{git_email}}" + name: set_git_config + - git: dest: ~/mitogen repo: https://github.com/dw/mitogen.git diff --git a/tests/ansible/hosts b/tests/ansible/hosts deleted file mode 100644 index 45bfb9ef..00000000 --- a/tests/ansible/hosts +++ /dev/null @@ -1,16 +0,0 @@ - -[test-targets] -localhost - -[connection-delegation-test] -cd-bastion -cd-rack11 mitogen_via=ssh-user@cd-bastion -cd-rack11a mitogen_via=root@cd-rack11 -cd-rack11a-docker mitogen_via=docker-admin@cd-rack11a ansible_connection=docker - -[connection-delegation-cycle] -# Create cycle with Docker container. -cdc-bastion mitogen_via=cdc-rack11a-docker -cdc-rack11 mitogen_via=ssh-user@cdc-bastion -cdc-rack11a mitogen_via=root@cdc-rack11 -cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker diff --git a/tests/ansible/hosts/common-hosts b/tests/ansible/hosts/common-hosts new file mode 120000 index 00000000..f3cc7f59 --- /dev/null +++ b/tests/ansible/hosts/common-hosts @@ -0,0 +1 @@ +../common-hosts \ No newline at end of file diff --git a/tests/ansible/hosts/localhost b/tests/ansible/hosts/localhost new file mode 100644 index 00000000..dc7df668 --- /dev/null +++ b/tests/ansible/hosts/localhost @@ -0,0 +1,2 @@ +[test-targets] +target ansible_host=localhost diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 83261208..eb39068b 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -1,63 +1,127 @@ +# +# Ensure _make_tmp_path returns the same result across invocations for a single +# user account, and that the path returned cleans itself up on connection +# termination. +# +# Related bugs prior to the new-style handling: +# https://github.com/dw/mitogen/issues/239 +# https://github.com/dw/mitogen/issues/301 - name: integration/action/make_tmp_path.yml hosts: test-targets any_errors_fatal: true tasks: - - name: "Find out root's homedir." - # Runs first because it blats regular Ansible facts with junk, so - # non-become run fixes that up. - setup: gather_subset=min - become: true - register: root_facts - - - name: "Find regular homedir" - setup: gather_subset=min - register: user_facts + - meta: end_play + when: not is_mitogen # - # non-become + # non-root # - - action_passthrough: + - name: "Find regular temp path" + action_passthrough: method: _make_tmp_path - register: out + register: tmp_path - - assert: - # This string must match ansible.cfg::remote_tmp - that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/") + - name: "Write some junk in regular temp path" + shell: hostname > {{tmp_path.result}}/hostname - - stat: - path: "{{out.result}}" - register: st + - name: "Verify junk did not persist across tasks" + stat: path={{tmp_path.result}}/hostname + register: junk_stat - - assert: - that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700" + - name: "Verify junk did not persist across tasks" + assert: + that: + - not junk_stat.stat.exists - - file: - path: "{{out.result}}" - state: absent + - name: "Verify temp path hasn't changed since start" + action_passthrough: + method: _make_tmp_path + register: tmp_path2 + + - name: "Verify temp path hasn't changed since start" + assert: + that: + - tmp_path2.result == tmp_path.result + + - name: "Verify temp path changes across connection reset" + mitogen_shutdown_all: + + - name: "Verify temp path changes across connection reset" + action_passthrough: + method: _make_tmp_path + register: tmp_path2 + + - name: "Verify temp path changes across connection reset" + assert: + that: + - tmp_path2.result != tmp_path.result + + - name: "Verify old path disappears across connection reset" + stat: path={{tmp_path.result}} + register: junk_stat + + - name: "Verify old path disappears across connection reset" + assert: + that: + - not junk_stat.stat.exists # - # become. make_tmp_path() must evaluate HOME in the context of the SSH - # user, not the become user. + # root # - - action_passthrough: + - name: "Find root temp path" + become: true + action_passthrough: method: _make_tmp_path - register: out + register: tmp_path_root + + - name: "Verify root temp path differs from regular path" + assert: + that: + - tmp_path2.result != tmp_path_root.result + + # + # readonly homedir + # + + - name: "Try writing to temp directory for the readonly_homedir user" become: true + become_user: mitogen__readonly_homedir + action_passthrough: + method: _make_tmp_path + register: tmp_path - - assert: - # This string must match ansible.cfg::remote_tmp - that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/") + - name: "Try writing to temp directory for the readonly_homedir user" + become: true + become_user: mitogen__readonly_homedir + shell: hostname > {{tmp_path.result}}/hostname - - stat: - path: "{{out.result}}" - register: st + # + # modules get the same temp dir + # + + - name: "Verify modules get the same tmpdir as the action plugin" + action_passthrough: + method: _make_tmp_path + register: tmp_path + + - name: "Verify modules get the same tmpdir as the action plugin" + custom_python_detect_environment: + register: out - - assert: - that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700" + # v2.6 related: https://github.com/ansible/ansible/pull/39833 + - name: "Verify modules get the same tmpdir as the action plugin (<2.5)" + when: ansible_version.full < '2.5' + assert: + that: + - out.module_path == tmp_path.result + - out.module_tmpdir == None - - file: - path: "{{out.result}}" - state: absent + - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" + when: ansible_version.full > '2.5' + assert: + that: + - out.module_path == tmp_path.result + - out.module_tmpdir == tmp_path.result diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index bf534aed..4550e203 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -8,11 +8,11 @@ - import_playbook: become/all.yml - import_playbook: connection_loader/all.yml - import_playbook: context_service/all.yml +- import_playbook: delegation/all.yml +- import_playbook: glibc_caches/all.yml - import_playbook: local/all.yml - import_playbook: module_utils/all.yml - import_playbook: playbook_semantics/all.yml -- import_playbook: remote_tmp/all.yml - import_playbook: runner/all.yml - import_playbook: ssh/all.yml - import_playbook: strategy/all.yml -- import_playbook: glibc_caches/all.yml diff --git a/tests/ansible/integration/delegation/all.yml b/tests/ansible/integration/delegation/all.yml new file mode 100644 index 00000000..30ea625f --- /dev/null +++ b/tests/ansible/integration/delegation/all.yml @@ -0,0 +1,2 @@ +- import_playbook: delegate_to_template.yml +- import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/delegation/delegate_to_template.yml b/tests/ansible/integration/delegation/delegate_to_template.yml new file mode 100644 index 00000000..8b1bd9af --- /dev/null +++ b/tests/ansible/integration/delegation/delegate_to_template.yml @@ -0,0 +1,41 @@ +# Ensure templated delegate_to field works. + +- name: integration/delegation/delegate_to_template.yml + vars: + physical_host: "cd-normal-alias" + physical_hosts: ["cd-normal-alias", "cd-normal-normal"] + hosts: test-targets + gather_facts: no + any_errors_fatal: true + tasks: + - mitogen_get_stack: + delegate_to: "{{ physical_host }}" + register: out + + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-alias', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml new file mode 100644 index 00000000..8ab6e51b --- /dev/null +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -0,0 +1,336 @@ +# https://github.com/dw/mitogen/issues/251 + +# ansible_mitogen.connection internally reinterprets Ansible state into a +# 'connection stack' -- this is just a list of dictionaries specifying a +# sequence of proxied Router connection methods and their kwargs used to +# establish the connection. That list is passed to ContextService, which loops +# over the stack specifying via=(None or previous entry) for each connection +# method. + +# mitogen_get_stack is a magic action that returns the stack, so we can test +# all kinds of scenarios without actually needing a real environmnt. + +# Updating this file? Install 'pprintpp' and hack lib/callbacks/nice_stdout.py +# to use it instead of the built-in function, then simply s/'/'/ to get the +# cutpasteable formatted dicts below. WARNING: remove the trailing comma from +# the result list element, it seems to cause assert to silently succeed! + + +- name: integration/delegation/stack_construction.yml + hosts: cd-normal + tasks: + # used later for local_action test. + - local_action: custom_python_detect_environment + register: local_env + + +- hosts: cd-normal + any_errors_fatal: true + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + "kwargs": { + "connect_timeout": 10, + "doas_path": None, + "password": None, + "python_path": ["/usr/bin/python"], + "username": "normal-user", + }, + "method": "doas", + } + ] + + +- hosts: cd-normal + tasks: + - mitogen_get_stack: + delegate_to: cd-alias + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-alias + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-normal-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-normal', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-normal-alias + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-alias', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-normal-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-newuser-normal-normal', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'newuser-normal-normal-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-normal-normal + tasks: + - mitogen_get_stack: + delegate_to: cd-alias + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-normal-normal + tasks: + - local_action: mitogen_get_stack + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'python_path': [ + hostvars['cd-normal'].local_env.sys_executable + ], + }, + 'method': 'local', + }, + ] + + +- hosts: cd-newuser-doas-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': ['/usr/bin/python'], + 'username': 'newuser-doas-normal-user', + }, + 'method': 'doas', + }, + ] diff --git a/tests/ansible/integration/remote_tmp/all.yml b/tests/ansible/integration/remote_tmp/all.yml deleted file mode 100644 index 5dff88d8..00000000 --- a/tests/ansible/integration/remote_tmp/all.yml +++ /dev/null @@ -1,2 +0,0 @@ - -- import_playbook: readonly_homedir.yml diff --git a/tests/ansible/integration/remote_tmp/readonly_homedir.yml b/tests/ansible/integration/remote_tmp/readonly_homedir.yml deleted file mode 100644 index ffad455a..00000000 --- a/tests/ansible/integration/remote_tmp/readonly_homedir.yml +++ /dev/null @@ -1,20 +0,0 @@ -# https://github.com/dw/mitogen/issues/239 -# While remote_tmp is used in the context of the SSH user by action code -# running on the controller, Ansiballz ignores it and uses the system default -# instead. - -- name: integration/remote_tmp/readonly_homedir.yml - hosts: test-targets - any_errors_fatal: true - tasks: - - custom_python_detect_environment: - become: true - become_user: mitogen__readonly_homedir - register: out - vars: - ansible_become_pass: readonly_homedir_password - - - name: Verify system temp directory was used. - assert: - that: - - out.__file__.startswith("/tmp/ansible_") diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 5242a405..ffb263fb 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -1,7 +1,7 @@ - import_playbook: builtin_command_module.yml +- import_playbook: custom_bash_hashbang_argument.yml - import_playbook: custom_bash_old_style_module.yml - import_playbook: custom_bash_want_json_module.yml -- import_playbook: custom_bash_hashbang_argument.yml - import_playbook: custom_binary_producing_json.yml - import_playbook: custom_binary_producing_junk.yml - import_playbook: custom_binary_single_null.yml @@ -13,4 +13,6 @@ - import_playbook: custom_python_want_json_module.yml - import_playbook: custom_script_interpreter.yml - import_playbook: environment_isolation.yml +- import_playbook: etc_environment.yml - import_playbook: forking_behaviour.yml +- import_playbook: missing_module.yml diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml new file mode 100644 index 00000000..0037698a --- /dev/null +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -0,0 +1,80 @@ +# issue #338: ensure /etc/environment is reloaded if it changes. +# Actually this test uses ~/.pam_environment, which is using the same logic, +# but less likely to brick a development workstation + +- name: integration/runner/etc_environment.yml + hosts: test-targets + any_errors_fatal: true + gather_facts: true + tasks: + # ~/.pam_environment + + - file: + path: ~/.pam_environment + state: absent + + - shell: echo $MAGIC_PAM_ENV + register: echo + + - assert: + that: echo.stdout == "" + + - copy: + dest: ~/.pam_environment + content: | + MAGIC_PAM_ENV=321 + + - shell: echo $MAGIC_PAM_ENV + register: echo + + - assert: + that: echo.stdout == "321" + + - file: + path: ~/.pam_environment + state: absent + + - shell: echo $MAGIC_PAM_ENV + register: echo + + - assert: + that: echo.stdout == "" + + + # /etc/environment + - meta: end_play + when: ansible_virtualization_type != "docker" + + - file: + path: /etc/environment + state: absent + become: true + + - shell: echo $MAGIC_ETC_ENV + register: echo + + - assert: + that: echo.stdout == "" + + - copy: + dest: /etc/environment + content: | + MAGIC_ETC_ENV=555 + become: true + + - shell: echo $MAGIC_ETC_ENV + register: echo + + - assert: + that: echo.stdout == "555" + + - file: + path: /etc/environment + state: absent + become: true + + - shell: echo $MAGIC_ETC_ENV + register: echo + + - assert: + that: echo.stdout == "" diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml new file mode 100644 index 00000000..064a9bf8 --- /dev/null +++ b/tests/ansible/integration/runner/missing_module.yml @@ -0,0 +1,19 @@ + +- name: integration/runner/missing_module.yml + hosts: test-targets + connection: local + tasks: + - connection: local + command: | + ansible -vvv + -i "{{inventory_file}}" + test-targets + -m missing_module + args: + chdir: ../.. + register: out + ignore_errors: true + + - assert: + that: | + 'The module missing_module was not found in configured module paths.' in out.stdout diff --git a/tests/ansible/lib/action/mitogen_get_stack.py b/tests/ansible/lib/action/mitogen_get_stack.py new file mode 100644 index 00000000..f1b87f35 --- /dev/null +++ b/tests/ansible/lib/action/mitogen_get_stack.py @@ -0,0 +1,22 @@ +""" +Fetch the connection configuration stack that would be used to connect to a +target, without actually connecting to it. +""" + +import ansible_mitogen.connection + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + if not isinstance(self._connection, + ansible_mitogen.connection.Connection): + return { + 'skipped': True, + } + + return { + 'changed': True, + 'result': self._connection._build_stack(), + } diff --git a/tests/ansible/lib/action/mitogen_shutdown_all.py b/tests/ansible/lib/action/mitogen_shutdown_all.py index 6ebdbf5c..4909dfe9 100644 --- a/tests/ansible/lib/action/mitogen_shutdown_all.py +++ b/tests/ansible/lib/action/mitogen_shutdown_all.py @@ -3,9 +3,6 @@ Arrange for all ContextService connections to be torn down unconditionally, required for reliable LRU tests. """ -import traceback -import sys - import ansible_mitogen.connection import ansible_mitogen.services import mitogen.service diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 8fe50bbc..2da9cddf 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -3,6 +3,7 @@ # interpreter I run within. from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import get_module_path from ansible.module_utils import six import os @@ -29,6 +30,8 @@ def main(): mitogen_loaded='mitogen.core' in sys.modules, hostname=socket.gethostname(), username=pwd.getpwuid(os.getuid()).pw_name, + module_tmpdir=getattr(module, 'tmpdir', None), + module_path=get_module_path(), ) if __name__ == '__main__': diff --git a/tests/ansible/osx_setup.yml b/tests/ansible/osx_setup.yml deleted file mode 100644 index 7a6ff23f..00000000 --- a/tests/ansible/osx_setup.yml +++ /dev/null @@ -1,155 +0,0 @@ - -# -# Add users expected by tests to an OS X machine. Assumes passwordless sudo to -# root. -# -# WARNING: this creates non-privilged accounts with pre-set passwords! -# - -- hosts: test-targets - gather_facts: true - become: true - tasks: - - name: Disable non-localhost SSH for Mitogen users - blockinfile: - path: /etc/ssh/sshd_config - block: | - Match User mitogen__* Address !127.0.0.1 - DenyUsers * - - # - # Hashed passwords. - # - - name: Create Mitogen test group - group: - name: "mitogen__group" - - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{ (item + '_password') | password_hash('sha256') }}" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - readonly_homedir - - require_tty_pw_required - - slow_user - when: ansible_system != 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "{{ ('user' + item + '_password') | password_hash('sha256') }}" - with_sequence: start=1 end=21 - when: ansible_system != 'Darwin' - - # - # Plaintext passwords - # - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{item}}_password" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - require_tty_pw_required - - readonly_homedir - - slow_user - when: ansible_system == 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "user{{item}}_password" - with_sequence: start=1 end=21 - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add '{{item}}' - with_items: - - mitogen__require_tty - - mitogen__pw_required - - mitogen__require_tty_pw_required - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add 'mitogen__user{{item}}' - with_sequence: start=1 end=21 - when: ansible_distribution == 'MacOSX' - - - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" - - - name: Slow bash profile for one account - copy: - dest: ~mitogen__slow_user/.{{item}} - src: ../data/docker/mitogen__slow_user.profile - with_items: - - bashrc - - profile - - - name: Install pubkey for one account - file: - path: ~mitogen__has_sudo_pubkey/.ssh - state: directory - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Install pubkey for one account - copy: - dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys - src: ../data/docker/mitogen__has_sudo_pubkey.key.pub - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Require a TTY for two accounts - lineinfile: - path: /etc/sudoers - line: "{{item}}" - with_items: - - Defaults>mitogen__pw_required targetpw - - Defaults>mitogen__require_tty requiretty - - Defaults>mitogen__require_tty_pw_required requiretty,targetpw - - - name: Require password for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" - with_items: - - mitogen__pw_required - - mitogen__require_tty_pw_required - - - name: Allow passwordless for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" - with_items: - - mitogen__require_tty - - mitogen__readonly_homedir - - - name: Allow passwordless for many accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__user{{item}}) NOPASSWD:ALL" - with_sequence: start=1 end=21 diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index ecb9638c..46798b3e 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -7,3 +7,4 @@ - import_playbook: issue_152__virtualenv_python_fails.yml - import_playbook: issue_154__module_state_leaks.yml - import_playbook: issue_177__copy_module_failing.yml +- import_playbook: issue_332_ansiblemoduleerror_first_occurrence.yml diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml new file mode 100644 index 00000000..0162c210 --- /dev/null +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -0,0 +1,14 @@ +# issue #332: Ansible 2.6 file.py started defining an excepthook and private +# AnsibleModuleError. Ensure file fails correctly. + +- name: regression/issue_332_ansiblemoduleerror_first_occurrence.yml + hosts: test-targets + tasks: + - file: path=/usr/bin/does-not-exist mode='a-s' state=file follow=yes + ignore_errors: true + register: out + + - assert: + that: + - out.state == 'absent' + - out.msg == 'file (/usr/bin/does-not-exist) is absent, cannot continue' diff --git a/tests/build_docker_images.py b/tests/build_docker_images.py deleted file mode 100755 index 7f856b2b..00000000 --- a/tests/build_docker_images.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -""" -Build the Docker images used for testing. -""" - -import commands -import os -import shlex -import subprocess -import tempfile - - -DEBIAN_DOCKERFILE = r""" -FROM debian:stretch -RUN apt-get update -RUN \ - apt-get install -y python2.7 openssh-server sudo rsync git strace \ - libjson-perl python-virtualenv && \ - apt-get clean && \ - rm -rf /var/cache/apt -""" - -CENTOS6_DOCKERFILE = r""" -FROM centos:6 -RUN yum clean all && \ - yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -CENTOS7_DOCKERFILE = r""" -FROM centos:7 -RUN yum clean all && \ - yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -DOCKERFILE = r""" -COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen -COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt -RUN \ - chsh -s /bin/bash && \ - mkdir -p /var/run/sshd && \ - echo i-am-mitogen-test-docker-image > /etc/sentinel && \ - echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \ - groupadd mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_nopw -G mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__webapp && \ - useradd -s /bin/bash -m mitogen__pw_required && \ - useradd -s /bin/bash -m mitogen__require_tty && \ - useradd -s /bin/bash -m mitogen__require_tty_pw_required && \ - useradd -s /bin/bash -m mitogen__readonly_homedir && \ - useradd -s /bin/bash -m mitogen__slow_user && \ - chown -R root: ~mitogen__readonly_homedir && \ - ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \ - ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \ - ( echo 'root:rootpassword' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_nopw:has_sudo_nopw_password' | chpasswd; ) && \ - ( echo 'mitogen__webapp:webapp_password' | chpasswd; ) && \ - ( echo 'mitogen__pw_required:pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \ - ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \ - mkdir ~mitogen__has_sudo_pubkey/.ssh && \ - ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; ) - -COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc - -RUN \ - chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \ - chmod -R go= ~mitogen__has_sudo_pubkey - -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -EXPOSE 22 -CMD ["/usr/sbin/sshd", "-D"] -""" - - -def sh(s, *args): - if args: - s %= tuple(map(commands.mkarg, args)) - return shlex.split(s) - - -for (distro, wheel, prefix) in ( - ('debian', 'sudo', DEBIAN_DOCKERFILE), - ('centos6', 'wheel', CENTOS6_DOCKERFILE), - ('centos7', 'wheel', CENTOS7_DOCKERFILE), - ): - mydir = os.path.abspath(os.path.dirname(__file__)) - with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp: - dockerfile_fp.write(prefix) - dockerfile_fp.write(DOCKERFILE.replace('SUDO_GROUP', wheel)) - dockerfile_fp.flush() - - subprocess.check_call(sh('docker build %s -t %s -f %s', - mydir, - 'mitogen/%s-test' % (distro,), - dockerfile_fp.name - )) diff --git a/tests/data/001-mitogen.sudo b/tests/data/docker/001-mitogen.sudo similarity index 100% rename from tests/data/001-mitogen.sudo rename to tests/data/docker/001-mitogen.sudo diff --git a/tests/data/fake_lxc.py b/tests/data/fake_lxc.py new file mode 100755 index 00000000..2fedb961 --- /dev/null +++ b/tests/data/fake_lxc.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/data/fake_lxc_attach.py b/tests/data/fake_lxc_attach.py new file mode 100755 index 00000000..2fedb961 --- /dev/null +++ b/tests/data/fake_lxc_attach.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/image_prep/README.md b/tests/image_prep/README.md new file mode 100644 index 00000000..d275672f --- /dev/null +++ b/tests/image_prep/README.md @@ -0,0 +1,25 @@ + +# `image_prep` + +This directory contains Ansible playbooks for building the Docker containers +used for testing, or for setting up an OS X laptop so the tests can (mostly) +run locally. + +The Docker config is more heavily jinxed to trigger adverse conditions in the +code, the OS X config just has the user accounts. + +See ../README.md for a (mostly) description of the accounts created. + + +## Building the containers + +``./build_docker_images.sh`` + + +## Preparing an OS X box + +WARNING: this creates a ton of accounts with preconfigured passwords. It is +generally impossible to restrict remote access to these, so your only option is +to disable remote login and sharing. + +``ansible-playbook -b -c local -i localhost, -l localhost setup.yml`` diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml new file mode 100644 index 00000000..db0d3789 --- /dev/null +++ b/tests/image_prep/_container_setup.yml @@ -0,0 +1,117 @@ + +- hosts: all + strategy: linear + gather_facts: false + tasks: + - raw: > + if ! python -c ''; then + if type -p yum; then + yum -y install python; + else + apt-get -y update && apt-get -y install python; + fi; + fi + +- hosts: all + strategy: mitogen_free + # Can't gather facts before here. + gather_facts: true + vars: + distro: "{{ansible_distribution}}" + ver: "{{ansible_distribution_major_version}}" + + packages: + common: + - git + - openssh-server + - rsync + - strace + - sudo + Debian: + "9": + - libjson-perl + - python-virtualenv + CentOS: + "6": + - perl-JSON + "7": + - perl-JSON + - python-virtualenv + + tasks: + - when: ansible_virtualization_type != "docker" + meta: end_play + + - apt: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "Debian" + + - yum: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "CentOS" + + - command: apt-get clean + when: distro == "Debian" + + - command: yum clean all + when: distro == "CentOS" + + - shell: rm -rf {{item}}/* + with_items: + - /var/cache/apt + - /var/lib/apt/lists + when: distro == "Debian" + + - user: + name: root + password: "{{ 'rootpassword' | password_hash('sha256') }}" + shell: /bin/bash + + - file: + path: /var/run/sshd + state: directory + + - command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key + args: + creates: /etc/ssh/ssh_host_rsa_key + + - group: + name: "{{sudo_group[distro]}}" + + - copy: + dest: /etc/sentinel + content: | + i-am-mitogen-test-docker-image + + - copy: + dest: /etc/ssh/banner.txt + src: ../data/docker/ssh_login_banner.txt + + - copy: + dest: /etc/sudoers.d/001-mitogen + src: ../data/docker/001-mitogen.sudo + + - lineinfile: + path: /etc/ssh/sshd_config + line: Banner /etc/ssh/banner.txt + + - lineinfile: + path: /etc/ssh/sshd_config + line: PermitRootLogin yes + regexp: '.*PermitRootLogin.*' + + - lineinfile: + path: /etc/pam.d/sshd + regexp: '.*session.*required.*pam_loginuid.so' + line: session optional pam_loginuid.so + + - copy: + mode: 'u+rwx,go=rx' + dest: /usr/local/bin/pywrap + content: | + #!/bin/bash + exec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml new file mode 100644 index 00000000..f9cac85c --- /dev/null +++ b/tests/image_prep/_user_accounts.yml @@ -0,0 +1,152 @@ +# +# Add users expected by tests. Assumes passwordless sudo to root. +# +# WARNING: this creates non-privilged accounts with pre-set passwords! +# + +- hosts: all + gather_facts: true + strategy: mitogen_free + become: true + vars: + distro: "{{ansible_distribution}}" + ver: "{{ansible_distribution_major_version}}" + + special_users: + - has_sudo + - has_sudo_nopw + - has_sudo_pubkey + - pw_required + - readonly_homedir + - require_tty + - require_tty_pw_required + - slow_user + - webapp + - sudo1 + - sudo2 + - sudo3 + - sudo4 + + user_groups: + has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] + has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] + has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] + sudo2: ['mitogen__group', '{{sudo_group[distro]}}'] + sudo3: ['mitogen__group', '{{sudo_group[distro]}}'] + sudo4: ['mitogen__group', '{{sudo_group[distro]}}'] + + normal_users: "{{ + lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) + }}" + + all_users: "{{ + special_users + + normal_users + }}" + tasks: + - name: Disable non-localhost SSH for Mitogen users + when: false + blockinfile: + path: /etc/ssh/sshd_config + block: | + Match User mitogen__* Address !127.0.0.1 + DenyUsers * + + - name: Create Mitogen test groups + group: + name: "mitogen__{{item}}" + with_items: + - group + - sudo_nopw + + - name: Create user accounts + block: + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{user_groups[item]|default(['mitogen__group'])}}" + password: "{{ (item + '_password') | password_hash('sha256') }}" + loop: "{{all_users}}" + when: ansible_system != 'Darwin' + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{user_groups[item]|default(['mitogen__group'])}}" + password: "{{item}}_password" + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + + - name: Hide users from login window. + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + osx_defaults: + array_add: true + domain: /Library/Preferences/com.apple.loginwindow + type: array + key: HiddenUsersList + value: ['mitogen_{{item}}'] + + - name: Readonly homedir for one account + shell: "chown -R root: ~mitogen__readonly_homedir" + + - name: Slow bash profile for one account + copy: + dest: ~mitogen__slow_user/.{{item}} + src: ../data/docker/mitogen__slow_user.profile + with_items: + - bashrc + - profile + + - name: Install pubkey for mitogen__has_sudo_pubkey + block: + - file: + path: ~mitogen__has_sudo_pubkey/.ssh + state: directory + mode: go= + owner: mitogen__has_sudo_pubkey + - copy: + dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys + src: ../data/docker/mitogen__has_sudo_pubkey.key.pub + mode: go= + owner: mitogen__has_sudo_pubkey + + - name: Install slow profile for one account + block: + - copy: + dest: ~mitogen__slow_user/.profile + src: ../data/docker/mitogen__slow_user.profile + - copy: + dest: ~mitogen__slow_user/.bashrc + src: ../data/docker/mitogen__slow_user.profile + + - name: Require a TTY for two accounts + lineinfile: + path: /etc/sudoers + line: "{{item}}" + with_items: + - Defaults>mitogen__pw_required targetpw + - Defaults>mitogen__require_tty requiretty + - Defaults>mitogen__require_tty_pw_required requiretty,targetpw + + - name: Require password for two accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" + with_items: + - mitogen__pw_required + - mitogen__require_tty_pw_required + + - name: Allow passwordless sudo for require_tty/readonly_homedir + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" + with_items: + - mitogen__require_tty + - mitogen__readonly_homedir + + - name: Allow passwordless for many accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}) NOPASSWD:ALL" + loop: "{{normal_users}}" diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg new file mode 100644 index 00000000..a3937825 --- /dev/null +++ b/tests/image_prep/ansible.cfg @@ -0,0 +1,4 @@ + +[defaults] +strategy_plugins = ../../ansible_mitogen/plugins/strategy +retry_files_enabled = false diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py new file mode 100755 index 00000000..c085d29e --- /dev/null +++ b/tests/image_prep/build_docker_images.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +""" +Build the Docker images used for testing. +""" + +import commands +import os +import tempfile +import shlex +import subprocess + + +BASEDIR = os.path.dirname(os.path.abspath(__file__)) + + +def sh(s, *args): + if args: + s %= args + return shlex.split(s) + + + +label_by_id = {} + +for base_image, label in [ + ('debian:stretch', 'debian'), + ('centos:6', 'centos6'), + ('centos:7', 'centos7') + ]: + args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash', + label, base_image) + container_id = subprocess.check_output(args).strip() + label_by_id[container_id] = label + +with tempfile.NamedTemporaryFile() as fp: + fp.write('[all]\n') + for id_, label in label_by_id.items(): + fp.write('%s ansible_host=%s\n' % (label, id_)) + fp.flush() + + try: + subprocess.check_call( + cwd=BASEDIR, + args=sh('ansible-playbook -i %s -c docker setup.yml', fp.name), + ) + + for container_id, label in label_by_id.items(): + subprocess.check_call(sh(''' + docker commit + --change 'EXPOSE 22' + --change 'CMD ["/usr/sbin/sshd", "-D"]' + %s + mitogen/%s-test + ''', container_id, label)) + finally: + subprocess.check_call(sh('docker rm -f %s', ' '.join(label_by_id))) diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml new file mode 100644 index 00000000..7a589239 --- /dev/null +++ b/tests/image_prep/setup.yml @@ -0,0 +1,13 @@ + +- hosts: all + gather_facts: false + tasks: + - set_fact: + # Hacktacular.. but easiest place for it with current structure. + sudo_group: + MacOSX: admin + Debian: sudo + CentOS: wheel + +- import_playbook: _container_setup.yml +- import_playbook: _user_accounts.yml diff --git a/tests/lxc_test.py b/tests/lxc_test.py new file mode 100644 index 00000000..a30cd186 --- /dev/null +++ b/tests/lxc_test.py @@ -0,0 +1,29 @@ +import os + +import mitogen + +import unittest2 + +import testlib + + +def has_subseq(seq, subseq): + return any(seq[x:x+len(subseq)] == subseq for x in range(0, len(seq))) + + +class FakeLxcAttachTest(testlib.RouterMixin, unittest2.TestCase): + def test_okay(self): + lxc_attach_path = testlib.data_path('fake_lxc_attach.py') + context = self.router.lxc( + container='container_name', + lxc_attach_path=lxc_attach_path, + ) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[0], lxc_attach_path) + self.assertTrue('--clear-env' in argv) + self.assertTrue(has_subseq(argv, ['--name', 'container_name'])) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/lxd_test.py b/tests/lxd_test.py new file mode 100644 index 00000000..c5e4c485 --- /dev/null +++ b/tests/lxd_test.py @@ -0,0 +1,26 @@ +import os + +import mitogen + +import unittest2 + +import testlib + + +class FakeLxcTest(testlib.RouterMixin, unittest2.TestCase): + def test_okay(self): + lxc_path = testlib.data_path('fake_lxc.py') + context = self.router.lxd( + container='container_name', + lxc_path=lxc_path, + ) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[0], lxc_path) + self.assertEquals(argv[1], 'exec') + self.assertEquals(argv[2], '--force-noninteractive') + self.assertEquals(argv[3], 'container_name') + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/parent_test.py b/tests/parent_test.py index 06eac97e..0b8b5e9a 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -5,6 +5,7 @@ import sys import tempfile import time +import mock import unittest2 import testlib @@ -28,6 +29,21 @@ def wait_for_child(pid, timeout=1.0): assert False, "wait_for_child() timed out" +class GetDefaultRemoteNameTest(testlib.TestCase): + func = staticmethod(mitogen.parent.get_default_remote_name) + + @mock.patch('os.getpid') + @mock.patch('getpass.getuser') + @mock.patch('socket.gethostname') + def test_slashes(self, mock_gethostname, mock_getuser, mock_getpid): + # Ensure slashes appearing in the remote name are replaced with + # underscores. + mock_gethostname.return_value = 'box' + mock_getuser.return_value = 'ECORP\\Administrator' + mock_getpid.return_value = 123 + self.assertEquals("ECORP_Administrator@box:123", self.func()) + + class ReapChildTest(testlib.RouterMixin, testlib.TestCase): def test_connect_timeout(self): # Ensure the child process is reaped if the connection times out.