diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index f1dc425c..c25697c6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,6 +4,10 @@ Please drag-drop large logs as text file attachments. Feel free to write an issue in your preferred format, however if in doubt, use the following checklist as a guide for what to include. +* Which version of Ansible are you running? +* Is your version of Ansible patched in any way? +* Are you running with any custom modules, or `module_utils` loaded? + * Have you tried the latest master version from Git? * Do you have some idea of what the underlying problem may be? https://mitogen.rtfd.io/en/stable/ansible.html#common-problems has diff --git a/LICENSE b/LICENSE index 61d21fee..70e43a94 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2017, David Wilson +Copyright 2019, David Wilson Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..1aba38f6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index d7ae45a6..57926516 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -156,11 +156,11 @@ class Policy(object): Assign the helper subprocess policy to this process. """ - -class LinuxPolicy(Policy): +class FixedPolicy(Policy): """ - :class:`Policy` for Linux machines. The scheme here was tested on an - otherwise idle 16 thread machine. + :class:`Policy` for machines where the only control method available is + fixed CPU placement. The scheme here was tested on an otherwise idle 16 + thread machine. - The connection multiplexer is pinned to CPU 0. - The Ansible top-level (strategy) is pinned to CPU 1. @@ -180,26 +180,35 @@ class LinuxPolicy(Policy): CPU-intensive children like SSH are not forced to share the same core as the (otherwise potentially very busy) parent. """ - def __init__(self): + def __init__(self, cpu_count=None): + #: For tests. + self.cpu_count = cpu_count or multiprocessing.cpu_count() self.mem = mmap.mmap(-1, 4096) self.state = State.from_buffer(self.mem) self.state.lock.init() - if self._cpu_count() < 4: - self._reserve_mask = 3 - self._reserve_shift = 2 - self._reserve_controller = True - else: + + if self.cpu_count < 2: + # uniprocessor + self._reserve_mux = False + self._reserve_controller = False + self._reserve_mask = 0 + self._reserve_shift = 0 + elif self.cpu_count < 4: + # small SMP + self._reserve_mux = True + self._reserve_controller = False self._reserve_mask = 1 self._reserve_shift = 1 - self._reserve_controller = False + else: + # big SMP + self._reserve_mux = True + self._reserve_controller = True + self._reserve_mask = 3 + self._reserve_shift = 2 def _set_affinity(self, mask): mitogen.parent._preexec_hook = self._clear - s = struct.pack('L', mask) - _sched_setaffinity(os.getpid(), len(s), s) - - def _cpu_count(self): - return multiprocessing.cpu_count() + self._set_cpu_mask(mask) def _balance(self): self.state.lock.acquire() @@ -210,14 +219,15 @@ class LinuxPolicy(Policy): self.state.lock.release() self._set_cpu(self._reserve_shift + ( - (n % max(1, (self._cpu_count() - self._reserve_shift))) + (n % (self.cpu_count - self._reserve_shift)) )) def _set_cpu(self, cpu): self._set_affinity(1 << cpu) def _clear(self): - self._set_affinity(0xffffffff & ~self._reserve_mask) + all_cpus = (1 << self.cpu_count) - 1 + self._set_affinity(all_cpus & ~self._reserve_mask) def assign_controller(self): if self._reserve_controller: @@ -235,6 +245,12 @@ class LinuxPolicy(Policy): self._clear() +class LinuxPolicy(FixedPolicy): + def _set_cpu_mask(self, mask): + s = struct.pack('L', mask) + _sched_setaffinity(os.getpid(), len(s), s) + + if _sched_setaffinity is not None: policy = LinuxPolicy() else: diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c7e70c43..254a4286 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -547,7 +547,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): def connected(self): return self.context is not None - def _spec_from_via(self, via_spec): + def _spec_from_via(self, proxied_inventory_name, via_spec): """ Produce a dict connection specifiction given a string `via_spec`, of the form `[[become_method:]become_user@]inventory_hostname`. @@ -555,17 +555,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): become_user, _, inventory_name = via_spec.rpartition('@') become_method, _, become_user = become_user.rpartition(':') - via_vars = self.host_vars[inventory_name] - if isinstance(via_vars, jinja2.runtime.Undefined): + # must use __contains__ to avoid a TypeError for a missing host on + # Ansible 2.3. + if self.host_vars is None or inventory_name not in self.host_vars: raise ansible.errors.AnsibleConnectionFailure( self.unknown_via_msg % ( via_spec, - inventory_name, + proxied_inventory_name, ) ) + via_vars = self.host_vars[inventory_name] return ansible_mitogen.transport_config.MitogenViaSpec( inventory_name=inventory_name, + play_context=self._play_context, host_vars=dict(via_vars), # TODO: make it lazy become_method=become_method or None, become_user=become_user or None, @@ -615,7 +618,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): if spec.mitogen_via(): stack = self._stack_from_spec( - self._spec_from_via(spec.mitogen_via()), + self._spec_from_via(spec.inventory_name(), spec.mitogen_via()), stack=stack, seen_names=seen_names + (spec.inventory_name(),), ) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 08c59278..ff06c0c5 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 97832938..1c439be8 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -54,7 +54,8 @@ class Handler(logging.Handler): #: may simply be to bury all target logs in DEBUG output, but not by #: overriding their log level as done here. NOISY_LOGGERS = frozenset([ - 'dnf', # issue #272; warns when a package is already installed. + 'dnf', # issue #272; warns when a package is already installed. + 'boto', # issue #541; normal boto retry logic can cause ERROR logs. ]) def emit(self, record): @@ -75,25 +76,28 @@ class Handler(logging.Handler): def setup(): """ - Install a handler for Mitogen's logger to redirect it into the Ansible - display framework, and prevent propagation to the root logger. + Install handlers for Mitogen loggers to redirect them into the Ansible + display framework. Ansible installs its own logging framework handlers when + C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers. """ - logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)] - mitogen.core.LOG.handlers = [Handler(display.vvv)] - mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] - mitogen.core.IOLOG.propagate = False + l_mitogen = logging.getLogger('mitogen') + l_mitogen_io = logging.getLogger('mitogen.io') + l_ansible_mitogen = logging.getLogger('ansible_mitogen') + + for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen: + logger.handlers = [Handler(display.vvv)] + logger.propagate = False if display.verbosity > 2: - mitogen.core.LOG.setLevel(logging.DEBUG) - logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) + l_ansible_mitogen.setLevel(logging.DEBUG) + l_mitogen.setLevel(logging.DEBUG) else: # Mitogen copies the active log level into new children, allowing them # to filter tiny messages before they hit the network, and therefore # before they wake the IO loop. Explicitly setting INFO saves ~4% # running against just the local machine. - mitogen.core.LOG.setLevel(logging.ERROR) - logging.getLogger('ansible_mitogen').setLevel(logging.ERROR) + l_mitogen.setLevel(logging.ERROR) + l_ansible_mitogen.setLevel(logging.ERROR) if display.verbosity > 3: - mitogen.core.IOLOG.setLevel(logging.DEBUG) - logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) + l_mitogen_io.setLevel(logging.DEBUG) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 7a180952..5f51cc6f 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 56e8b82e..633e3cad 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/parsing.py b/ansible_mitogen/parsing.py index fa79282a..525e60cf 100644 --- a/ansible_mitogen/parsing.py +++ b/ansible_mitogen/parsing.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index f3e4500e..3c5bd64f 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/action/mitogen_get_stack.py b/ansible_mitogen/plugins/action/mitogen_get_stack.py index ed7520cf..12afbfba 100644 --- a/ansible_mitogen/plugins/action/mitogen_get_stack.py +++ b/ansible_mitogen/plugins/action/mitogen_get_stack.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_doas.py b/ansible_mitogen/plugins/connection/mitogen_doas.py index 873b0d9d..1113d7c6 100644 --- a/ansible_mitogen/plugins/connection/mitogen_doas.py +++ b/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index 5904c83e..b71ef5f1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index fb7bce54..c7475fb1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index fcd9c030..24b84a03 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index ce394102..696c9abd 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 77efe6c1..95e692a0 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index 9b332a3f..0f5a0d28 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 23f62135..20c6f137 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index dbaba407..df0e87cb 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_su.py b/ansible_mitogen/plugins/connection/mitogen_su.py index 104a7190..4ab2711e 100644 --- a/ansible_mitogen/plugins/connection/mitogen_su.py +++ b/ansible_mitogen/plugins/connection/mitogen_su.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py index 367dd61b..130f5445 100644 --- a/ansible_mitogen/plugins/connection/mitogen_sudo.py +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index f8608745..66872663 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index d3b1cdc6..ffe2fbd9 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py index 175e1f8b..23eccd36 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py +++ b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index 51b03096..1b198e61 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 8137af9c..d7f36496 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -236,6 +236,41 @@ class MuxProcess(object): if secs: mitogen.debug.dump_to_logger(secs=secs) + def _setup_simplejson(self, responder): + """ + We support serving simplejson for Python 2.4 targets on Ansible 2.3, at + least so the package's own CI Docker scripts can run without external + help, however newer versions of simplejson no longer support Python + 2.4. Therefore override any installed/loaded version with a + 2.4-compatible version we ship in the compat/ directory. + """ + responder.whitelist_prefix('simplejson') + + # issue #536: must be at end of sys.path, in case existing newer + # version is already loaded. + compat_path = os.path.join(os.path.dirname(__file__), 'compat') + sys.path.append(compat_path) + + for fullname, is_pkg, suffix in ( + (u'simplejson', True, '__init__.py'), + (u'simplejson.decoder', False, 'decoder.py'), + (u'simplejson.encoder', False, 'encoder.py'), + (u'simplejson.scanner', False, 'scanner.py'), + ): + path = os.path.join(compat_path, 'simplejson', suffix) + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + responder.add_source_override( + fullname=fullname, + path=path, + source=source, + is_pkg=is_pkg, + ) + def _setup_responder(self, responder): """ Configure :class:`mitogen.master.ModuleResponder` to only permit @@ -243,9 +278,7 @@ class MuxProcess(object): """ responder.whitelist_prefix('ansible') responder.whitelist_prefix('ansible_mitogen') - responder.whitelist_prefix('simplejson') - simplejson_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.insert(0, simplejson_path) + self._setup_simplejson(responder) # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 768cc57c..04c70e78 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 61286382..a7c0e46f 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 4d1636e2..50486841 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 01877e34..809165da 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -377,6 +377,11 @@ def init_child(econtext, log_level, candidate_temp_dirs): LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) + # issue #536: if the json module is available, remove simplejson from the + # importer whitelist to avoid confusing certain Ansible modules. + if json.__name__ == 'json': + econtext.importer.whitelist.remove('simplejson') + global _fork_parent if FORK_SUPPORTED: mitogen.parent.upgrade_router(econtext) @@ -497,7 +502,7 @@ class AsyncRunner(object): ) result = json.loads(filtered) result.setdefault('warnings', []).extend(warnings) - result['stderr'] = dct['stderr'] + result['stderr'] = dct['stderr'] or result.get('stderr', '') self._update(result) def _run(self): diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 290c12d5..8ef12165 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -329,9 +329,11 @@ class PlayContextSpec(Spec): return self._play_context.port def python_path(self): - return parse_python_path( - self._connection.get_task_var('ansible_python_interpreter') - ) + s = self._connection.get_task_var('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') def private_key_file(self): return self._play_context.private_key_file @@ -428,12 +430,33 @@ class MitogenViaSpec(Spec): having a configruation problem with connection delegation, the answer to your problem lies in the method implementations below! """ - def __init__(self, inventory_name, host_vars, - become_method, become_user): + def __init__(self, inventory_name, host_vars, become_method, become_user, + play_context): + """ + :param str inventory_name: + The inventory name of the intermediary machine, i.e. not the target + machine. + :param dict host_vars: + The HostVars magic dictionary provided by Ansible in task_vars. + :param str become_method: + If the mitogen_via= spec included a become method, the method it + specifies. + :param str become_user: + If the mitogen_via= spec included a become user, the user it + specifies. + :param PlayContext play_context: + For some global values **only**, the PlayContext used to describe + the real target machine. Values from this object are **strictly + restricted** to values that are Ansible-global, e.g. the passwords + specified interactively. + """ self._inventory_name = inventory_name self._host_vars = host_vars self._become_method = become_method self._become_user = become_user + # Dangerous! You may find a variable you want in this object, but it's + # almost certainly for the wrong machine! + self._dangerous_play_context = play_context def transport(self): return ( @@ -445,15 +468,17 @@ class MitogenViaSpec(Spec): return self._inventory_name def remote_addr(self): + # play_context.py::MAGIC_VARIABLE_MAPPING return ( + self._host_vars.get('ansible_ssh_host') or self._host_vars.get('ansible_host') or self._inventory_name ) def remote_user(self): return ( - self._host_vars.get('ansible_user') or self._host_vars.get('ansible_ssh_user') or + self._host_vars.get('ansible_user') or C.DEFAULT_REMOTE_USER ) @@ -461,37 +486,40 @@ class MitogenViaSpec(Spec): return bool(self._become_user) def become_method(self): - return self._become_method or C.DEFAULT_BECOME_METHOD + return ( + self._become_method or + self._host_vars.get('ansible_become_method') or + C.DEFAULT_BECOME_METHOD + ) def become_user(self): return self._become_user def become_pass(self): return optional_secret( - # TODO: Might have to come from PlayContext. self._host_vars.get('ansible_become_password') or self._host_vars.get('ansible_become_pass') ) def password(self): return optional_secret( - # TODO: Might have to come from PlayContext. self._host_vars.get('ansible_ssh_pass') or self._host_vars.get('ansible_password') ) def port(self): return ( + self._host_vars.get('ansible_ssh_port') or self._host_vars.get('ansible_port') or C.DEFAULT_REMOTE_PORT ) def python_path(self): - return parse_python_path( - self._host_vars.get('ansible_python_interpreter') - # This variable has no default for remote hosts. For local hosts it - # is sys.executable. - ) + s = self._host_vars.get('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') def private_key_file(self): # TODO: must come from PlayContext too. diff --git a/docs/_static/style.css b/docs/_static/style.css index ec25901f..456473a9 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -1,4 +1,70 @@ +body { + font-size: 100%; +} + +.sphinxsidebarwrapper { + padding-top: 0 !important; +} + +.sphinxsidebar { + font-size: 80% !important; +} + +.sphinxsidebar h3 { + font-size: 130% !important; +} + +img + p, +h1 + p, +h2 + p, +h3 + p, +h4 + p, +h5 + p +{ + margin-top: 0; +} + +.section > h3:first-child { + margin-top: 15px !important; +} + +.body h1 { font-size: 200% !important; } +.body h2 { font-size: 165% !important; } +.body h3 { font-size: 125% !important; } +.body h4 { font-size: 110% !important; font-weight: bold; } +.body h5 { font-size: 100% !important; font-weight: bold; } + +.body h1, +.body h2, +.body h3, +.body h4, +.body h5 { + margin-top: 30px !important; + color: #7f0000; +} + +.body h1 { + margin-top: 0 !important; +} + +body, +.sphinxsidebar, +.sphinxsidebar h1, +.sphinxsidebar h2, +.sphinxsidebar h3, +.sphinxsidebar h4, +.sphinxsidebar h5, +.body h1, +.body h2, +.body h3, +.body h4, +.body h5 { + /*font-family: sans-serif !important;*/ + font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol !important; +} + + .document { width: 1000px !important; } @@ -38,10 +104,10 @@ div.body p, div.body dd, div.body li, div.body blockquote { width: 150px; } -.mitogen-right-200 { +.mitogen-right-180 { float: right; padding-left: 8px; - width: 200px; + width: 180px; } .mitogen-right-225 { diff --git a/docs/_templates/github.html b/docs/_templates/github.html index 12533b52..bb2b5ee5 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,8 +1,4 @@


-Star -

- -

-GitHub Repository +Star

diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html new file mode 100644 index 00000000..76accef7 --- /dev/null +++ b/docs/_templates/globaltoc.html @@ -0,0 +1 @@ +{{ toctree() }} diff --git a/docs/ansible.rst b/docs/ansible.rst index 17354755..f7788011 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -3,7 +3,7 @@ Mitogen for Ansible =================== .. image:: images/ansible/ansible_mitogen.svg - :class: mitogen-right-200 mitogen-logo-wrap + :class: mitogen-right-180 mitogen-logo-wrap An extension to `Ansible`_ is included that implements connections over Mitogen, replacing embedded shell invocations with pure-Python equivalents @@ -246,6 +246,11 @@ container. as duplicate connections between hops, due to not perfectly replicating the configuration Ansible would normally use for the intermediary. + * Intermediary machines cannot use login and become passwords that were + supplied to Ansible interactively. If an intermediary requires a + password, it must be supplied via ``ansible_ssh_pass``, + ``ansible_password``, or ``ansible_become_pass`` inventory variables. + * Automatic tunnelling of SSH-dependent actions, such as the ``synchronize`` module, is not yet supported. This will be added in the 0.3 series. diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d1d9079..f7ecadbd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Release Notes @@ -125,6 +125,67 @@ Core Library series. +v0.2.5 (2019-02-14) +------------------- + +Fixes +~~~~~ + +* `#511 `_, + `#536 `_: changes in 0.2.4 to + repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` + handling. Test coverage was added. + +* `#532 `_: fix a race in the service + used to propagate Ansible modules, that could easily manifest when starting + asynchronous tasks in a loop. + +* `#536 `_: changes in 0.2.4 to + support Python 2.4 interacted poorly with modules that imported + ``simplejson`` from a controller that also loaded an incompatible newer + version of ``simplejson``. + +* `#537 `_: a swapped operator in the + CPU affinity logic meant 2 cores were reserved on 1`_: the source distribution + includes a ``LICENSE`` file. + +* `#539 `_: log output is no longer + duplicated when the Ansible ``log_path`` setting is enabled. + +* `#540 `_: the ``stderr`` stream of + async module invocations was previously discarded. + +* `#541 `_: Python error logs + originating from the ``boto`` package are quiesced, and only appear in + ``-vvv`` output. This is since EC2 modules may trigger errors during normal + operation, when retrying transiently failing requests. + +* `748f5f67 `_, + `21ad299d `_, + `8ae6ca1d `_, + `7fd0d349 `_: + the ``ansible_ssh_host``, ``ansible_ssh_user``, ``ansible_user``, + ``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly + match typical behaviour when ``mitogen_via=`` is active. + +* `2a8567b4 `_: fix a race + initializing a child's service thread pool on Python 3.4+, due to a change in + locking scheme used by the Python import mechanism. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +bug reports, testing, features and fixes in this release contributed by +`Carl George `_, +`Guy Knights `_, and +`Josh Smift `_. + + v0.2.4 (2019-02-10) ------------------- diff --git a/docs/conf.py b/docs/conf.py index abb6e97e..3708a943 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,6 +17,10 @@ html_theme = 'alabaster' html_theme_options = { 'font_family': "Georgia, serif", 'head_font_family': "Georgia, serif", + 'fixed_sidebar': True, + 'show_powered_by': False, + 'pink_2': 'fffafaf', + 'pink_1': '#fff0f0', } htmlhelp_basename = 'mitogendoc' intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/docs/index.rst b/docs/index.rst index 066d6716..6b5deb71 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,7 @@ Mitogen .. image:: images/mitogen.svg - :class: mitogen-right-200 mitogen-logo-wrap + :class: mitogen-right-180 mitogen-logo-wrap Mitogen is a Python library for writing distributed self-replicating programs. diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 26e48aff..08f875d4 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 4) +__version__ = (0, 2, 5) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index a48e13ed..470b00ca 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -1125,6 +1125,11 @@ class Importer(object): self.whitelist = list(whitelist) or [''] self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST + # Preserve copies of the original server-supplied whitelist/blacklist + # for later use by children. + self.master_whitelist = self.whitelist[:] + self.master_blacklist = self.blacklist[:] + # Presence of an entry in this map indicates in-flight GET_MODULE. self._callbacks = {} self._cache = {} @@ -3131,10 +3136,21 @@ class ExternalContext(object): if not self.config['profiling']: os.kill(os.getpid(), signal.SIGTERM) + #: On Python >3.4, the global importer lock has been sharded into a + #: per-module lock, meaning there is no guarantee the import statement in + #: service_stub_main will be truly complete before a second thread + #: attempting the same import will see a partially initialized module. + #: Sigh. Therefore serialize execution of the stub itself. + service_stub_lock = threading.Lock() + def _service_stub_main(self, msg): - import mitogen.service - pool = mitogen.service.get_or_create_pool(router=self.router) - pool._receiver._on_receive(msg) + self.service_stub_lock.acquire() + try: + import mitogen.service + pool = mitogen.service.get_or_create_pool(router=self.router) + pool._receiver._on_receive(msg) + finally: + self.service_stub_lock.release() def _on_call_service_msg(self, msg): """ diff --git a/mitogen/debug.py b/mitogen/debug.py index 8f290c4d..3d13347f 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/doas.py b/mitogen/doas.py index 250b6faf..1b687fb2 100644 --- a/mitogen/doas.py +++ b/mitogen/doas.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/docker.py b/mitogen/docker.py index 074f0e90..0c0d40e7 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index 2f2726eb..d39a710d 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/fork.py b/mitogen/fork.py index 081f7e3d..d6685d70 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -98,6 +98,7 @@ def on_fork(): fixup_prngs() mitogen.core.Latch._on_fork() mitogen.core.Side._on_fork() + mitogen.core.ExternalContext.service_stub_lock = threading.Lock() mitogen__service = sys.modules.get('mitogen.service') if mitogen__service: diff --git a/mitogen/jail.py b/mitogen/jail.py index fade8cbb..6e0ac68b 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 6d4acba6..879d19a1 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/lxd.py b/mitogen/lxd.py index 7de4903a..faea2561 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/master.py b/mitogen/master.py index 257fb81b..1396f4e1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -485,8 +485,10 @@ class ModuleFinder(object): return path, source, is_pkg def _get_module_via_sys_modules(self, fullname): - """Attempt to fetch source code via sys.modules. This is specifically - to support __main__, but it may catch a few more cases.""" + """ + Attempt to fetch source code via sys.modules. This is specifically to + support __main__, but it may catch a few more cases. + """ module = sys.modules.get(fullname) LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if not isinstance(module, types.ModuleType): @@ -883,10 +885,13 @@ class ModuleResponder(object): if msg.is_dead: return - LOG.debug('%r._on_get_module(%r)', self, msg.data) - self.get_module_count += 1 stream = self._router.stream_by_id(msg.src_id) + if stream is None: + return + fullname = msg.data.decode() + LOG.debug('%s requested module %s', stream.name, fullname) + self.get_module_count += 1 if fullname in stream.sent_modules: LOG.warning('_on_get_module(): dup request for %r from %r', fullname, stream) diff --git a/mitogen/parent.py b/mitogen/parent.py index 91a4e5eb..7e567aaa 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -2054,12 +2054,12 @@ class Router(mitogen.core.Router): def get_module_blacklist(self): if mitogen.context_id == 0: return self.responder.blacklist - return self.importer.blacklist + return self.importer.master_blacklist def get_module_whitelist(self): if mitogen.context_id == 0: return self.responder.whitelist - return self.importer.whitelist + return self.importer.master_whitelist def allocate_id(self): return self.id_allocator.allocate() diff --git a/mitogen/profiler.py b/mitogen/profiler.py index 10ec6086..74bbdb23 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/select.py b/mitogen/select.py index 6b87e671..fd2cbe9a 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/service.py b/mitogen/service.py index c67b35e8..3254e69a 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -603,8 +603,6 @@ class PushFileService(Service): This service will eventually be merged into FileService. """ - invoker_class = SerializedInvoker - def __init__(self, **kwargs): super(PushFileService, self).__init__(**kwargs) self._lock = threading.Lock() @@ -613,13 +611,16 @@ class PushFileService(Service): self._sent_by_stream = {} def get(self, path): + """ + Fetch a file from the cache. + """ assert isinstance(path, mitogen.core.UnicodeType) self._lock.acquire() try: if path in self._cache: return self._cache[path] - waiters = self._waiters.setdefault(path, []) latch = mitogen.core.Latch() + waiters = self._waiters.setdefault(path, []) waiters.append(lambda: latch.put(None)) finally: self._lock.release() @@ -633,14 +634,15 @@ class PushFileService(Service): stream = self.router.stream_by_id(context.context_id) child = mitogen.core.Context(self.router, stream.remote_id) sent = self._sent_by_stream.setdefault(stream, set()) - if path in sent and child.context_id != context.context_id: - child.call_service_async( - service_name=self.name(), - method_name='forward', - path=path, - context=context - ).close() - elif path not in sent: + if path in sent: + if child.context_id != context.context_id: + child.call_service_async( + service_name=self.name(), + method_name='forward', + path=path, + context=context + ).close() + else: child.call_service_async( service_name=self.name(), method_name='store_and_forward', @@ -680,14 +682,6 @@ class PushFileService(Service): fp.close() self._forward(context, path) - def _store(self, path, data): - self._lock.acquire() - try: - self._cache[path] = data - return self._waiters.pop(path, []) - finally: - self._lock.release() - @expose(policy=AllowParents()) @no_reply() @arg_spec({ @@ -696,9 +690,16 @@ class PushFileService(Service): 'context': mitogen.core.Context, }) def store_and_forward(self, path, data, context): - LOG.debug('%r.store_and_forward(%r, %r, %r)', - self, path, data, context) - waiters = self._store(path, data) + LOG.debug('%r.store_and_forward(%r, %r, %r) %r', + self, path, data, context, + threading.currentThread().getName()) + self._lock.acquire() + try: + self._cache[path] = data + waiters = self._waiters.pop(path, []) + finally: + self._lock.release() + if context.context_id != mitogen.context_id: self._forward(context, path) for callback in waiters: @@ -712,10 +713,17 @@ class PushFileService(Service): }) def forward(self, path, context): LOG.debug('%r.forward(%r, %r)', self, path, context) - if path not in self._cache: - LOG.error('%r: %r is not in local cache', self, path) - return - self._forward(context, path) + func = lambda: self._forward(context, path) + + self._lock.acquire() + try: + if path in self._cache: + func() + else: + LOG.debug('%r: %r not cached yet, queueing', self, path) + self._waiters.setdefault(path, []).append(func) + finally: + self._lock.release() class FileService(Service): diff --git a/mitogen/setns.py b/mitogen/setns.py index d38aa092..b1d69783 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 47c90fff..11b74c1b 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/su.py b/mitogen/su.py index 7eff60a6..5ff9e177 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 05a04989..868d4d76 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/unix.py b/mitogen/unix.py index 3e315d6f..66141eec 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/utils.py b/mitogen/utils.py index 6c56d6d5..94a171fb 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/setup.py b/setup.py index 6f31133d..c3257996 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index a968f84a..bec749f7 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,5 +1,5 @@ [defaults] -inventory = hosts,lib/inventory +inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy action_plugins = lib/action diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 02f3c614..d40c3dd0 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,8 +1,9 @@ # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the -# local machine. -target ansible_host=localhost +# local machine. The ansible_user override is necessary since some tests want a +# fixed ansible.cfg remote_user setting to test against. +target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" [test-targets] target diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts index 89bf7b38..41af412e 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts @@ -1,9 +1,8 @@ # vim: syntax=dosini -# This must be defined explicitly, otherwise _create_implicit_localhost() -# generates its own copy, which includes an ansible_python_interpreter that -# varies according to host machine. -localhost +# issue #511, #536: we must not define an explicit localhost, as some +# transport_config/python_path.yml needs to test the implicit localhost +# behaviour. # This is only used for manual testing. [localhost-x10] diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts new file mode 100644 index 00000000..d68b2d84 --- /dev/null +++ b/tests/ansible/hosts/transport_config.hosts @@ -0,0 +1,49 @@ +# integration/transport_config +# Hosts with twiddled configs that need to be checked somehow. + + +# tansport() +tc-transport-unset +tc-transport-local ansible_connection=local + +# python_path() +tc-python-path-unset +tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python +tc-python-path-local-unset ansible_connection=local +tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c + +# remote_addr() +tc-remote-addr-unset # defaults to inventory_hostname +tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host +tc-remote-addr-explicit-host ansible_host=ansi.host +tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d + +# password() +tc-password-unset +tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass +tc-password-explicit-user ansible_password=ansi-pass +tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a + +# become() +tc-become-unset +tc-become-set + +# become_method() +tc-become-method-unset +tc-become-method-su ansible_become_method=su + +# become_user() +tc-become-user-unset +tc-become-user-set ansible_become_user=ansi-become-user + +# become_pass() +tc-become-pass-unset +tc-become-pass-password ansible_become_password=apassword +tc-become-pass-pass ansible_become_pass=apass +tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a + +# port() +tc-port-unset +tc-port-explicit-port ansible_port=1234 +tc-port-explicit-ssh ansible_ssh_port=4321 +tc-port-both ansible_port=1717 ansible_ssh_port=1532 diff --git a/tests/ansible/integration/_mitogen_only.yml b/tests/ansible/integration/_mitogen_only.yml new file mode 100644 index 00000000..85ef378e --- /dev/null +++ b/tests/ansible/integration/_mitogen_only.yml @@ -0,0 +1,4 @@ +# Include me for plays that can't run on vanilla. +# +- meta: end_play + when: not is_mitogen diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index bd68b4ab..5898b9cd 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -19,3 +19,4 @@ - include: ssh/all.yml - include: strategy/all.yml - include: stub_connections/all.yml +- include: transport_config/all.yml diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index c2d2dc42..e1068587 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -5,7 +5,7 @@ any_errors_fatal: true tasks: - - shell: echo hi + - shell: echo hi; echo there >&2 async: 100 poll: 0 register: job @@ -21,10 +21,10 @@ - assert: that: - async_out.changed == True - - async_out.cmd == "echo hi" + - async_out.cmd == "echo hi; echo there >&2" - 'async_out.delta.startswith("0:00:")' - async_out.end.startswith("20") - - async_out.invocation.module_args._raw_params == "echo hi" + - async_out.invocation.module_args._raw_params == "echo hi; echo there >&2" - async_out.invocation.module_args._uses_shell == True - async_out.invocation.module_args.chdir == None - async_out.invocation.module_args.creates == None @@ -33,7 +33,7 @@ - async_out.invocation.module_args.warn == True - async_out.rc == 0 - async_out.start.startswith("20") - - async_out.stderr == "" + - async_out.stderr == "there" - async_out.stdout == "hi" vars: async_out: "{{result.content|b64decode|from_json}}" diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index a5c0216c..6e18ab6d 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -39,7 +39,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -66,7 +66,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', diff --git a/tests/ansible/integration/connection_delegation/local_action.yml b/tests/ansible/integration/connection_delegation/local_action.yml index d166c0d9..9d2cb65c 100644 --- a/tests/ansible/integration/connection_delegation/local_action.yml +++ b/tests/ansible/integration/connection_delegation/local_action.yml @@ -15,7 +15,7 @@ right: [ { 'kwargs': { - 'python_path': null + 'python_path': ["{{ansible_playbook_python}}"], }, 'method': 'local', }, @@ -23,7 +23,7 @@ 'enable_lru': true, 'kwargs': { 'connect_timeout': 10, - 'python_path': null, + 'python_path': ["{{ansible_playbook_python}}"], 'password': null, 'username': 'root', 'sudo_path': null, diff --git a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml index a761c432..4a1fa681 100644 --- a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml +++ b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml @@ -24,7 +24,7 @@ 'lxc_info_path': null, 'lxc_path': null, 'machinectl_path': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'ansible-cfg-remote-user', }, 'method': 'setns', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 0c48be3f..1b1f249d 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -43,7 +43,7 @@ "connect_timeout": 10, "doas_path": null, "password": null, - "python_path": null, + "python_path": ["/usr/bin/python"], "username": "normal-user", }, "method": "doas", @@ -72,7 +72,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -112,7 +112,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -147,7 +147,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -162,7 +162,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -202,7 +202,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -229,7 +229,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -264,7 +264,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -279,7 +279,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -320,7 +320,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -352,7 +352,7 @@ right: [ { 'kwargs': { - 'python_path': null + "python_path": ["{{ansible_playbook_python}}"], }, 'method': 'local', }, @@ -374,7 +374,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -384,7 +384,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'newuser-doas-normal-user', }, 'method': 'doas', diff --git a/tests/ansible/integration/transport_config/README.md b/tests/ansible/integration/transport_config/README.md new file mode 100644 index 00000000..bec55d04 --- /dev/null +++ b/tests/ansible/integration/transport_config/README.md @@ -0,0 +1,7 @@ + +# Tests for correct selection of connection variables. + +This directory is a placeholder for a work-in-progress test set that tries +every combination of the variables extracted via `transport_config.py`. + +In the meantime, it has ad-hoc scripts for bugs already encountered. diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml new file mode 100644 index 00000000..64199314 --- /dev/null +++ b/tests/ansible/integration/transport_config/all.yml @@ -0,0 +1,10 @@ +- include: become_method.yml +- include: become_pass.yml +- include: become_user.yml +- include: become.yml +- include: password.yml +- include: port.yml +- include: python_path.yml +- include: remote_addr.yml +- include: remote_user.yml +- include: transport.yml diff --git a/tests/ansible/integration/transport_config/become.yml b/tests/ansible/integration/transport_config/become.yml new file mode 100644 index 00000000..baa2085e --- /dev/null +++ b/tests/ansible/integration/transport_config/become.yml @@ -0,0 +1,68 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# No become set. +- name: integration/transport_config/become.yml + hosts: tc-become-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + +- hosts: tc-become-unset + vars: {mitogen_via: becomeuser@tc-become-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "becomeuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-unset" + + +# Become set. +- name: integration/transport_config/become.yml + hosts: tc-become-set + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "becomeuser" + +- hosts: tc-become-set + vars: {mitogen_via: tc-become-unset} + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-unset" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-set" + + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "becomeuser" diff --git a/tests/ansible/integration/transport_config/become_method.yml b/tests/ansible/integration/transport_config/become_method.yml new file mode 100644 index 00000000..5129e5b8 --- /dev/null +++ b/tests/ansible/integration/transport_config/become_method.yml @@ -0,0 +1,83 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# No become-method set. +- name: integration/transport_config/become-method.yml + hosts: tc-become-method-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + +- hosts: tc-become-method-unset + vars: {mitogen_via: becomeuser@tc-become-method-su} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "su" + - out.result[1].kwargs.username == "becomeuser" + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-method-unset" + + +# ansible_become_method=su +- hosts: tc-become-method-su + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "su" + - out.result[1].kwargs.username == "becomeuser" + +- hosts: tc-become-method-su + vars: {mitogen_via: tc-become-method-unset} + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-method-unset" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-method-su" + + - out.result[2].method == "su" + - out.result[2].kwargs.username == "becomeuser" + + + +# mitogen_via used to specify explicit become method +- hosts: tc-become-method-unset + vars: {mitogen_via: "doas:doasuser@tc-become-method-su"} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-method-su" + + - out.result[1].method == "doas" + - out.result[1].kwargs.username == "doasuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-method-unset" diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml new file mode 100644 index 00000000..02c6528d --- /dev/null +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -0,0 +1,142 @@ +# Each case is followed by mitogen_via= case to test hostvars pass. + + +# No become-pass set, defaults to "root" +- name: integration/transport_config/become-pass.yml + hosts: tc-become-pass-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == None + +# Not set, unbecoming mitogen_via= +- hosts: tc-become-pass-unset + become: true + vars: {mitogen_via: tc-become-pass-password} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "ssh" + - out.result[2].method == "sudo" + - out.result[2].kwargs.password == None + +# Not set, becoming mitogen_via= +- hosts: tc-become-pass-unset + become: true + vars: {mitogen_via: viapass@tc-become-pass-password} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == None + + +# ansible_become_password= set. +- hosts: tc-become-pass-password + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + + +# ansible_become_password=, via= +- hosts: tc-become-pass-password + vars: {mitogen_via: root@tc-become-pass-pass} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apass" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == "apassword" + + +# ansible_become_pass= +- hosts: tc-become-pass-pass + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apass" + + +# ansible_become_pass=, via= +- hosts: tc-become-pass-pass + vars: {mitogen_via: root@tc-become-pass-password} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == "apass" + + + +# ansible_become_pass & ansible_become_password set, password takes precedence +- hosts: tc-become-pass-both + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "a.b.c" + + +# both, mitogen_via +- hosts: tc-become-pass-unset + vars: {mitogen_via: root@tc-become-pass-both} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "a.b.c" + - out.result[2].method == "ssh" diff --git a/tests/ansible/integration/transport_config/become_user.yml b/tests/ansible/integration/transport_config/become_user.yml new file mode 100644 index 00000000..43cbca2a --- /dev/null +++ b/tests/ansible/integration/transport_config/become_user.yml @@ -0,0 +1,106 @@ +# Each case is followed by mitogen_via= case to test hostvars user. + + +# No become-user set, defaults to "root" +- name: integration/transport_config/become-user.yml + hosts: tc-become-user-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "root" + +# Not set, unbecoming mitogen_via= +- hosts: tc-become-user-unset + become: true + vars: {mitogen_via: tc-become-user-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "ssh" + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "root" + +# Not set, becoming mitogen_via= +- hosts: tc-become-user-unset + become: true + vars: {mitogen_via: viauser@tc-become-user-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "viauser" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.username == "root" + + +# ansible_become_user= set. +- hosts: tc-become-user-set + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "ansi-become-user" + + +# ansible_become_user=, unbecoming via= +- hosts: tc-become-user-set + vars: {mitogen_via: tc-become-user-unset} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-user-unset" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-user-set" + + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "ansi-become-user" + + +# ansible_become_user=, becoming via= +- hosts: tc-become-user-set + vars: {mitogen_via: "doas:doasuser@tc-become-user-unset"} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-user-unset" + + - out.result[1].method == "doas" + - out.result[1].kwargs.username == "doasuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-user-set" + + - out.result[3].method == "sudo" + - out.result[3].kwargs.username == "ansi-become-user" + diff --git a/tests/ansible/integration/transport_config/password.yml b/tests/ansible/integration/transport_config/password.yml new file mode 100644 index 00000000..ac236d66 --- /dev/null +++ b/tests/ansible/integration/transport_config/password.yml @@ -0,0 +1,94 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_ssh_pass/ansible_password= is set, password comes via +# interactive input. +- name: integration/transport_config/password.yml + hosts: tc-password-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" # actually null, but assert_equal limitation + +- hosts: tc-password-unset + vars: {mitogen_via: tc-password-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-ssh-pass" + - assert_equal: + left: out.result[1].kwargs.password + right: "" + + +# ansible_ssh_user= + +- hosts: tc-password-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-ssh-pass" + +- hosts: tc-password-explicit-ssh + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "ansi-ssh-pass" + + +# ansible_user= + +- hosts: tc-password-explicit-pass + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-pass" + +- hosts: tc-password-explicit-pass + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "ansi-pass" + + +# both; ansible_ssh_user= takes precedence according to play_context.py. + +- hosts: tc-password-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "c.b.a" + +- hosts: tc-password-explicit-both + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "c.b.a" diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml new file mode 100644 index 00000000..2781081a --- /dev/null +++ b/tests/ansible/integration/transport_config/port.yml @@ -0,0 +1,101 @@ +# Each case is followed by mitogen_via= case to test hostvars pass. + + +# No port set +- name: integration/transport_config/port.yml + hosts: tc-port-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == None + +# Not set, mitogen_via= +- hosts: tc-port-explicit-ssh + vars: {mitogen_via: tc-port-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == None + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == 4321 + +# ansible_ssh_port= +- hosts: tc-port-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 4321 + +- hosts: tc-port-explicit-unset + vars: {mitogen_via: tc-port-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].kwargs.port == 4321 + - out.result[1].method == "ssh" + - out.result[0].kwargs.port == None + +# ansible_port= +- hosts: tc-port-explicit-port + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1234 + +- hosts: tc-port-unset + vars: {mitogen_via: tc-port-explicit-port} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1234 + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == None + + +# both, ssh takes precedence +- hosts: tc-port-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1532 + +- hosts: tc-port-unset + vars: {mitogen_via: tc-port-both} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1532 + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == None diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml new file mode 100644 index 00000000..c5359e93 --- /dev/null +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -0,0 +1,114 @@ +# related: issue #511, #536 +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_python_interpreter is set, executor/module_common.py chooses +# "/usr/bin/python". +- name: integration/transport_config/python_path.yml + hosts: tc-python-path-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + +- hosts: tc-python-path-hostvar + vars: {mitogen_via: tc-python-path-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/hostvar/path/to/python"] + + +# Non-localhost with explicit ansible_python_interpreter +- hosts: tc-python-path-hostvar + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: [/hostvar/path/to/python] + +- hosts: tc-python-path-unset + vars: {mitogen_via: tc-python-path-hostvar} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/hostvar/path/to/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/usr/bin/python"] + + +# Implicit localhost gets ansible_python_interpreter=virtualenv interpreter +- hosts: localhost + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + +- hosts: tc-python-path-unset + vars: {mitogen_via: localhost} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/usr/bin/python"] + + +# explicit local connections get the same treatment as everything else. +- hosts: tc-python-path-local-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + +- hosts: localhost + vars: {mitogen_via: tc-python-path-local-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + + +# explicit local connection with explicit interpreter +- hosts: tc-python-path-local-explicit + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/a/b/c"] + +- hosts: localhost + vars: {mitogen_via: tc-python-path-local-explicit} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/a/b/c"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["{{ansible_playbook_python}}"] diff --git a/tests/ansible/integration/transport_config/remote_addr.yml b/tests/ansible/integration/transport_config/remote_addr.yml new file mode 100644 index 00000000..b9887202 --- /dev/null +++ b/tests/ansible/integration/transport_config/remote_addr.yml @@ -0,0 +1,95 @@ + +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_host/ansible_ssh_host= is set, hostname is same as inventory +# name. +- name: integration/transport_config/remote_addr.yml + hosts: tc-remote-addr-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + +- hosts: tc-remote-addr-unset + vars: {mitogen_via: tc-remote-addr-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.ssh.host" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "tc-remote-addr-unset" + + +# ansible_ssh_host= + +- hosts: tc-remote-addr-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.ssh.host" + +- hosts: tc-remote-addr-explicit-ssh + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "ansi.ssh.host" + + +# ansible_host= + +- hosts: tc-remote-addr-explicit-host + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.host" + +- hosts: tc-remote-addr-explicit-host + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "ansi.host" + + +# both; ansible_ssh_host= takes precedence according to play_context.py. + +- hosts: tc-remote-addr-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "a.b.c" + +- hosts: tc-remote-addr-explicit-both + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "a.b.c" diff --git a/tests/ansible/integration/transport_config/remote_user.yml b/tests/ansible/integration/transport_config/remote_user.yml new file mode 100644 index 00000000..b873fcbe --- /dev/null +++ b/tests/ansible/integration/transport_config/remote_user.yml @@ -0,0 +1,96 @@ + +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_user/ansible_ssh_user= is set, username is +# C.DEFAULT_REMOTE_USER. +- name: integration/transport_config/remote_user.yml + hosts: tc-remote-user-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + # We set DEFAULT_REMOTE_USER in our ansible.cfg + right: "ansible-cfg-remote-user" + +- hosts: tc-remote-user-unset + vars: {mitogen_via: tc-remote-user-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-ssh-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansible-cfg-remote-user" + + +# ansible_ssh_user= + +- hosts: tc-remote-user-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-ssh-user" + +- hosts: tc-remote-user-explicit-ssh + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansi-ssh-user" + + +# ansible_user= + +- hosts: tc-remote-user-explicit-user + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-user" + +- hosts: tc-remote-user-explicit-host + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansi-user" + + +# both; ansible_ssh_user= takes precedence according to play_context.py. + +- hosts: tc-remote-user-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "c.b.a" + +- hosts: tc-remote-user-explicit-both + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "c.b.a" diff --git a/tests/ansible/integration/transport_config/transport.yml b/tests/ansible/integration/transport_config/transport.yml new file mode 100644 index 00000000..efedc8d4 --- /dev/null +++ b/tests/ansible/integration/transport_config/transport.yml @@ -0,0 +1,48 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_connection= is set, transport comes via ansible.cfg ("smart" +# is parsed away to either paramiko or ssh). +- name: integration/transport_config/transport.yml + hosts: tc-transport-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "ssh" + +- hosts: tc-transport-local + vars: {mitogen_via: tc-transport-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "ssh" + - assert_equal: + left: out.result[1].method + right: "local" + + +# ansible_connection=local + +- hosts: tc-transport-local + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "local" + +- hosts: tc-transport-unset + vars: {mitogen_via: tc-transport-local} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "local" + - assert_equal: + left: out.result[1].method + right: "ssh" diff --git a/tests/ansible/run_ansible_playbook.py b/tests/ansible/run_ansible_playbook.py index 51f864f4..b5b459a1 100755 --- a/tests/ansible/run_ansible_playbook.py +++ b/tests/ansible/run_ansible_playbook.py @@ -46,6 +46,10 @@ if '-i' in sys.argv: extra['MITOGEN_INVENTORY_FILE'] = ( os.path.abspath(sys.argv[1 + sys.argv.index('-i')]) ) +else: + extra['MITOGEN_INVENTORY_FILE'] = ( + os.path.join(GIT_BASEDIR, 'tests/ansible/hosts') + ) args = ['ansible-playbook'] args += ['-e', json.dumps(extra)] diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index d898c782..8fa8cdb6 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -11,11 +11,156 @@ import mitogen.parent import ansible_mitogen.affinity + +class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy): + def _set_cpu_mask(self, mask): + self.mask = mask + + +class FixedPolicyTest(testlib.TestCase): + klass = NullFixedPolicy + + def test_assign_controller_1core(self): + # Uniprocessor . + policy = self.klass(cpu_count=1) + policy.assign_controller() + self.assertEquals(0x1, policy.mask) + + def test_assign_controller_2core(self): + # Small SMP gets 1.. % cpu_count + policy = self.klass(cpu_count=2) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + + def test_assign_controller_3core(self): + # Small SMP gets 1.. % cpu_count + policy = self.klass(cpu_count=3) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x4, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x4, policy.mask) + policy.assign_controller() + + def test_assign_controller_4core(self): + # Big SMP gets a dedicated core. + policy = self.klass(cpu_count=4) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + + def test_assign_muxprocess_1core(self): + # Uniprocessor . + policy = self.klass(cpu_count=1) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_muxprocess_2core(self): + # Small SMP gets dedicated core. + policy = self.klass(cpu_count=2) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + + def test_assign_muxprocess_3core(self): + # Small SMP gets a dedicated core. + policy = self.klass(cpu_count=3) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_muxprocess_4core(self): + # Big SMP gets a dedicated core. + policy = self.klass(cpu_count=4) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_worker_1core(self): + # Balance n % 1 + policy = self.klass(cpu_count=1) + policy.assign_worker() + self.assertEquals(0x1, policy.mask) + policy.assign_worker() + self.assertEquals(0x1, policy.mask) + + def test_assign_worker_2core(self): + # Balance n % 1 + policy = self.klass(cpu_count=2) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + + def test_assign_worker_3core(self): + # Balance n % 1 + policy = self.klass(cpu_count=3) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + policy.assign_worker() + self.assertEquals(0x4, policy.mask) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + + def test_assign_worker_4core(self): + # Balance n % 1 + policy = self.klass(cpu_count=4) + policy.assign_worker() + self.assertEquals(4, policy.mask) + policy.assign_worker() + self.assertEquals(8, policy.mask) + policy.assign_worker() + self.assertEquals(4, policy.mask) + + def test_assign_subprocess_1core(self): + # allow all except reserved. + policy = self.klass(cpu_count=1) + policy.assign_subprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_subprocess_2core(self): + # allow all except reserved. + policy = self.klass(cpu_count=2) + policy.assign_subprocess() + self.assertEquals(0x2, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x2, policy.mask) + + def test_assign_subprocess_3core(self): + # allow all except reserved. + policy = self.klass(cpu_count=3) + policy.assign_subprocess() + self.assertEquals(0x2 + 0x4, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x2 + 0x4, policy.mask) + + def test_assign_subprocess_4core(self): + # allow all except reserved. + policy = self.klass(cpu_count=4) + policy.assign_subprocess() + self.assertEquals(0x4 + 0x8, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x4 + 0x8, policy.mask) + + @unittest2.skipIf( reason='Linux/SMP only', condition=(not ( os.uname()[0] == 'Linux' and - multiprocessing.cpu_count() >= 4 + multiprocessing.cpu_count() > 2 )) ) class LinuxPolicyTest(testlib.TestCase): @@ -33,12 +178,15 @@ class LinuxPolicyTest(testlib.TestCase): finally: fp.close() - def test_set_clear(self): - before = self._get_cpus() - self.policy._set_cpu(3) - self.assertEquals(self._get_cpus(), 1 << 3) - self.policy._clear() - self.assertEquals(self._get_cpus(), before) + def test_set_cpu_mask(self): + self.policy._set_cpu_mask(0x1) + self.assertEquals(0x1, self._get_cpus()) + + self.policy._set_cpu_mask(0x2) + self.assertEquals(0x2, self._get_cpus()) + + self.policy._set_cpu_mask(0x3) + self.assertEquals(0x3, self._get_cpus()) def test_clear_on_popen(self): tf = tempfile.NamedTemporaryFile() diff --git a/tests/data/stubs/stub-sudo.py b/tests/data/stubs/stub-sudo.py index a7f2704f..71364df7 100755 --- a/tests/data/stubs/stub-sudo.py +++ b/tests/data/stubs/stub-sudo.py @@ -8,7 +8,12 @@ import sys os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) os.environ['THIS_IS_STUB_SUDO'] = '1' -# This must be a child process and not exec() since Mitogen replaces its stderr -# descriptor, causing the last user of the slave PTY to close it, resulting in -# the master side indicating EIO. -subprocess.check_call(sys.argv[sys.argv.index('--') + 1:]) +if os.environ.get('PREHISTORIC_SUDO'): + # issue #481: old versions of sudo did in fact use execve, thus we must + # have TTY handle preservation in core.py. + os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) +else: + # This must be a child process and not exec() since Mitogen replaces its + # stderr descriptor, causing the last user of the slave PTY to close it, + # resulting in the master side indicating EIO. + subprocess.check_call(sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/push_file_service_test.py b/tests/push_file_service_test.py new file mode 100644 index 00000000..1dfff241 --- /dev/null +++ b/tests/push_file_service_test.py @@ -0,0 +1,56 @@ + +import os +import tempfile +import unittest2 + +import mitogen.core +import mitogen.service +import testlib +from mitogen.core import b + + +def prepare(): + # ensure module loading delay is complete before loading PushFileService. + pass + + +@mitogen.core.takes_router +def wait_for_file(path, router): + pool = mitogen.service.get_or_create_pool(router=router) + service = pool.get_service(u'mitogen.service.PushFileService') + return service.get(path) + + +class PropagateToTest(testlib.RouterMixin, testlib.TestCase): + klass = mitogen.service.PushFileService + + def test_two_grandchild_one_intermediary(self): + tf = tempfile.NamedTemporaryFile() + path = mitogen.core.to_text(tf.name) + + try: + tf.write(b('test')) + tf.flush() + + interm = self.router.local(name='interm') + c1 = self.router.local(via=interm, name='c1') + c2 = self.router.local(via=interm) + + c1.call(prepare) + c2.call(prepare) + + service = self.klass(router=self.router) + service.propagate_to(context=c1, path=path) + service.propagate_to(context=c2, path=path) + + s = c1.call(wait_for_file, path=path) + self.assertEquals(b('test'), s) + + s = c2.call(wait_for_file, path=path) + self.assertEquals(b('test'), s) + finally: + tf.close() + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/sudo_test.py b/tests/sudo_test.py index 5bf9f4de..1d10ba9a 100644 --- a/tests/sudo_test.py +++ b/tests/sudo_test.py @@ -55,6 +55,15 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): '--' ]) + def test_tty_preserved(self): + # issue #481 + os.environ['PREHISTORIC_SUDO'] = '1' + try: + context, argv = self.run_sudo() + self.assertEquals('1', context.call(os.getenv, 'PREHISTORIC_SUDO')) + finally: + del os.environ['PREHISTORIC_SUDO'] + class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase): # Only mitogen/debian-test has a properly configured sudo.