diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index 12cad13c..26da7cfa 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -30,7 +30,7 @@ docker run \ --detach \ --publish 0.0.0.0:2201:22/tcp \ --name=target \ - d2mw/mitogen-${MITOGEN_TEST_DISTRO}-test + mitogen/${MITOGEN_TEST_DISTRO}-test echo travis_fold:end:docker_setup @@ -39,6 +39,7 @@ pip install -U ansible=="${ANSIBLE_VERSION}" cd ${TRAVIS_BUILD_DIR}/tests/ansible chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key +echo '[test-targets]' > ${TMPDIR}/hosts echo \ target \ ansible_host=$DOCKER_HOSTNAME \ @@ -59,7 +60,6 @@ echo travis_fold:end:job_setup echo travis_fold:start:mitogen_linear /usr/bin/time ./mitogen_ansible_playbook.sh \ all.yml \ - -vvv \ -i "${TMPDIR}/hosts" echo travis_fold:end:mitogen_linear @@ -67,6 +67,5 @@ echo travis_fold:end:mitogen_linear echo travis_fold:start:vanilla_ansible /usr/bin/time ./run_ansible_playbook.sh \ all.yml \ - -vvv \ -i "${TMPDIR}/hosts" echo travis_fold:end:vanilla_ansible diff --git a/.travis/debops_common_tests.sh b/.travis/debops_common_tests.sh index 30f5e776..eff7c901 100755 --- a/.travis/debops_common_tests.sh +++ b/.travis/debops_common_tests.sh @@ -60,7 +60,7 @@ do --detach \ --publish 0.0.0.0:$port:22/tcp \ --name=target$i \ - d2mw/mitogen-${MITOGEN_TEST_DISTRO}-test + mitogen/${MITOGEN_TEST_DISTRO}-test echo \ target$i \ diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 5baba223..93ad0df6 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -356,7 +356,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): executing. We use the opportunity to grab relevant bits from the task-specific data. """ - self.ansible_ssh_timeout = task_vars.get('ansible_ssh_timeout') + self.ansible_ssh_timeout = task_vars.get('ansible_ssh_timeout', + C.DEFAULT_TIMEOUT) self.python_path = task_vars.get('ansible_python_interpreter', '/usr/bin/python') self.mitogen_via = task_vars.get('mitogen_via') diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 79d71928..b9cbd3e7 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -319,7 +319,6 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): connection=self._connection, module_name=mitogen.utils.cast(module_name), module_args=mitogen.utils.cast(module_args), - remote_tmp=mitogen.utils.cast(self._get_remote_tmp()), task_vars=task_vars, templar=self._templar, env=mitogen.utils.cast(env), diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 7e3aa39a..f007205c 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -92,7 +92,7 @@ class Invocation(object): target.run_module() or helpers.run_module_async() in the target context. """ def __init__(self, action, connection, module_name, module_args, - remote_tmp, task_vars, templar, env, wrap_async): + task_vars, templar, env, wrap_async): #: ActionBase instance invoking the module. Required to access some #: output postprocessing methods that don't belong in ActionBase at #: all. @@ -104,9 +104,6 @@ class Invocation(object): self.module_name = module_name #: Final module arguments. self.module_args = module_args - #: Value of 'remote_tmp' parameter, to allow target to create temporary - #: files in correct location. - self.remote_tmp = remote_tmp #: Task variables, needed to extract ansible_*_interpreter. self.task_vars = task_vars #: Templar, needed to extract ansible_*_interpreter. @@ -198,7 +195,6 @@ class BinaryPlanner(Planner): path=invocation.module_path, args=invocation.module_args, env=invocation.env, - remote_tmp=invocation.remote_tmp, **kwargs ) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 323a0c1a..30c0ca7b 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -74,7 +74,7 @@ def reopen_readonly(fp): """ Replace the file descriptor belonging to the file object `fp` with one open on the same file (`fp.name`), but opened with :py:data:`os.O_RDONLY`. - This enables temporary files to be executed on Linux, which usually theows + This enables temporary files to be executed on Linux, which usually throws ``ETXTBUSY`` if any writeable handle exists pointing to a file passed to `execve()`. """ @@ -91,34 +91,32 @@ class Runner(object): returned by `run()`. Subclasses may override `_run`()` and extend `setup()` and `revert()`. + + :param str module: + Name of the module to execute, e.g. "shell" + :param mitogen.core.Context service_context: + Context to which we should direct FileService calls. For now, always + the connection multiplexer process on the controller. + :param dict args: + Ansible module arguments. A strange mixture of user and internal keys + created by ActionBase._execute_module(). + :param dict env: + Additional environment variables to set during the run. """ - def __init__(self, module, remote_tmp, service_context, - emulate_tty=None, raw_params=None, args=None, env=None): + def __init__(self, module, service_context, args=None, env=None): if args is None: args = {} - if raw_params is not None: - args['_raw_params'] = raw_params self.module = utf8(module) - self.remote_tmp = utf8(os.path.expanduser(remote_tmp)) self.service_context = service_context - self.emulate_tty = emulate_tty - self.raw_params = raw_params self.args = args self.env = env - self._temp_dir = None - - def get_temp_dir(self): - if not self._temp_dir: - self._temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') - # https://github.com/dw/mitogen/issues/239 - #ansible_mitogen.target.make_temp_directory(self.remote_tmp) - return self._temp_dir def setup(self): """ - Prepare the current process for running a module. The base - implementation simply prepares the environment. + Prepare for running a module, including fetching necessary dependencies + from the parent, as :meth:`run` may detach prior to beginning + execution. The base implementation simply prepares the environment. """ self._env = TemporaryEnvironment(self.env) @@ -128,8 +126,19 @@ class Runner(object): implementation simply restores the original environment. """ self._env.revert() - if self._temp_dir: - shutil.rmtree(self._temp_dir) + self._cleanup_temp() + + def _cleanup_temp(self): + """ + Empty temp_dir in time for the next module invocation. + """ + for name in os.listdir(ansible_mitogen.target.temp_dir): + if name in ('.', '..'): + continue + + path = os.path.join(ansible_mitogen.target.temp_dir, name) + LOG.debug('Deleting %r', path) + ansible_mitogen.target.prune_tree(path) def _run(self): """ @@ -175,7 +184,7 @@ class TemporaryEnvironment(object): class TemporaryArgv(object): def __init__(self, argv): self.original = sys.argv[:] - sys.argv[:] = argv + sys.argv[:] = map(str, argv) def revert(self): sys.argv[:] = self.original @@ -206,23 +215,44 @@ class NewStyleStdio(object): class ProgramRunner(Runner): - def __init__(self, path, **kwargs): + """ + Base class for runners that run external programs. + + :param str path: + Absolute path to the program file on the master, as it can be retrieved + via :class:`ansible_mitogen.services.FileService`. + :param bool emulate_tty: + If :data:`True`, execute the program with `stdout` and `stderr` merged + into a single pipe, emulating Ansible behaviour when an SSH TTY is in + use. + """ + def __init__(self, path, emulate_tty=None, **kwargs): super(ProgramRunner, self).__init__(**kwargs) - self.path = path + self.emulate_tty = emulate_tty + self.path = utf8(path) def setup(self): super(ProgramRunner, self).setup() self._setup_program() + def _get_program_filename(self): + """ + Return the filename used for program on disk. Ansible uses the original + filename for non-Ansiballz runs, and "ansible_module_+filename for + Ansiballz runs. + """ + return os.path.basename(self.path) + + program_fp = None + def _setup_program(self): """ Create a temporary file containing the program code. The code is fetched via :meth:`_get_program`. """ - self.program_fp = open( - os.path.join(self.get_temp_dir(), self.module), - 'wb' - ) + filename = self._get_program_filename() + path = os.path.join(ansible_mitogen.target.temp_dir, filename) + self.program_fp = open(path, 'wb') self.program_fp.write(self._get_program()) self.program_fp.flush() os.chmod(self.program_fp.name, int('0700', 8)) @@ -248,7 +278,8 @@ class ProgramRunner(Runner): """ Delete the temporary program file. """ - self.program_fp.close() + if self.program_fp: + self.program_fp.close() super(ProgramRunner, self).revert() def _run(self): @@ -285,7 +316,7 @@ class ArgsFileRunner(Runner): self.args_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-args', - dir=self.get_temp_dir(), + dir=ansible_mitogen.target.temp_dir, ) self.args_fp.write(self._get_args_contents()) self.args_fp.flush() @@ -362,22 +393,39 @@ class NewStyleRunner(ScriptRunner): def setup(self): super(NewStyleRunner, self).setup() self._stdio = NewStyleStdio(self.args) - self._argv = TemporaryArgv([self.program_fp.name]) + # It is possible that not supplying the script filename will break some + # module, but this has never been a bug report. Instead act like an + # interpreter that had its script piped on stdin. + self._argv = TemporaryArgv(['']) def revert(self): self._argv.revert() self._stdio.revert() super(NewStyleRunner, self).revert() + def _get_program_filename(self): + """ + See ProgramRunner._get_program_filename(). + """ + return 'ansible_module_' + os.path.basename(self.path) + + def _setup_args(self): + pass + + def _setup_program(self): + pass + def _get_code(self): + self.source = ansible_mitogen.target.get_file( + context=self.service_context, + path=self.path, + ) + try: return self._code_by_path[self.path] except KeyError: return self._code_by_path.setdefault(self.path, compile( - source=ansible_mitogen.target.get_file( - context=self.service_context, - path=self.path, - ), + source=self.source, filename=self.path, mode='exec', dont_inherit=True, @@ -385,14 +433,21 @@ class NewStyleRunner(ScriptRunner): def _run(self): code = self._get_code() + mod = types.ModuleType('__main__') - mod.__file__ = self.program_fp.name mod.__package__ = None - d = vars(mod) + # Some Ansible modules use __file__ to find the Ansiballz temporary + # directory. We must provide some temporary path in __file__, but we + # don't want to pointlessly write the module to disk when it never + # actually needs to exist. So just pass the filename as it would exist. + mod.__file__ = os.path.join( + ansible_mitogen.target.temp_dir, + 'ansible_module_' + os.path.basename(self.path), + ) e = None try: - exec code in d, d + exec code in vars(mod) except SystemExit, e: pass diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index f6f17687..96661c4c 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -270,7 +270,7 @@ class ContextService(mitogen.service.Service): # We don't need to wait for the result of this. Ideally we'd check its # return value somewhere, but logs will catch a failure anyway. - context.call_async(ansible_mitogen.target.start_fork_parent) + context.call_async(ansible_mitogen.target.init_child) if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'): from mitogen import debug diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index b42ecf31..5dc1dfa7 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -33,6 +33,7 @@ for file transfer, module execution and sundry bits like changing file modes. from __future__ import absolute_import import cStringIO +import errno import grp import json import logging @@ -58,6 +59,10 @@ import mitogen.service LOG = logging.getLogger(__name__) +#: Set by init_child() to the single temporary directory that will exist for +#: the duration of the process. +temp_dir = None + #: Caching of fetched file data. _file_cache = {} @@ -191,8 +196,68 @@ def transfer_file(context, in_path, out_path, sync=False, set_owner=False): os.utime(out_path, (metadata['atime'], metadata['mtime'])) +def prune_tree(path): + """ + Like shutil.rmtree(), but log errors rather than discard them, and do not + waste multiple os.stat() calls discovering whether the object can be + deleted, just try deleting it instead. + """ + try: + os.unlink(path) + return + except OSError, e: + if not (os.path.isdir(path) and + e.args[0] in (errno.EPERM, errno.EISDIR)): + LOG.error('prune_tree(%r): %s', path, e) + return + + try: + # Ensure write access for readonly directories. Ignore error in case + # path is on a weird filesystem (e.g. vfat). + os.chmod(path, int('0700', 8)) + except OSError, e: + LOG.warning('prune_tree(%r): %s', path, e) + + try: + for name in os.listdir(path): + if name not in ('.', '..'): + prune_tree(os.path.join(path, name)) + os.rmdir(path) + except OSError, e: + LOG.error('prune_tree(%r): %s', path, e) + + +def _on_broker_shutdown(): + """ + Respond to broker shutdown (graceful termination by parent, or loss of + connection to parent) by deleting our sole temporary directory. + """ + prune_tree(temp_dir) + + +@mitogen.core.takes_econtext +def reset_temp_dir(econtext): + """ + Create one temporary directory to be reused by all runner.py invocations + for the lifetime of the process. The temporary directory is changed for + each forked job, and emptied as necessary by runner.py::_cleanup_temp() + after each module invocation. + + The result is that a context need only create and delete one directory + during startup and shutdown, and no further filesystem writes need occur + assuming no modules execute that create temporary files. + """ + global temp_dir + # https://github.com/dw/mitogen/issues/239 + temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') + + # This must be reinstalled in forked children too, since the Broker + # instance from the parent process does not carry over to the new child. + mitogen.core.listen(econtext.broker, 'shutdown', _on_broker_shutdown) + + @mitogen.core.takes_econtext -def start_fork_parent(econtext): +def init_child(econtext): """ Called by ContextService immediately after connection; arranges for the (presently) spotless Python interpreter to be forked, where the newly @@ -206,12 +271,14 @@ def start_fork_parent(econtext): global _fork_parent mitogen.parent.upgrade_router(econtext) _fork_parent = econtext.router.fork() + reset_temp_dir(econtext) @mitogen.core.takes_econtext def start_fork_child(wrap_async, kwargs, econtext): mitogen.parent.upgrade_router(econtext) context = econtext.router.fork() + context.call(reset_temp_dir) if not wrap_async: try: return context.call(run_module, kwargs) @@ -331,7 +398,9 @@ def run_module_async(job_id, kwargs, econtext): def make_temp_directory(base_dir): """ Handle creation of `base_dir` if it is absent, in addition to a unique - temporary directory within `base_dir`. + temporary directory within `base_dir`. This is the temporary directory that + becomes 'remote_tmp', not the one used by Ansiballz. It always uses the + system temporary directory. :returns: Newly created temporary directory. diff --git a/docs/ansible.rst b/docs/ansible.rst index 9e03b190..e08fca9b 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -157,9 +157,16 @@ Noteworthy Differences * Performance does not scale perfectly linearly with target count. This will improve over time. -* Timeouts normally apply to the combined runtime of the SSH and become steps - of a task. As Mitogen treats SSH and sudo distincly, during a failure the - effective timeout may appear to double. +* SSH and ``become`` are treated distinctly when applying timeouts, and + timeouts apply up to the point when the new interpreter is ready to accept + messages. Ansible has two timeouts: ``ConnectTimeout`` for SSH, applying up + to when authentication completes, and a separate parallel timeout up to when + ``become`` authentication completes. + + For busy targets, Ansible may successfully execute a module where Mitogen + would fail without increasing the timeout. For sick targets, Ansible may hang + indefinitely after authentication without executing a command, for example + due to a stuck filesystem IO appearing in ``$HOME/.profile``. New Features & Notes diff --git a/docs/api.rst b/docs/api.rst index f373664d..a1711f11 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -859,14 +859,14 @@ Router Class Port number to connect to; default is unspecified, which causes SSH to pick the port number. :param str check_host_keys: - Specifies the SSH host key checking mode: + Specifies the SSH host key checking mode. Defaults to ``enforce``. * ``ignore``: no host key checking is performed. Connections never fail due to an unknown or changed host key. * ``accept``: known hosts keys are checked to ensure they match, new host keys are automatically accepted and verified in future connections. - * ``enforce``: known host keys are checke to ensure they match, + * ``enforce``: known host keys are checked to ensure they match, unknown hosts cause a connection failure. :param str password: Password to type if/when ``ssh`` requests it. If not specified and @@ -887,6 +887,16 @@ Router Class remaining message in the otherwise uncompressed stream protocol, such as function call arguments and return values. + :raises mitogen.ssh.PasswordError: + A password was requested but none was specified, or the specified + password was incorrect. + + :raises mitogen.ssh.HostKeyError: + When `check_host_keys` is set to either ``accept``, indicates a + previously recorded key no longer matches the remote machine. When + set to ``enforce``, as above, but additionally indicates no + previously recorded key exists for the remote machine. + Context Class ============= diff --git a/docs/conf.py b/docs/conf.py index 7c218f8b..90c0f446 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ def grep_version(): author = u'David Wilson' -copyright = u'2016, David Wilson' +copyright = u'2018, David Wilson' exclude_patterns = ['_build'] extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput'] html_show_sourcelink = False diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 3b50e757..aec86bd7 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -115,15 +115,15 @@ Logging Environment Variables Overrides the :py:mod:`logging` package log level set by any call to :py:func:`mitogen.utils.log_to_file`. Defaults to ``INFO``. + If set to ``IO``, equivalent to ``DEBUG`` but additionally enabled IO + logging for any call to :py:func:`mitogen.utils.log_to_file`. IO logging + produces verbose records of any IO interaction, which is useful for + debugging hangs and deadlocks. + ``MITOGEN_LOG_USEC`` If present, forces microsecond-level timestamps for any call to :py:func:`mitogen.utils.log_to_file`. -``MITOGEN_LOG_IO`` - If present, forces IO logging for any call to - :py:func:`mitogen.utils.log_to_file`. IO logging produces extremely verbose - logs of any IO interaction, which is useful when debugging deadlocks. - Logging Records diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 0e48a538..5c8bb018 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -445,6 +445,20 @@ also listen on the following handles: In this way, the master need never re-send a module it has already sent to a direct descendant. +.. currentmodule:: mitogen.core +.. data:: DETACHING + + Sent to inform a parent that user code has invoked + :meth:`ExternalContext.detach` to decouple the lifecycle of a directly + connected context and its subtree from the running program. + + A child usually shuts down immediately if it loses its parent connection, + and parents usually terminate any related Python/SSH subprocess on + disconnection. Receiving :data:`DETACHING` informs the parent the + connection will soon drop, but the process intends to continue life + independently, and to avoid terminating the related subprocess if that + subprocess is the child itself. + Additional handles are created to receive the result of every function call triggered by :py:meth:`call_async() `. diff --git a/docs/images/billing.graphml b/docs/images/billing.graphml index 1830135f..f2b34fb3 100644 --- a/docs/images/billing.graphml +++ b/docs/images/billing.graphml @@ -15,7 +15,6 @@ - @@ -33,7 +32,6 @@ - @@ -51,13 +49,12 @@ - - + - sudo + sudo:root @@ -69,13 +66,12 @@ - - + - ssh:billing0 + docker:billing0 @@ -87,10 +83,9 @@ - - + run-nightly-billing.py @@ -104,8 +99,24 @@ + + + + + + + ssh:docker-a + + + + + + + + + + - @@ -115,8 +126,7 @@ - - + @@ -127,7 +137,6 @@ - @@ -138,6 +147,16 @@ + + + + + + + + + + diff --git a/docs/images/billing.png b/docs/images/billing.png index a5dbf10d..f919acc3 100644 Binary files a/docs/images/billing.png and b/docs/images/billing.png differ diff --git a/docs/images/disconnected-subtree.graphml b/docs/images/detached-subtree.graphml similarity index 100% rename from docs/images/disconnected-subtree.graphml rename to docs/images/detached-subtree.graphml diff --git a/docs/images/disconnected-subtree.png b/docs/images/detached-subtree.png similarity index 100% rename from docs/images/disconnected-subtree.png rename to docs/images/detached-subtree.png diff --git a/docs/index.rst b/docs/index.rst index 215dc4d0..62158736 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -95,15 +95,20 @@ to your network topology**. hostname='jump-box.mycorp.com' ) - ssh_account = router.sudo( + docker_host = router.ssh( via=bastion_host, + hostname='docker-a.prod.mycorp.com' + ) + + sudo_account = router.sudo( + via=docker_host, username='user_with_magic_ssh_key', password='sudo password', ) - internal_box = router.ssh( - via=ssh_account, - hostname='billing0.internal.mycorp.com' + internal_box = router.docker( + via=sudo_account, + container='billing0', ) internal_box.call(os.system, './run-nightly-billing.py') @@ -232,6 +237,32 @@ uptime')** without further need to capture or manage output. 18:17:56 I mitogen.ctx.k3: stdout: 17:37:10 up 562 days, 2:25, 5 users, load average: 1.24, 1.13, 1.14 +Detached Subtrees +################# + +.. image:: images/detached-subtree.png + +Contexts may detach from and outlive the running program, while maintaining +communication with descendents in their subtree. This enables persistent +background tasks that reuse Mitogen features. + +.. code:: + + @mitogen.core.takes_econtext + def become_monitoring_master(children, econtext): + kill_old_process('/var/run/mydaemon.pid') + write_pid_file('/var/run/mydaemon.pid') + econtext.detach() + + while True: + for child in children: + if child.call(get_cpu_load) > 0.9: + alert_operator('Child is too busy! ' + str(child)) + time.sleep(1) + + dc1.call_async(become_monitoring_master, children) + + Blocking Code Friendly ###################### diff --git a/docs/internals.rst b/docs/internals.rst index d22053c8..f3771343 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2,6 +2,11 @@ Internal API Reference ********************** +.. toctree:: + :hidden: + + signals + mitogen.core ============ @@ -462,3 +467,9 @@ Helper Functions :returns str: The minimized source. + + +Signals +======= + +:ref:`Please refer to Signals `. diff --git a/docs/signals.rst b/docs/signals.rst index 8a314b5c..1c41353a 100644 --- a/docs/signals.rst +++ b/docs/signals.rst @@ -1,11 +1,20 @@ +.. _signals: + Signals ======= -Mitogen exposes a simplistic signal mechanism to help decouple its internal +Mitogen contains a simplistic signal mechanism to help decouple its internal components. When a signal is fired by a particular instance of a class, any functions registered to receive it will be called back. +.. warning:: + + As signals execute on the Broker thread, and without exception handling, + they are generally unsafe for consumption by user code, as any bugs could + trigger crashes and hangs for which the broker is unable to forward logs, + or ensure the buggy context always shuts down on disconnect. + Functions --------- diff --git a/mitogen/core.py b/mitogen/core.py index 56fc3266..f47b8380 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -75,6 +75,7 @@ DEL_ROUTE = 104 ALLOCATE_ID = 105 SHUTDOWN = 106 LOAD_MODULE = 107 +DETACHING = 108 IS_DEAD = 999 PY3 = sys.version_info > (3,) @@ -953,6 +954,7 @@ class Context(object): raise SystemError('Cannot making blocking call on broker thread') receiver = Receiver(self.router, persist=persist, respondent=self) + msg.dst_id = self.context_id msg.reply_to = receiver.handle _v and LOG.debug('%r.send_async(%r)', self, msg) @@ -1277,6 +1279,10 @@ class Router(object): self.broker.start_receive(stream) listen(stream, 'disconnect', lambda: self.on_stream_disconnect(stream)) + def stream_by_id(self, dst_id): + return self._stream_by_id.get(dst_id, + self._stream_by_id.get(mitogen.parent_id)) + def del_handler(self, handle): del self._handle_map[handle] @@ -1501,6 +1507,8 @@ class Broker(object): class ExternalContext(object): + detached = False + def _on_broker_shutdown(self): self.channel.close() @@ -1514,8 +1522,34 @@ class ExternalContext(object): self.broker.shutdown() def _on_parent_disconnect(self): - _v and LOG.debug('%r: parent stream is gone, dying.', self) - self.broker.shutdown() + if self.detached: + mitogen.parent_ids = [] + mitogen.parent_id = None + LOG.info('Detachment complete') + else: + _v and LOG.debug('%r: parent stream is gone, dying.', self) + self.broker.shutdown() + + def _sync(self, func): + latch = Latch() + self.broker.defer(lambda: latch.put(func())) + return latch.get() + + def detach(self): + self.detached = True + stream = self.router.stream_by_id(mitogen.parent_id) + if stream: # not double-detach()'d + os.setsid() + self.parent.send_await(Message(handle=DETACHING)) + LOG.info('Detaching from %r; parent is %s', stream, self.parent) + for x in range(20): + pending = self._sync(lambda: stream.pending_bytes()) + if not pending: + break + time.sleep(0.05) + if pending: + LOG.error('Stream had %d bytes after 2000ms', pending) + self.broker.defer(stream.on_disconnect, self.broker) def _setup_master(self, max_message_size, profiling, parent_id, context_id, in_fd, out_fd): diff --git a/mitogen/docker.py b/mitogen/docker.py index c265dc4f..148c132b 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -36,6 +36,8 @@ LOG = logging.getLogger(__name__) class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + container = None image = None username = None diff --git a/mitogen/fork.py b/mitogen/fork.py index fdf40c39..70737fc8 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -81,6 +81,8 @@ def handle_child_crash(): class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = True + #: Reference to the importer, if any, recovered from the parent. importer = None diff --git a/mitogen/jail.py b/mitogen/jail.py index 37c89483..ed04da00 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__) class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False create_child_args = { 'merge_stdio': True } diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 8a8e4b78..eb8ad173 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__) class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False create_child_args = { # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY, # to prevent input injection it creates a proxy pty, forcing all IO to diff --git a/mitogen/master.py b/mitogen/master.py index d0bc55a8..95202e35 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -690,6 +690,11 @@ class Router(mitogen.parent.Router): self.responder = ModuleResponder(self) self.log_forwarder = LogForwarder(self) self.route_monitor = mitogen.parent.RouteMonitor(router=self) + self.add_handler( # TODO: cutpaste. + fn=self._on_detaching, + handle=mitogen.core.DETACHING, + persist=True, + ) def enable_debug(self): mitogen.core.enable_debug_logging() diff --git a/mitogen/parent.py b/mitogen/parent.py index 76ca50bd..feac28a8 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -599,12 +599,23 @@ class Stream(mitogen.core.Stream): ) ) + #: If :data:`True`, indicates the subprocess managed by us should not be + #: killed during graceful detachment, as it the actual process implementing + #: the child context. In all other cases, the subprocess is SSH, sudo, or a + #: similar tool that should be reminded to quit during disconnection. + child_is_immediate_subprocess = True + + detached = False _reaped = False def _reap_child(self): """ Reap the child process during disconnection. """ + if self.detached and self.child_is_immediate_subprocess: + LOG.debug('%r: immediate child is detached, won\'t reap it', self) + return + if self._reaped: # on_disconnect() may be invoked more than once, for example, if # there is still a pending message to be sent after the first @@ -929,10 +940,22 @@ class Router(mitogen.core.Router): importer=importer, ) self.route_monitor = RouteMonitor(self, parent) + self.add_handler( + fn=self._on_detaching, + handle=mitogen.core.DETACHING, + persist=True, + ) - def stream_by_id(self, dst_id): - return self._stream_by_id.get(dst_id, - self._stream_by_id.get(mitogen.parent_id)) + def _on_detaching(self, msg): + if msg.is_dead: + return + stream = self.stream_by_id(msg.src_id) + if stream.remote_id != msg.src_id or stream.detached: + LOG.warning('bad DETACHING received on %r: %r', stream, msg) + return + LOG.debug('%r: marking as detached', stream) + stream.detached = True + msg.reply(None) def add_route(self, target_id, stream): LOG.debug('%r.add_route(%r, %r)', self, target_id, stream) @@ -974,6 +997,8 @@ class Router(mitogen.core.Router): self._context_by_id[context_id] = context return context + connection_timeout_msg = "Connection timed out." + def _connect(self, klass, name=None, **kwargs): context_id = self.allocate_id() context = self.context_class(self, context_id) @@ -982,7 +1007,11 @@ class Router(mitogen.core.Router): stream = klass(self, context_id, **kwargs) if name is not None: stream.name = name - stream.connect() + try: + stream.connect() + except mitogen.core.TimeoutError: + e = sys.exc_info()[1] + raise mitogen.core.StreamError(self.connection_timeout_msg) context.name = stream.name self.route_monitor.notice_stream(stream) self.register(context, stream) diff --git a/mitogen/service.py b/mitogen/service.py index 53c5ff46..0d4bd304 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -330,7 +330,10 @@ class Pool(object): thread.start() self._threads.append(thread) + closed = False + def stop(self): + self.closed = True self._select.close() for th in self._threads: th.join() @@ -338,7 +341,7 @@ class Pool(object): service.on_shutdown() def _worker_run(self): - while True: + while not self.closed: try: msg = self._select.get() except (mitogen.core.ChannelError, mitogen.core.LatchError): diff --git a/mitogen/setns.py b/mitogen/setns.py index 979e6bf4..c95b3217 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -105,6 +105,8 @@ def get_machinectl_pid(path, name): class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + container = None username = None kind = None diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 327a4efd..76fad2a0 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -59,6 +59,7 @@ class HostKeyError(mitogen.core.StreamError): class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) + child_is_immediate_subprocess = False python_path = 'python2.7' #: Once connected, points to the corresponding TtyLogStream, allowing it to @@ -154,8 +155,8 @@ class Stream(mitogen.parent.Stream): 'configuration.' ) hostkey_failed_msg = ( - 'check_host_keys is set to enforce, and SSH reported an unknown ' - 'or changed host key.' + 'Host key checking is enabled, and SSH reported an unrecognized or ' + 'mismatching host key.' ) def _host_key_prompt(self): diff --git a/mitogen/su.py b/mitogen/su.py index bfaada11..2cc3406b 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -46,6 +46,7 @@ class Stream(mitogen.parent.Stream): # for hybrid_tty_create_child(), there just needs to be either a shell # snippet or bootstrap support for fixing things up afterwards. create_child = staticmethod(mitogen.parent.tty_create_child) + child_is_immediate_subprocess = False #: Once connected, points to the corresponding TtyLogStream, allowing it to #: be disconnected at the same time this stream is being torn down. diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 9377c960..5d2911fc 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -104,6 +104,7 @@ class PasswordError(mitogen.core.StreamError): class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) + child_is_immediate_subprocess = False #: Once connected, points to the corresponding TtyLogStream, allowing it to #: be disconnected at the same time this stream is being torn down. diff --git a/mitogen/utils.py b/mitogen/utils.py index ab8a673a..480940ab 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -71,13 +71,14 @@ def log_to_file(path=None, io=False, usec=False, level='INFO'): fp = sys.stderr level = os.environ.get('MITOGEN_LOG_LEVEL', level).upper() - level = getattr(logging, level, logging.INFO) - log.setLevel(level) - - io = ('MITOGEN_LOG_IO' in os.environ) or io + io = level == 'IO' if io: + level = 'DEBUG' logging.getLogger('mitogen.io').setLevel(level) + level = getattr(logging, level, logging.INFO) + log.setLevel(level) + handler = logging.StreamHandler(fp) handler.formatter = log_get_formatter(usec=usec) log.handlers.insert(0, handler) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 0abd2594..9a2887a4 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -8,6 +8,9 @@ library = lib/modules retry_files_enabled = False forks = 50 +# Required by integration/ssh/timeouts.yml +timeout = 10 + # On Travis, paramiko check fails due to host key checking enabled. host_key_checking = False diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index efb2614d..54841a67 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -3,11 +3,12 @@ # This playbook imports all tests that are known to work at present. # -- import_playbook: action/all.yml -- import_playbook: async/all.yml -- import_playbook: become/all.yml -- import_playbook: connection_loader/all.yml -- import_playbook: context_service/all.yml -- import_playbook: playbook_semantics/all.yml -- import_playbook: remote_tmp/all.yml -- import_playbook: runner/all.yml +#- import_playbook: action/all.yml +#- import_playbook: async/all.yml +#- import_playbook: become/all.yml +#- import_playbook: connection_loader/all.yml +#- import_playbook: context_service/all.yml +#- import_playbook: playbook_semantics/all.yml +#- import_playbook: remote_tmp/all.yml +#- import_playbook: runner/all.yml +- import_playbook: ssh/all.yml diff --git a/tests/ansible/integration/remote_tmp/readonly_homedir.yml b/tests/ansible/integration/remote_tmp/readonly_homedir.yml index 1cce891a..62435189 100644 --- a/tests/ansible/integration/remote_tmp/readonly_homedir.yml +++ b/tests/ansible/integration/remote_tmp/readonly_homedir.yml @@ -14,8 +14,7 @@ vars: ansible_become_pass: readonly_homedir_password - - debug: msg={{out}} - name: Verify system temp directory was used. assert: that: - - out.argv[0].startswith("/tmp/ansible_mitogen_") + - out.__file__.startswith("/tmp/ansible_mitogen_") diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 9ede6984..a5da1c87 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -12,4 +12,3 @@ - import_playbook: custom_python_want_json_module.yml - import_playbook: custom_script_interpreter.yml - import_playbook: forking_behaviour.yml -- import_playbook: remote_tmp.yml diff --git a/tests/ansible/integration/runner/builtin_command_module.yml b/tests/ansible/integration/runner/builtin_command_module.yml index afa34902..0bc5bd34 100644 --- a/tests/ansible/integration/runner/builtin_command_module.yml +++ b/tests/ansible/integration/runner/builtin_command_module.yml @@ -1,5 +1,5 @@ -- name: integration/runner__builtin_command_module.yml +- name: integration/runner/builtin_command_module.yml hosts: test-targets any_errors_fatal: true gather_facts: true diff --git a/tests/ansible/integration/runner/custom_bash_old_style_module.yml b/tests/ansible/integration/runner/custom_bash_old_style_module.yml index 96bc9297..ff963665 100644 --- a/tests/ansible/integration/runner/custom_bash_old_style_module.yml +++ b/tests/ansible/integration/runner/custom_bash_old_style_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_bash_old_style_module.yml +- name: integration/runner/custom_bash_old_style_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_bash_want_json_module.yml b/tests/ansible/integration/runner/custom_bash_want_json_module.yml index f46a32c7..075c95b2 100644 --- a/tests/ansible/integration/runner/custom_bash_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_bash_want_json_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_bash_want_json_module.yml +- name: integration/runner/custom_bash_want_json_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml index ed3da57a..00f03f07 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_json.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_binary_producing_json.yml +- name: integration/runner/custom_binary_producing_json.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml index 32797b55..93d98065 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_junk.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_binary_producing_junk.yml +- name: integration/runner/custom_binary_producing_junk.yml hosts: test-targets tasks: - custom_binary_producing_junk: diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml index f23ec1c2..bab84381 100644 --- a/tests/ansible/integration/runner/custom_binary_single_null.yml +++ b/tests/ansible/integration/runner/custom_binary_single_null.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_binary_single_null.yml +- name: integration/runner/custom_binary_single_null.yml hosts: test-targets tasks: - custom_binary_single_null: diff --git a/tests/ansible/integration/runner/custom_perl_json_args_module.yml b/tests/ansible/integration/runner/custom_perl_json_args_module.yml index a59ab441..3485463d 100644 --- a/tests/ansible/integration/runner/custom_perl_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_perl_json_args_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_perl_json_args_module.yml +- name: integration/runner/custom_perl_json_args_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_perl_want_json_module.yml b/tests/ansible/integration/runner/custom_perl_want_json_module.yml index c1dc1a2d..69a1b57b 100644 --- a/tests/ansible/integration/runner/custom_perl_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_perl_want_json_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_perl_want_json_module.yml +- name: integration/runner/custom_perl_want_json_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_python_json_args_module.yml b/tests/ansible/integration/runner/custom_python_json_args_module.yml index a7d6937d..338f9180 100644 --- a/tests/ansible/integration/runner/custom_python_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_python_json_args_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_python_json_args_module.yml +- name: integration/runner/custom_python_json_args_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index a095018b..9f7d08ba 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -1,5 +1,5 @@ -- name: integration/runner__custom_python_new_style_module.yml +- name: integration/runner/custom_python_new_style_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index 5a2a98a8..7e722253 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_python_new_style_module.yml +- name: integration/runner/custom_python_new_style_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/custom_python_want_json_module.yml b/tests/ansible/integration/runner/custom_python_want_json_module.yml index c3ff2614..f6d8c355 100644 --- a/tests/ansible/integration/runner/custom_python_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_python_want_json_module.yml @@ -1,4 +1,4 @@ -- name: integration/runner__custom_python_want_json_module.yml +- name: integration/runner/custom_python_want_json_module.yml hosts: test-targets any_errors_fatal: true tasks: diff --git a/tests/ansible/integration/runner/remote_tmp.yml b/tests/ansible/integration/runner/remote_tmp.yml deleted file mode 100644 index a0b05d50..00000000 --- a/tests/ansible/integration/runner/remote_tmp.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# The ansible.cfg remote_tmp setting should be copied to the target and used -# when generating temporary paths created by the runner.py code executing -# remotely. -# -- name: integration/runner__remote_tmp.yml - hosts: test-targets - any_errors_fatal: true - gather_facts: true - tasks: - - bash_return_paths: - register: output - - - assert: - that: output.argv0.startswith('%s/.ansible/mitogen-tests/' % ansible_user_dir) - - - assert: - that: output.argv1.startswith('%s/.ansible/mitogen-tests/' % ansible_user_dir) diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml new file mode 100644 index 00000000..8a3b7f88 --- /dev/null +++ b/tests/ansible/integration/ssh/all.yml @@ -0,0 +1 @@ +- import_playbook: timeouts.yml diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml new file mode 100644 index 00000000..71e41218 --- /dev/null +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -0,0 +1,20 @@ +# Ensure 'ssh' connections time out correctly. + +- name: integration/ssh/timeouts.yml + hosts: test-targets + tasks: + - connection: local + command: ansible -vvv -i "{{inventory_file}}" test-targets -m custom_python_detect_environment -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password + register: out + ignore_errors: true + when: is_mitogen + + - assert: + that: + - | + '"changed": false' in out.stdout + - | + '"unreachable": true' in out.stdout + - | + '"msg": "Connection timed out."' in out.stdout + when: is_mitogen diff --git a/tests/ansible/lib/action/mitogen_shutdown_all.py b/tests/ansible/lib/action/mitogen_shutdown_all.py index 65be8b20..9af21e11 100644 --- a/tests/ansible/lib/action/mitogen_shutdown_all.py +++ b/tests/ansible/lib/action/mitogen_shutdown_all.py @@ -6,6 +6,7 @@ required for reliable LRU tests. import traceback import sys +import ansible_mitogen.connection import ansible_mitogen.services import mitogen.service @@ -15,9 +16,10 @@ from ansible.plugins.action import ActionBase class ActionModule(ActionBase): def run(self, tmp=None, task_vars=None): - if not type(self._connection).__module__.startswith('ansible_mitogen'): + if not isinstance(self._connection, + ansible_mitogen.connection.Connection): return { - 'changed': False + 'skipped': True, } self._connection._connect() diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 62250f79..8f369e86 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -14,6 +14,7 @@ def main(): module = AnsibleModule(argument_spec={}) module.exit_json( argv=sys.argv, + __file__=__file__, argv_types=[str(type(s)) for s in sys.argv], env=dict(os.environ), cwd=os.getcwd(), diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 4d1bb23f..26022ff3 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -11,16 +11,10 @@ def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) -# Also must slurp in our own source code, to verify the encoding string was -# added. -with open(sys.argv[0]) as fp: - me = fp.read() - input_json = sys.stdin.read() print "{" print " \"changed\": false," print " \"msg\": \"Here is my input\"," -print " \"source\": [%s]," % (json.dumps(me),) print " \"input\": [%s]" % (input_json,) print "}" diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 4bc9794d..69b4dae0 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -12,11 +12,6 @@ def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) -# Also must slurp in our own source code, to verify the encoding string was -# added. -with open(sys.argv[0]) as fp: - me = fp.read() - input_json = sys.stdin.read() print "{" @@ -27,6 +22,5 @@ print " \"__file__\": \"%s\"," % (__file__,) # Python sets this during a regular import. print " \"__package__\": \"%s\"," % (__package__,) print " \"msg\": \"Here is my input\"," -print " \"source\": [%s]," % (json.dumps(me),) print " \"input\": [%s]" % (input_json,) print "}" diff --git a/tests/ansible/osx_setup.yml b/tests/ansible/osx_setup.yml index d06c5fc2..3c53fd8f 100644 --- a/tests/ansible/osx_setup.yml +++ b/tests/ansible/osx_setup.yml @@ -29,6 +29,7 @@ - require_tty - pw_required - require_tty_pw_required + - slow_user when: ansible_system != 'Darwin' - name: Create Mitogen test users @@ -52,6 +53,7 @@ - pw_required - require_tty_pw_required - readonly_homedir + - slow_user when: ansible_system == 'Darwin' - name: Create Mitogen test users @@ -88,6 +90,14 @@ - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" + - name: Slow bash profile for one account + copy: + dest: ~mitogen__slow_user/.{{item}} + src: ../data/docker/mitogen__slow_user.profile + with_items: + - bashrc + - profile + - name: Require a TTY for two accounts lineinfile: path: /etc/sudoers diff --git a/tests/build_docker_images.py b/tests/build_docker_images.py index 0915aa46..7bceda4d 100755 --- a/tests/build_docker_images.py +++ b/tests/build_docker_images.py @@ -47,8 +47,10 @@ RUN \ useradd -s /bin/bash -m mitogen__require_tty && \ useradd -s /bin/bash -m mitogen__require_tty_pw_required && \ useradd -s /bin/bash -m mitogen__readonly_homedir && \ + useradd -s /bin/bash -m mitogen__slow_user && \ chown -R root: ~mitogen__readonly_homedir && \ { for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user$i; done; } && \ + { for i in `seq 1 21`; do echo mitogen__user$i:user$i_password | chpasswd; } && \ ( echo 'root:rootpassword' | chpasswd; ) && \ ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \ ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \ @@ -58,10 +60,14 @@ RUN \ ( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \ ( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \ ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \ + ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \ mkdir ~mitogen__has_sudo_pubkey/.ssh && \ { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; } COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys +COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile +COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc + RUN \ chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \ chmod -R go= ~mitogen__has_sudo_pubkey @@ -93,6 +99,6 @@ for (distro, wheel, prefix) in (('debian', 'sudo', DEBIAN_DOCKERFILE), subprocess.check_call(sh('docker build %s -t %s -f %s', mydir, - 'd2mw/mitogen-%s-test' % (distro,), + 'mitogen/%s-test' % (distro,), dockerfile_fp.name )) diff --git a/tests/data/docker/mitogen__slow_user.profile b/tests/data/docker/mitogen__slow_user.profile new file mode 100644 index 00000000..e0933cfa --- /dev/null +++ b/tests/data/docker/mitogen__slow_user.profile @@ -0,0 +1,3 @@ + +# mitogen__slow_user takes forever to log in. +sleep 30 diff --git a/tests/testlib.py b/tests/testlib.py index 4e01b4ea..0997752d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -175,7 +175,7 @@ class DockerizedSshDaemon(object): def get_image(self): if not self.image: distro = os.environ.get('MITOGEN_TEST_DISTRO', 'debian') - self.image = 'd2mw/mitogen-%s-test' % (distro,) + self.image = 'mitogen/%s-test' % (distro,) return self.image def __init__(self):