diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index fba7a86a..65f68efb 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -175,16 +175,10 @@ Noteworthy Differences your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c* -* The `buildah `_, - `docker `_, - `jail `_, - `kubectl `_, - `local `_, - `lxc `_, - `lxd `_, - and `ssh `_ - built-in connection types are supported, along with Mitogen-specific - :ref:`machinectl `, :ref:`mitogen_doas `, +* The :ans:conn:`~buildah`, :ans:conn:`~docker`, :ans:conn:`~jail`, + :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, and + :ans:conn:`~ssh` built-in connection types are supported, along with + Mitogen-specific :ref:`machinectl `, :ref:`mitogen_doas `, :ref:`mitogen_su `, :ref:`mitogen_sudo `, and :ref:`setns ` types. File bugs to register interest in others. @@ -199,16 +193,14 @@ Noteworthy Differences artificial serialization, causing slowdown equivalent to `task_duration * num_targets`. This will be addressed soon. -* The Ansible 2.7 `reboot - `_ module - may require a ``pre_reboot_delay`` on systemd hosts, as insufficient time - exists for the reboot command's exit status to be reported before necessary - processes are torn down. +* The Ansible 2.7 :ans:mod:`reboot` may require a ``pre_reboot_delay`` on + systemd hosts, as insufficient time exists for the reboot command's exit + status to be reported before necessary processes are torn down. * On OS X when a SSH password is specified and the default connection type of - ``smart`` is used, Ansible may select the Paramiko plug-in rather than - Mitogen. If you specify a password on OS X, ensure ``connection: ssh`` - appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the + :ans:conn:`~smart` is used, Ansible may select the :ans:conn:`paramiko_ssh` + rather than Mitogen. If you specify a password on OS X, ensure ``connection: + ssh`` appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the command-line. * Ansible permits up to ``forks`` connections to be setup in parallel, whereas @@ -345,19 +337,12 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally `sftp(1)`_ or `scp(1)`_ are used to copy files by the -`assemble `_, -`copy `_, -`patch `_, -`script `_, -`template `_, and -`unarchive `_ -actions, or when uploading modules with pipelining disabled. With Mitogen -copies are implemented natively using the same interpreters, connection tree, -and routed message bus that carries RPCs. - -.. _scp(1): https://linux.die.net/man/1/scp -.. _sftp(1): https://linux.die.net/man/1/sftp +Normally :linux:man1:`sftp` or :linux:man1:`scp` are used to copy files by the +:ans:mod:`~assemble`, :ans:mod:`~aws_s3`, :ans:mod:`~copy`, :ans:mod:`~patch`, +:ans:mod:`~script`, :ans:mod:`~template`, :ans:mod:`~unarchive`, and +:ans:mod:`~uri` actions, or when uploading modules with pipelining disabled. +With Mitogen copies are implemented natively using the same interpreters, +connection tree, and routed message bus that carries RPCs. This permits direct streaming between endpoints regardless of execution environment, without necessitating temporary copies in intermediary accounts or @@ -373,15 +358,15 @@ Safety ^^^^^^ Transfers proceed to a hidden file in the destination directory, with content -and metadata synced using `fsync(2) `_ prior -to rename over any existing file. This ensures the file remains consistent at -all times, in the event of a crash, or when overlapping `ansible-playbook` runs -deploy differing file contents. +and metadata synced using :linux:man2:`fsync` prior to rename over any existing +file. This ensures the file remains consistent at all times, in the event of a +crash, or when overlapping `ansible-playbook` runs deploy differing file +contents. -The `sftp(1)`_ and `scp(1)`_ tools may cause undetected data corruption -in the form of truncated files, or files containing intermingled data segments -from overlapping runs. As part of normal operation, both tools expose a window -where readers may observe inconsistent file contents. +The :linux:man1:`sftp` and :linux:man1:`scp` tools may cause undetected data +corruption in the form of truncated files, or files containing intermingled +data segments from overlapping runs. As part of normal operation, both tools +expose a window where readers may observe inconsistent file contents. Performance @@ -499,11 +484,11 @@ Ansible may: * Create a directory owned by the SSH user either under ``remote_tmp``, or a system-default directory, * Upload action dependencies such as non-new style modules or rendered - templates to that directory via `sftp(1)`_ or `scp(1)`_. + templates to that directory via :linux:man1:`sftp` or :linux:man1:`scp`. * Attempt to modify the directory's access control list to grant access to the - target user using `setfacl(1) `_, - requiring that tool to be installed and a supported filesystem to be in use, - or for the ``allow_world_readable_tmpfiles`` setting to be :data:`True`. + target user using :linux:man1:`setfacl`, requiring that tool to be installed + and a supported filesystem to be in use, or for the + ``allow_world_readable_tmpfiles`` setting to be :data:`True`. * Create a directory owned by the target user either under ``remote_tmp``, or a system-default directory, if a new-style module needs a temporary directory and one was not previously created for a supporting file earlier in the @@ -569,9 +554,9 @@ in regular Ansible: operations relating to modifying the directory to support cross-account access are avoided. -* An explicit work-around is included to avoid the `copy` and `template` - actions needlessly triggering a round-trip to set their temporary file as - executable. +* An explicit work-around is included to avoid the :ans:mod:`~copy` and + :ans:mod:`~template` actions needlessly triggering a round-trip to set their + temporary file as executable. * During task shutdown, it is not necessary to wait to learn if the target has succeeded in deleting a temporary directory, since any error that may occur @@ -601,10 +586,10 @@ DNS Resolution ^^^^^^^^^^^^^^ Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to -be reloaded via `res_init(3) `_. This -isn't necessary on some Linux distributions carrying glibc patches to -automatically check ``/etc/resolv.conf`` periodically, however it is necessary -on at least Debian and BSD derivatives. +be reloaded via :linux:man3:`res_init`. This isn't necessary on some Linux +distributions carrying glibc patches to automatically check +``/etc/resolv.conf`` periodically, however it is necessary on at least Debian +and BSD derivatives. ``/etc/environment`` @@ -728,9 +713,7 @@ configuration of each task. Buildah ~~~~~~~ -Like `buildah -`_ except -connection delegation is supported. +Like the :ans:conn:`buildah` except connection delegation is supported. * ``ansible_host``: Name of Buildah container (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. @@ -771,9 +754,7 @@ When used as the ``mitogen_doas`` connection method: Docker ~~~~~~ -Like `docker -`_ except -connection delegation is supported. +Like the :ans:conn:`docker` except connection delegation is supported. * ``ansible_host``: Name of Docker container (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. @@ -789,9 +770,7 @@ connection delegation is supported. FreeBSD Jail ~~~~~~~~~~~~ -Like `jail -`_ except -connection delegation is supported. +Like the :ans:conn:`jail` except connection delegation is supported. * ``ansible_host``: Name of jail (default: inventory hostname). * ``ansible_user``: Name of user within the jail to execute as. @@ -807,9 +786,7 @@ connection delegation is supported. Kubernetes Pod ~~~~~~~~~~~~~~ -Like `kubectl -`_ except -connection delegation is supported. +Like the :ans:conn:`kubectl` except connection delegation is supported. * ``ansible_host``: Name of pod (default: inventory hostname). * ``ansible_user``: Name of user to authenticate to API as. @@ -823,9 +800,7 @@ connection delegation is supported. Local ~~~~~ -Like `local -`_ except -connection delegation is supported. +Like the :ans:conn:`local` except connection delegation is supported. * ``ansible_python_interpreter`` @@ -852,10 +827,9 @@ additional differences exist that may break existing playbooks. LXC ~~~ -Connect to classic LXC containers, like `lxc -`_ except -connection delegation is supported, and ``lxc-attach`` is always used rather -than the LXC Python bindings, as is usual with ``lxc``. +Connect to classic LXC containers, like the :ans:conn:`lxc` except connection +delegation is supported, and ``lxc-attach`` is always used rather than the LXC +Python bindings, as is usual with ``lxc``. * ``ansible_python_interpreter`` * ``ansible_host``: Name of LXC container (default: inventory hostname). @@ -873,10 +847,9 @@ than the LXC Python bindings, as is usual with ``lxc``. LXD ~~~ -Connect to modern LXD containers, like `lxd -`_ except -connection delegation is supported. The ``lxc`` command must be available on -the host machine. +Connect to modern LXD containers, like the :ans:conn:`lxd` except connection +delegation is supported. The ``lxc`` command must be available on the host +machine. * ``ansible_python_interpreter`` * ``ansible_host``: Name of LXC container (default: inventory hostname). @@ -1001,8 +974,7 @@ When used as the ``mitogen_sudo`` connection method: SSH ~~~ -Like `ssh `_ -except connection delegation is supported. +Like the :ans:conn:`ssh` except connection delegation is supported. * ``ansible_ssh_timeout`` * ``ansible_host``, ``ansible_ssh_host`` diff --git a/docs/changelog.rst b/docs/changelog.rst index 998821e2..51cdd2df 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,35 +24,32 @@ To avail of fixes in an unreleased version, please download a ZIP file Enhancements ~~~~~~~~~~~~ -* `#556 `_, - `#587 `_: Ansible 2.8 is partially +* :gh:issue:`556`, + :gh:issue:`587`: Ansible 2.8 is partially supported. `Become plugins `_ and `interpreter discovery `_ are not yet handled. -* `#419 `_, - `#470 `_, file descriptor usage - during large runs is halved, as it is no longer necessary to manage read and - write sides distinctly in order to work around a design problem. +* :gh:issue:`419`, :gh:issue:`470`, file descriptor usage during large runs is + halved, as it is no longer necessary to manage read and write sides + distinctly in order to work around a design problem. -* `#419 `_: almost all connection - setup happens on one thread, reducing contention and context switching early - in a run. +* :gh:issue:`419`: almost all connection setup happens on one thread, reducing + contention and context switching early in a run. -* `#419 `_: Connection setup is - better pipelined, eliminating some network round-trips. Most infrastructure - is in place to support future removal of the final round-trips between a - target fully booting and receiving function calls. +* :gh:issue:`419`: Connection setup is better pipelined, eliminating some + network round-trips. Most infrastructure is in place to support future + removal of the final round-trips between a target fully booting and receiving + function calls. -* `#595 `_: the - :meth:`Router.buildah() ` connection method is - available to manipulate `Buildah `_ containers, and is - exposed to Ansible as the ``buildah`` transport. +* :gh:pull:`595`: the :meth:`~mitogen.parent.Router.buildah` connection method + is available to manipulate `Buildah `_ containers, and + is exposed to Ansible as the :ans:conn:`buildah`. -* `#615 `_: the ``mitogen_fetch`` - action is included, and the standard Ansible ``fetch`` action is redirected +* :gh:issue:`615`: the ``mitogen_fetch`` + action is included, and the standard Ansible :ans:mod:`fetch` is redirected to it. This implements streaming file transfer in every case, including when ``become`` is active, preventing excessive CPU usage and memory spikes, and significantly improving throughput. A copy of 2 files of 512 MiB each drops @@ -67,10 +64,10 @@ Enhancements involving large file transfers, and is required for future in-process SSH support. One multiplexer starts by default, to match existing behaviour. -* `d6faff06 `_, - `807cbef9 `_, - `e93762b3 `_, - `50bfe4c7 `_: locking is +* :gh:commit:`d6faff06`, + :gh:commit:`807cbef9`, + :gh:commit:`e93762b3`, + :gh:commit:`50bfe4c7`: locking is avoided on hot paths, and some locks are released earlier, before waking a thread that must immediately take the same lock. @@ -78,20 +75,18 @@ Enhancements Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#363 `_: fix an obscure race - matching *Permission denied* errors from some versions of ``su`` running on - heavily loaded machines. +* :gh:issue:`363`: fix an obscure race matching *Permission denied* errors from + some versions of ``su`` running on heavily loaded machines. -* `#410 `_: Use of ``AF_UNIX`` - sockets automatically replaced with plain UNIX pipes when SELinux is - detected, to work around a broken heuristic in popular SELinux policies that - prevents inheriting ``AF_UNIX`` sockets across privilege domains. +* :gh:issue:`410`: Uses of :linux:man7:`unix` sockets are replaced with + traditional :linux:man7:`pipe` pairs when SELinux is detected, to work around + a broken heuristic in popular SELinux policies that prevents inheriting + :linux:man7:`unix` sockets across privilege domains. * `#467 `_: an incompatibility running Mitogen under Molecule was resolved. -* `#547 `_, - `#598 `_: fix a serious deadlock +* :gh:issue:`547`, :gh:issue:`598`: fix a serious deadlock possible while initializing the service pool of any child, such as during connection, ``async`` tasks, tasks using custom :mod:`module_utils`, ``mitogen_task_isolation: fork`` modules, and those present on an internal @@ -102,60 +97,57 @@ Mitogen for Ansible a *Connection timed out* error, for forked tasks it could manifest as a timeout or an apparent hang. -* `#549 `_: the open file descriptor - limit for the Ansible process is increased to the available hard limit. It is - common for distributions to ship with a much higher hard limit than their - default soft limit, allowing *"too many open files"* errors to be avoided - more often in large runs without user configuration. - -* `#558 `_, - `#582 `_: on Ansible 2.3 a remote - directory was unconditionally deleted after the first module belonging to an - action plug-in had executed, causing the ``unarchive`` module to fail. - -* `#578 `_: the extension could crash - while rendering an error message, due to an incorrect format string. - -* `#590 `_: the importer can handle - modules that replace themselves in :mod:`sys.modules` during import. - -* `#591 `_: the target's current - working directory is restored to a known-existent directory between tasks to - ensure :func:`os.getcwd` will not fail when called, in the same way that - :class:`AnsibleModule` restores it during initialization. However this - restore happens before the module ever executes, ensuring any code that calls - :func:`os.getcwd` prior to :class:`AnsibleModule` initialization, such as the - Ansible 2.7 ``pip`` module, cannot fail due to the behavior of a prior task. - -* `#593 `_: the SSH connection method - exposes ``mitogen_ssh_keepalive_interval`` and - ``mitogen_ssh_keepalive_count`` variables, and the default timeout for an SSH - server has been increased from `15*3` seconds to `30*10` seconds. - -* `#600 `_: functionality to reflect - changes to ``/etc/environment`` did not account for Unicode file contents. - The file may now use any single byte encoding. - -* `#602 `_: connection configuration - is more accurately inferred for `meta: reset_connection`, the `synchronize` - module, and for any action plug-ins that establish additional connections. - -* `#598 `_, - `#605 `_: fix a deadlock managing a - shared counter used for load balancing. - -* `#615 `_: streaming file transfer - is implemented for ``fetch`` and other actions that transfer files from the - target to the controller. Previously the file was sent in one message, - requiring it to fit in RAM and be smaller than the internal message size - limit. - -* `7ae926b3 `_: the - ``lineinfile`` module began leaking writable temporary file descriptors since - Ansible 2.7.0. When ``lineinfile`` was used to create or modify a script, and - that script was later executed, the execution could fail with "*text file - busy*" due to the leaked descriptor. Temporary descriptors are now tracked - and cleaned up on exit for all modules. +* :gh:issue:`549`: the open file descriptor limit for the Ansible process is + increased to the available hard limit. It is common for distributions to ship + with a much higher hard limit than their default soft limit, allowing *"too + many open files"* errors to be avoided more often in large runs without user + configuration. + +* :gh:issue:`558`, :gh:issue:`582`: on Ansible 2.3 a remote directory was + unconditionally deleted after the first module belonging to an action plug-in + had executed, causing the :ans:mod:`unarchive` module to fail. + +* :gh:issue:`578`: the extension could crash while rendering an error message, + due to an incorrect format string. + +* :gh:issue:`590`: the importer can handle modules that replace themselves in + :data:`sys.modules` during import. + +* :gh:issue:`591`: the target's current working directory is restored to a + known-existent directory between tasks to ensure :func:`os.getcwd` will not + fail when called, in the same way that :class:`AnsibleModule` restores it + during initialization. However this restore happens before the module ever + executes, ensuring any code that calls :func:`os.getcwd` prior to + :class:`AnsibleModule` initialization, such as the Ansible 2.7 + :ans:mod:`pip`, cannot fail due to the behavior of a prior task. + +* :gh:issue:`593`: the SSH connection method exposes + ``mitogen_ssh_keepalive_interval`` and ``mitogen_ssh_keepalive_count`` + variables, and the default timeout for an SSH server has been increased from + `15*3` seconds to `30*10` seconds. + +* :gh:issue:`600`: functionality to reflect changes to ``/etc/environment`` did + not account for Unicode file contents. The file may now use any single byte + encoding. + +* :gh:issue:`602`: connection configuration is more accurately inferred for + `meta: reset_connection`, the `synchronize` module, and for any action + plug-ins that establish additional connections. + +* :gh:issue:`598`, :gh:issue:`605`: fix a deadlock managing a shared counter + used for load balancing. + +* :gh:issue:`615`: streaming file transfer is implemented for ``fetch`` and + other actions that transfer files from the target to the controller. + Previously the file was sent in one message, requiring it to fit in RAM and + be smaller than the internal message size limit. + +* :gh:commit:`7ae926b3`: the Ansible :ans:mod:`lineinfile` began leaking + writable temporary file descriptors since Ansible 2.7.0. When + :ans:mod:`~lineinfile` was used to create or modify a script, and that script + was later executed, the execution could fail with "*text file busy*" due to + the leaked descriptor. Temporary descriptors are now tracked and cleaned up + on exit for all modules. Core Library @@ -170,80 +162,86 @@ Core Library It was never portable between Python versions, unused, and never made much sense to support. -* `#170 `_: to improve subprocess - management and asynchronous connect, a :class:`mitogen.parent.TimerList` +* :gh:issue:`170`: to improve subprocess + management and asynchronous connect, a :class:`~mitogen.parent.TimerList` interface is available, accessible as :attr:`Broker.timers` in an asynchronous context. -* `#419 `_: the internal - :class:`mitogen.core.Stream` has been refactored into 7 new classes, +* :gh:issue:`419`: the internal + :class:`~mitogen.core.Stream` has been refactored into 7 new classes, modularizing protocol behaviour, output buffering, line-oriented input parsing, option handling and connection management. Connection setup is internally asynchronous, laying almost all the groundwork needed for fully asynchronous connect, proxied Ansible become plug-ins, and integrating `libssh `_. -* `#169 `_, - `#419 `_: zombie child reaping has - vastly improved, by using timers to efficiently poll for a slow child to - finish exiting. Polling avoids relying on process-global configuration such - as a `SIGCHLD` handler, or :func:`signal.set_wakeup_fd` available in modern - Python. +* :gh:issue:`169`, + :gh:issue:`419`: zombie subprocess reaping + has vastly improved, by using timers to efficiently poll for a slow child to + finish exiting, and delaying broker shutdown while any subprocess remains. + Polling avoids relying on process-global configuration such as a `SIGCHLD` + handler, or :func:`signal.set_wakeup_fd` available in modern Python. -* `#256 `_, - `#419 `_: most :func:`os.dup` use +* :gh:issue:`256`, + :gh:issue:`419`: most :func:`os.dup` use was eliminated, along with almost all manual file descriptor management. Descriptors are trapped in :func:`os.fdopen` objects at creation, ensuring a leaked object will close itself, and ensuring every descriptor is fused to a `closed` flag, preventing historical bugs where a double close could destroy descriptors belonging to unrelated streams. -* `#533 `_: routing accounts for - a race between a parent sending a message to a child via an intermediary, - where the child had recently disconnected, and ``DEL_ROUTE`` propagating from - the intermediary to the parent, informing it that the child no longer exists. - This condition is detected at the intermediary and a dead message is returned - to the parent. +* :gh:issue:`533`: routing accounts for + a race between a parent (or cousin) sending a message to a child via an + intermediary, where the child had recently disconnected, and + :data:`~mitogen.core.DEL_ROUTE` propagating from the intermediary + to the sender, informing it that the child no longer exists. This condition + is detected at the intermediary and a :ref:`dead message ` is + returned to the sender. Previously since the intermediary had already removed its route for the child, the *route messages upwards* rule would be triggered, causing the - message (with a privileged ``src_id``/``auth_id``) to be sent upstream, - resulting in a ``bad auth_id`` log message and a hang. + message (with a privileged :ref:`src_id/auth_id `) to be + sent upstream, resulting in a ``bad auth_id`` error logged at the first + upstream parent, and a possible hang due to a request message being dropped. -* `#586 `_: fix import of +* :gh:issue:`586`: fix import of :mod:`__main__` on later versions of Python 3 when running from the interactive console. -* `#606 `_: fix example code on the +* :gh:issue:`606`: fix example code on the documentation front page. -* `#612 `_: fix various errors +* :gh:issue:`612`: fix various errors introduced by stream refactoring. -* `#615 `_: when routing fails to +* :gh:issue:`615`: when routing fails to deliver a message for some reason other than the sender cannot or should not reach the recipient, and no reply-to address is present on the message, - instead send a dead message to the original recipient. This ensures a - descriptive messages is delivered to a thread sleeping on the reply to a - function call, where the reply might be dropped due to exceeding the maximum - configured message size. + instead send a :ref:`dead message ` to the original recipient. This + ensures a descriptive messages is delivered to a thread sleeping on the reply + to a function call, where the reply might be dropped due to exceeding the + maximum configured message size. + +* :gh:issue:`624`: the number of threads used for a child's auto-started thread + pool has been reduced from 16 to 2. This may drop to 1 in future, and become + configurable via a :class:`Router` option. -* `a5536c35 `_: avoid quadratic +* :gh:commit:`a5536c35`: avoid quadratic buffer management when logging lines received from a child's redirected standard IO. -* `49a6446a `_: the - :meth:`empty` method of :class:`mitogen.core.Latch`, - :class:`mitogen.core.Receiver` and :class:`mitogen.select.Select` has been - replaced by a more general :meth:`size` method. :meth:`empty` will be removed - in 0.3 +* :gh:commit:`49a6446a`: the + :meth:`empty` methods of :class:`~mitogen.core.Latch`, + :class:`~mitogen.core.Receiver` and :class:`~mitogen.select.Select` are + obsoleted by a more general :meth:`size` method. :meth:`empty` will be + removed in 0.3 -* `ecc570cb `_: previously +* :gh:commit:`ecc570cb`: previously :meth:`mitogen.select.Select.add` would enqueue one wake event when adding an existing receiver, latch or subselect that contained multiple buffered items, causing :meth:`get` calls to block or fail even though data existed to return. -* `5924af15 `_: *[security]* +* :gh:commit:`5924af15`: *[security]* unidirectional routing, where contexts may optionally only communicate with parents and never siblings (so that air-gapped networks cannot be unintentionally bridged) was not inherited when a child was initiated @@ -294,27 +292,27 @@ on Ansible 2.8, which is not yet supported. Fixes ~~~~~ -* `#557 `_: fix a crash when running +* :gh:issue:`557`: fix a crash when running on machines with high CPU counts. -* `#570 `_: the ``firewalld`` module - internally caches a dbus name that changes across ``firewalld`` restarts, - causing a failure if the service is restarted between ``firewalld`` module invocations. +* :gh:issue:`570`: the :ans:mod:`firewalld` internally caches a dbus name that + changes across :ans:mod:`~firewalld` restarts, causing a failure if the + service is restarted between :ans:mod:`~firewalld` module invocations. -* `#575 `_: fix a crash when +* :gh:issue:`575`: fix a crash when rendering an error message to indicate no usable temporary directories could be found. -* `#576 `_: fix a crash during +* :gh:issue:`576`: fix a crash during startup on SuSE Linux 11, due to an incorrect version compatibility check in the Mitogen code. -* `#581 `_: a +* :gh:issue:`581`: a ``mitogen_mask_remote_name`` Ansible variable is exposed, to allow masking the username, hostname and process ID of ``ansible-playbook`` running on the controller machine. -* `#587 `_: display a friendly +* :gh:issue:`587`: display a friendly message when running on an unsupported version of Ansible, to cope with potential influx of 2.8-related bug reports. @@ -336,53 +334,53 @@ v0.2.6 (2019-03-06) Fixes ~~~~~ -* `#542 `_: some versions of OS X +* :gh:issue:`542`: some versions of OS X ship a default Python that does not support :func:`select.poll`. Restore the 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer :func:`select.poll` if it is available. -* `#545 `_: an optimization - introduced in `#493 `_ caused a +* :gh:issue:`545`: an optimization + introduced in :gh:issue:`493` caused a 64-bit integer to be assigned to a 32-bit field on ARM 32-bit targets, causing runs to fail. -* `#548 `_: `mitogen_via=` could fail +* :gh:issue:`548`: `mitogen_via=` could fail when the selected transport was set to ``smart``. -* `#550 `_: avoid some broken +* :gh:issue:`550`: avoid some broken TTY-related `ioctl()` calls on Windows Subsystem for Linux 2016 Anniversary Update. -* `#554 `_: third party Ansible +* :gh:issue:`554`: third party Ansible action plug-ins that invoked :func:`_make_tmp_path` repeatedly could trigger an assertion failure. -* `#555 `_: work around an old idiom +* :gh:issue:`555`: work around an old idiom that reloaded :mod:`sys` in order to change the interpreter's default encoding. -* `ffae0355 `_: needless +* :gh:commit:`ffae0355`: needless information was removed from the documentation and installation procedure. Core Library ~~~~~~~~~~~~ -* `#535 `_: to support function calls +* :gh:issue:`535`: to support function calls on a service pool from another thread, :class:`mitogen.select.Select` additionally permits waiting on :class:`mitogen.core.Latch`. -* `#535 `_: +* :gh:issue:`535`: :class:`mitogen.service.Pool.defer` allows any function to be enqueued for the thread pool from another thread. -* `#535 `_: a new +* :gh:issue:`535`: a new :mod:`mitogen.os_fork` module provides a :func:`os.fork` wrapper that pauses thread activity during fork. On Python<2.6, :class:`mitogen.core.Broker` and :class:`mitogen.service.Pool` automatically record their existence so that a :func:`os.fork` monkey-patch can automatically pause them for any attempt to start a subprocess. -* `ca63c26e `_: +* :gh:commit:`ca63c26e`: :meth:`mitogen.core.Latch.put`'s `obj` argument was made optional. @@ -407,47 +405,47 @@ v0.2.5 (2019-02-14) Fixes ~~~~~ -* `#511 `_, - `#536 `_: changes in 0.2.4 to +* :gh:issue:`511`, + :gh:issue:`536`: changes in 0.2.4 to repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. -* `#532 `_: fix a race in the service +* :gh:issue:`532`: fix a race in the service used to propagate Ansible modules, that could easily manifest when starting asynchronous tasks in a loop. -* `#536 `_: changes in 0.2.4 to +* :gh:issue:`536`: changes in 0.2.4 to support Python 2.4 interacted poorly with modules that imported ``simplejson`` from a controller that also loaded an incompatible newer version of ``simplejson``. -* `#537 `_: a swapped operator in the +* :gh:issue:`537`: a swapped operator in the CPU affinity logic meant 2 cores were reserved on 1`_: the source distribution +* :gh:issue:`538`: the source distribution includes a ``LICENSE`` file. -* `#539 `_: log output is no longer +* :gh:issue:`539`: log output is no longer duplicated when the Ansible ``log_path`` setting is enabled. -* `#540 `_: the ``stderr`` stream of +* :gh:issue:`540`: the ``stderr`` stream of async module invocations was previously discarded. -* `#541 `_: Python error logs +* :gh:issue:`541`: Python error logs originating from the ``boto`` package are quiesced, and only appear in ``-vvv`` output. This is since EC2 modules may trigger errors during normal operation, when retrying transiently failing requests. -* `748f5f67 `_, - `21ad299d `_, - `8ae6ca1d `_, - `7fd0d349 `_: +* :gh:commit:`748f5f67`, + :gh:commit:`21ad299d`, + :gh:commit:`8ae6ca1d`, + :gh:commit:`7fd0d349`: the ``ansible_ssh_host``, ``ansible_ssh_user``, ``ansible_user``, ``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly match typical behaviour when ``mitogen_via=`` is active. -* `2a8567b4 `_: fix a race +* :gh:commit:`2a8567b4`: fix a race initializing a child's service thread pool on Python 3.4+, due to a change in locking scheme used by the Python import mechanism. @@ -475,57 +473,54 @@ on the connection multiplexer. Enhancements ^^^^^^^^^^^^ -* `#76 `_, - `#351 `_, - `#352 `_: disconnect propagation +* :gh:issue:`76`, + :gh:issue:`351`, + :gh:issue:`352`: disconnect propagation has improved, allowing Ansible to cancel waits for responses from abruptly disconnected targets. This ensures a task will reliably fail rather than hang, for example on network failure or EC2 instance maintenance. -* `#369 `_, - `#407 `_: :meth:`Connection.reset` - is implemented, allowing `meta: reset_connection - `_ to shut +* :gh:issue:`369`, + :gh:issue:`407`: :meth:`Connection.reset` + is implemented, allowing :ans:mod:`meta: reset_connection ` to shut down the remote interpreter as documented, and improving support for the - `reboot - `_ - module. + :ans:mod:`reboot`. -* `09aa27a6 `_: the +* :gh:commit:`09aa27a6`: the ``mitogen_host_pinned`` strategy wraps the ``host_pinned`` strategy introduced in Ansible 2.7. -* `#477 `_: Python 2.4 is fully +* :gh:issue:`477`: Python 2.4 is fully supported by the core library and tested automatically, in any parent/child combination of 2.4, 2.6, 2.7 and 3.6 interpreters. -* `#477 `_: Ansible 2.3 is fully +* :gh:issue:`477`: Ansible 2.3 is fully supported and tested automatically. In combination with the core library Python 2.4 support, this allows Red Hat Enterprise Linux 5 targets to be managed with Mitogen. The ``simplejson`` package need not be installed on such targets, as is usually required by Ansible. -* `#412 `_: to simplify diagnosing +* :gh:issue:`412`: to simplify diagnosing connection configuration problems, Mitogen ships a ``mitogen_get_stack`` action that is automatically added to the action plug-in path. See :ref:`mitogen-get-stack` for more information. -* `152effc2 `_, - `bd4b04ae `_: a CPU affinity +* :gh:commit:`152effc2`, + :gh:commit:`bd4b04ae`: a CPU affinity policy was added for Linux controllers, reducing latency and SMP overhead on hot paths exercised for every task. This yielded a 19% speedup in a 64-target job composed of many short tasks, and should easily be visible as a runtime improvement in many-host runs. -* `2b44d598 `_: work around a +* :gh:commit:`2b44d598`: work around a defective caching mechanism by pre-heating it before spawning workers. This saves 40% runtime on a synthetic repetitive task. -* `0979422a `_: an expensive +* :gh:commit:`0979422a`: an expensive dependency scanning step was redundantly invoked for every task, bottlenecking the connection multiplexer. -* `eaa990a97 `_: a new +* :gh:commit:`eaa990a97`: a new ``mitogen_ssh_compression`` variable is supported, allowing Mitogen's default SSH compression to be disabled. SSH compression is a large contributor to CPU usage in many-target runs, and severely limits file transfer. On a `"shell: @@ -533,124 +528,115 @@ Enhancements task with compression, rising to 3 KiB without. File transfer throughput rises from ~25MiB/s when enabled to ~200MiB/s when disabled. -* `#260 `_, - `a18a083c `_: brokers no +* :gh:issue:`260`, + :gh:commit:`a18a083c`: brokers no longer wait for readiness indication to transmit, and instead assume transmission will succeed. As this is usually true, one loop iteration and two poller reconfigurations are avoided, yielding a significant reduction in interprocess round-trip latency. -* `#415 `_, - `#491 `_, - `#493 `_: the interface employed - for in-process queues changed from `kqueue - `_ / `epoll - `_ to `poll() - `_, which requires no setup - or teardown, yielding a 38% latency reduction for inter-thread communication. +* :gh:issue:`415`, :gh:issue:`491`, :gh:issue:`493`: the interface employed + for in-process queues changed from :freebsd:man2:`kqueue` / + :linux:man7:`epoll` to :linux:man2:`poll`, which requires no setup or + teardown, yielding a 38% latency reduction for inter-thread communication. Fixes ^^^^^ -* `#251 `_, - `#359 `_, - `#396 `_, - `#401 `_, - `#404 `_, - `#412 `_, - `#434 `_, - `#436 `_, - `#465 `_: connection delegation and +* :gh:issue:`251`, + :gh:issue:`359`, + :gh:issue:`396`, + :gh:issue:`401`, + :gh:issue:`404`, + :gh:issue:`412`, + :gh:issue:`434`, + :gh:issue:`436`, + :gh:issue:`465`: connection delegation and ``delegate_to:`` handling suffered a major regression in 0.2.3. The 0.2.2 behaviour has been restored, and further work has been made to improve the compatibility of connection delegation's configuration building methods. -* `#323 `_, - `#333 `_: work around a Windows +* :gh:issue:`323`, + :gh:issue:`333`: work around a Windows Subsystem for Linux bug that caused tracebacks to appear during shutdown. -* `#334 `_: the SSH method +* :gh:issue:`334`: the SSH method tilde-expands private key paths using Ansible's logic. Previously the path was passed unmodified to SSH, which expanded it using :func:`pwd.getpwnam`. This differs from :func:`os.path.expanduser`, which uses the ``HOME`` environment variable if it is set, causing behaviour to diverge when Ansible was invoked across user accounts via ``sudo``. -* `#364 `_: file transfers from +* :gh:issue:`364`: file transfers from controllers running Python 2.7.2 or earlier could be interrupted due to a forking bug in the :mod:`tempfile` module. -* `#370 `_: the Ansible - `reboot `_ - module is supported. +* :gh:issue:`370`: the Ansible :ans:mod:`reboot` is supported. -* `#373 `_: the LXC and LXD methods - print a useful hint on failure, as no useful error is normally logged to the - console by these tools. +* :gh:issue:`373`: the LXC and LXD methods print a useful hint on failure, as + no useful error is normally logged to the console by these tools. -* `#374 `_, - `#391 `_: file transfer and module +* :gh:issue:`374`, + :gh:issue:`391`: file transfer and module execution from 2.x controllers to 3.x targets was broken due to a regression - caused by refactoring, and compounded by `#426 - `_. + caused by refactoring, and compounded by :gh:issue:`426`. -* `#400 `_: work around a threading +* :gh:issue:`400`: work around a threading bug in the AWX display callback when running with high verbosity setting. -* `#409 `_: the setns method was +* :gh:issue:`409`: the setns method was silently broken due to missing tests. Basic coverage was added to prevent a recurrence. -* `#409 `_: the LXC and LXD methods +* :gh:issue:`409`: the LXC and LXD methods support ``mitogen_lxc_path`` and ``mitogen_lxc_attach_path`` variables to control the location of third pary utilities. -* `#410 `_: the sudo method supports +* :gh:issue:`410`: the sudo method supports the SELinux ``--type`` and ``--role`` options. -* `#420 `_: if a :class:`Connection` +* :gh:issue:`420`: if a :class:`Connection` was constructed in the Ansible top-level process, for example while executing ``meta: reset_connection``, resources could become undesirably shared in subsequent children. -* `#426 `_: an oversight while +* :gh:issue:`426`: an oversight while porting to Python 3 meant no automated 2->3 tests were running. A significant number of 2->3 bugs were fixed, mostly in the form of Unicode/bytes mismatches. -* `#429 `_: the ``sudo`` method can +* :gh:issue:`429`: the ``sudo`` method can now recognize internationalized password prompts. -* `#362 `_, - `#435 `_: the previous fix for slow +* :gh:issue:`362`, + :gh:issue:`435`: the previous fix for slow Python 2.x subprocess creation on Red Hat caused newly spawned children to have a reduced open files limit. A more intrusive fix has been added to directly address the problem without modifying the subprocess environment. -* `#397 `_, - `#454 `_: the previous approach to +* :gh:issue:`397`, + :gh:issue:`454`: the previous approach to handling modern Ansible temporary file cleanup was too aggressive, and could trigger early finalization of Cython-based extension modules, leading to segmentation faults. -* `#499 `_: the ``allow_same_user`` +* :gh:issue:`499`: the ``allow_same_user`` Ansible configuration setting is respected. -* `#527 `_: crashes in modules are +* :gh:issue:`527`: crashes in modules are trapped and reported in a manner that matches Ansible. In particular, a module crash no longer leads to an exception that may crash the corresponding action plug-in. -* `dc1d4251 `_: the - ``synchronize`` module could fail with the Docker transport due to a missing - attribute. +* :gh:commit:`dc1d4251`: the :ans:mod:`synchronize` could fail with the Docker + transport due to a missing attribute. -* `599da068 `_: fix a race +* :gh:commit:`599da068`: fix a race when starting async tasks, where it was possible for the controller to observe no status file on disk before the task had a chance to write one. -* `2c7af9f04 `_: Ansible +* :gh:commit:`2c7af9f04`: Ansible modules were repeatedly re-transferred. The bug was hidden by the previously mandatorily enabled SSH compression. @@ -658,7 +644,7 @@ Fixes Core Library ~~~~~~~~~~~~ -* `#76 `_: routing records the +* :gh:issue:`76`: routing records the destination context IDs ever received on each stream, and when disconnection occurs, propagates :data:`mitogen.core.DEL_ROUTE` messages towards every stream that ever communicated with the disappearing peer, rather than simply @@ -667,166 +653,166 @@ Core Library receivers to wake with :class:`mitogen.core.ChannelError`, even when one participant is not a parent of the other. -* `#109 `_, - `57504ba6 `_: newer Python 3 +* :gh:issue:`109`, + :gh:commit:`57504ba6`: newer Python 3 releases explicitly populate :data:`sys.meta_path` with importer internals, causing Mitogen to install itself at the end of the importer chain rather than the front. -* `#310 `_: support has returned for +* :gh:issue:`310`: support has returned for trying to figure out the real source of non-module objects installed in :data:`sys.modules`, so they can be imported. This is needed to handle syntax sugar used by packages like :mod:`plumbum`. -* `#349 `_: an incorrect format +* :gh:issue:`349`: an incorrect format string could cause large stack traces when attempting to import built-in modules on Python 3. -* `#387 `_, - `#413 `_: dead messages include an +* :gh:issue:`387`, + :gh:issue:`413`: dead messages include an optional reason in their body. This is used to cause :class:`mitogen.core.ChannelError` to report far more useful diagnostics at the point the error occurs that previously would have been buried in debug log output from an unrelated context. -* `#408 `_: a variety of fixes were +* :gh:issue:`408`: a variety of fixes were made to restore Python 2.4 compatibility. -* `#399 `_, - `#437 `_: ignore a +* :gh:issue:`399`, + :gh:issue:`437`: ignore a :class:`DeprecationWarning` to avoid failure of the ``su`` method on Python 3.7. -* `#405 `_: if an oversized message +* :gh:issue:`405`: if an oversized message is rejected, and it has a ``reply_to`` set, a dead message is returned to the sender. This ensures function calls exceeding the configured maximum size crash rather than hang. -* `#406 `_: +* :gh:issue:`406`: :class:`mitogen.core.Broker` did not call :meth:`mitogen.core.Poller.close` during shutdown, leaking the underlying poller FD in masters and parents. -* `#406 `_: connections could leak +* :gh:issue:`406`: connections could leak FDs when a child process failed to start. -* `#288 `_, - `#406 `_, - `#417 `_: connections could leave +* :gh:issue:`288`, + :gh:issue:`406`, + :gh:issue:`417`: connections could leave FD wrapper objects that had not been closed lying around to be closed during garbage collection, causing reused FD numbers to be closed at random moments. -* `#411 `_: the SSH method typed +* :gh:issue:`411`: the SSH method typed "``y``" rather than the requisite "``yes``" when `check_host_keys="accept"` was configured. This would lead to connection timeouts due to the hung response. -* `#414 `_, - `#425 `_: avoid deadlock of forked +* :gh:issue:`414`, + :gh:issue:`425`: avoid deadlock of forked children by reinitializing the :mod:`mitogen.service` pool lock. -* `#416 `_: around 1.4KiB of memory +* :gh:issue:`416`: around 1.4KiB of memory was leaked on every RPC, due to a list of strong references keeping alive any handler ever registered for disconnect notification. -* `#418 `_: the +* :gh:issue:`418`: the :func:`mitogen.parent.iter_read` helper would leak poller FDs, because execution of its :keyword:`finally` block was delayed on Python 3. Now callers explicitly close the generator when finished. -* `#422 `_: the fork method could +* :gh:issue:`422`: the fork method could fail to start if :data:`sys.stdout` was opened in block buffered mode, and buffered data was pending in the parent prior to fork. -* `#438 `_: a descriptive error is +* :gh:issue:`438`: a descriptive error is logged when stream corruption is detected. -* `#439 `_: descriptive errors are +* :gh:issue:`439`: descriptive errors are raised when attempting to invoke unsupported function types. -* `#444 `_: messages regarding +* :gh:issue:`444`: messages regarding unforwardable extension module are no longer logged as errors. -* `#445 `_: service pools unregister +* :gh:issue:`445`: service pools unregister the :data:`mitogen.core.CALL_SERVICE` handle at shutdown, ensuring any outstanding messages are either processed by the pool as it shuts down, or have dead messages sent in reply to them, preventing peer contexts from hanging due to a forgotten buffered message. -* `#446 `_: given thread A calling +* :gh:issue:`446`: given thread A calling :meth:`mitogen.core.Receiver.close`, and thread B, C, and D sleeping in :meth:`mitogen.core.Receiver.get`, previously only one sleeping thread would be woken with :class:`mitogen.core.ChannelError` when the receiver was closed. Now all threads are woken per the docstring. -* `#447 `_: duplicate attempts to +* :gh:issue:`447`: duplicate attempts to invoke :meth:`mitogen.core.Router.add_handler` cause an error to be raised, ensuring accidental re-registration of service pools are reported correctly. -* `#448 `_: the import hook +* :gh:issue:`448`: the import hook implementation now raises :class:`ModuleNotFoundError` instead of :class:`ImportError` in Python 3.6 and above, to cope with an upcoming version of the :mod:`subprocess` module requiring this new subclass to be raised. -* `#453 `_: the loggers used in +* :gh:issue:`453`: the loggers used in children for standard IO redirection have propagation disabled, preventing accidental reconfiguration of the :mod:`logging` package in a child from setting up a feedback loop. -* `#456 `_: a descriptive error is +* :gh:issue:`456`: a descriptive error is logged when :meth:`mitogen.core.Broker.defer` is called after the broker has shut down, preventing new messages being enqueued that will never be sent, and subsequently producing a program hang. -* `#459 `_: the beginnings of a +* :gh:issue:`459`: the beginnings of a :meth:`mitogen.master.Router.get_stats` call has been added. The initial statistics cover the module loader only. -* `#462 `_: Mitogen could fail to +* :gh:issue:`462`: Mitogen could fail to open a PTY on broken Linux systems due to a bad interaction between the glibc :func:`grantpt` function and an incorrectly mounted ``/dev/pts`` filesystem. Since correct group ownership is not required in most scenarios, when this problem is detected, the PTY is allocated and opened directly by the library. -* `#479 `_: Mitogen could fail to +* :gh:issue:`479`: Mitogen could fail to import :mod:`__main__` on Python 3.4 and newer due to a breaking change in the :mod:`pkgutil` API. The program's main script is now handled specially. -* `#481 `_: the version of `sudo` +* :gh:issue:`481`: the version of `sudo` that shipped with CentOS 5 replaced itself with the program to be executed, and therefore did not hold any child PTY open on our behalf. The child context is updated to preserve any PTY FD in order to avoid the kernel sending `SIGHUP` early during startup. -* `#523 `_: the test suite didn't +* :gh:issue:`523`: the test suite didn't generate a code coverage report if any test failed. -* `#524 `_: Python 3.6+ emitted a +* :gh:issue:`524`: Python 3.6+ emitted a :class:`DeprecationWarning` for :func:`mitogen.utils.run_with_router`. -* `#529 `_: Code coverage of the +* :gh:issue:`529`: Code coverage of the test suite was not measured across all Python versions. -* `16ca111e `_: handle OpenSSH +* :gh:commit:`16ca111e`: handle OpenSSH 7.5 permission denied prompts when ``~/.ssh/config`` rewrites are present. -* `9ec360c2 `_: a new +* :gh:commit:`9ec360c2`: a new :meth:`mitogen.core.Broker.defer_sync` utility function is provided. -* `f20e0bba `_: +* :gh:commit:`f20e0bba`: :meth:`mitogen.service.FileService.register_prefix` permits granting unprivileged access to whole filesystem subtrees, rather than single files at a time. -* `8f85ee03 `_: +* :gh:commit:`8f85ee03`: :meth:`mitogen.core.Router.myself` returns a :class:`mitogen.core.Context` referring to the current process. -* `824c7931 `_: exceptions +* :gh:commit:`824c7931`: exceptions raised by the import hook were updated to include probable reasons for a failure. -* `57b652ed `_: a stray import +* :gh:commit:`57b652ed`: a stray import meant an extra roundtrip and ~4KiB of data was wasted for any context that imported :mod:`mitogen.parent`. @@ -883,51 +869,40 @@ Mitogen for Ansible Enhancements ^^^^^^^^^^^^ -* `#315 `_, - `#392 `_: Ansible 2.6 and 2.7 are +* :gh:pull:`315`, + :gh:issue:`392`: Ansible 2.6 and 2.7 are supported. -* `#321 `_, - `#336 `_: temporary file handling - was simplified, undoing earlier damage caused by compatibility fixes, - improving 2.6 compatibility, and avoiding two network roundtrips for every - related action - (`assemble `_, - `aws_s3 `_, - `copy `_, - `patch `_, - `script `_, - `template `_, - `unarchive `_, - `uri `_). See - :ref:`ansible_tempfiles` for a complete description. - -* `#376 `_, - `#377 `_: the ``kubectl`` connection - type is now supported. Contributed by Yannig Perré. - -* `084c0ac0 `_: avoid a - roundtrip in - `copy `_ and - `template `_ - due to an unfortunate default. - -* `7458dfae `_: avoid a +* :gh:issue:`321`, :gh:issue:`336`: temporary file handling was simplified, + undoing earlier damage caused by compatibility fixes, improving 2.6 + compatibility, and avoiding two network roundtrips for every related action + (:ans:mod:`~assemble`, :ans:mod:`~aws_s3`, :ans:mod:`~copy`, + :ans:mod:`~patch`, :ans:mod:`~script`, :ans:mod:`~template`, + :ans:mod:`~unarchive`, :ans:mod:`~uri`). See :ref:`ansible_tempfiles` for a + complete description. + +* :gh:pull:`376`, :gh:pull:`377`: the ``kubectl`` connection type is now + supported. Contributed by Yannig Perré. + +* :gh:commit:`084c0ac0`: avoid a roundtrip in :ans:mod:`~copy` and + :ans:mod:`~template` due to an unfortunate default. + +* :gh:commit:`7458dfae`: avoid a roundtrip when transferring files smaller than 124KiB. Copy and template actions are now 2-RTT, reducing runtime for a 20-iteration template loop over a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 seconds compared to vanilla. -* `#337 `_: To avoid a scaling +* :gh:issue:`337`: To avoid a scaling limitation, a PTY is no longer allocated for an SSH connection unless the configuration specifies a password. -* `d62e6e2a `_: many-target +* :gh:commit:`d62e6e2a`: many-target runs executed the dependency scanner redundantly due to missing synchronization, wasting significant runtime in the connection multiplexer. In one case work was reduced by 95%, which may manifest as faster runs. -* `5189408e `_: threads are +* :gh:commit:`5189408e`: threads are cooperatively scheduled, minimizing `GIL `_ contention, and reducing context switching by around 90%. This manifests as an overall @@ -946,62 +921,62 @@ Enhancements Fixes ^^^^^ -* `#251 `_, - `#340 `_: Connection Delegation +* :gh:issue:`251`, + :gh:issue:`340`: Connection Delegation could establish connections to the wrong target when ``delegate_to:`` is present. -* `#291 `_: when Mitogen had +* :gh:issue:`291`: when Mitogen had previously been installed using ``pip`` or ``setuptools``, the globally installed version could conflict with a newer version bundled with an extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. -* `#324 `_: plays with a +* :gh:issue:`324`: plays with a `custom module_utils `_ would fail due to fallout from the Python 3 port and related tests being disabled. -* `#331 `_: the connection +* :gh:issue:`331`: the connection multiplexer subprocess always exits before the main Ansible process, ensuring logs generated by it do not overwrite the user's prompt when ``-vvv`` is enabled. -* `#332 `_: support a new +* :gh:issue:`332`: support a new :func:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. -* `#338 `_: compatibility: changes to +* :gh:issue:`338`: compatibility: changes to ``/etc/environment`` and ``~/.pam_environment`` made by a task are reflected in the runtime environment of subsequent tasks. See :ref:`ansible_process_env` for a complete description. -* `#343 `_: the sudo ``--login`` +* :gh:issue:`343`: the sudo ``--login`` option is supported. -* `#344 `_: connections no longer +* :gh:issue:`344`: connections no longer fail when the controller's login username contains slashes. -* `#345 `_: the ``IdentitiesOnly +* :gh:issue:`345`: the ``IdentitiesOnly yes`` option is no longer supplied to OpenSSH by default, better matching Ansible's behaviour. -* `#355 `_: tasks configured to run +* :gh:issue:`355`: tasks configured to run in an isolated forked subprocess were forked from the wrong parent context. This meant built-in modules overridden via a custom ``module_utils`` search path may not have had any effect. -* `#362 `_: to work around a slow +* :gh:issue:`362`: to work around a slow algorithm in the :mod:`subprocess` module, the maximum number of open files in processes running on the target is capped to 512, reducing the work required to start a subprocess by >2000x in default CentOS configurations. -* `#397 `_: recent Mitogen master +* :gh:issue:`397`: recent Mitogen master versions could fail to clean up temporary directories in a number of circumstances, and newer Ansibles moved to using :mod:`atexit` to effect temporary directory cleanup in some circumstances. -* `b9112a9c `_, - `2c287801 `_: OpenSSH 7.5 +* :gh:commit:`b9112a9c`, + :gh:commit:`2c287801`: OpenSSH 7.5 permission denied prompts are now recognized. Contributed by Alex Willmer. * A missing check caused an exception traceback to appear when using the @@ -1021,53 +996,53 @@ Core Library related function calls to a target context, cancelling the chain if an exception occurs. -* `#305 `_: fix a long-standing minor +* :gh:issue:`305`: fix a long-standing minor race relating to the logging framework, where *no route for Message..* would frequently appear during startup. -* `#313 `_: +* :gh:issue:`313`: :meth:`mitogen.parent.Context.call` was documented as capable of accepting static methods. While possible on Python 2.x the result is ugly, and in every case it should be trivial to replace with a classmethod. The documentation was fixed. -* `#337 `_: to avoid a scaling +* :gh:issue:`337`: to avoid a scaling limitation, a PTY is no longer allocated for each OpenSSH client if it can be avoided. PTYs are only allocated if a password is supplied, or when `host_key_checking=accept`. This is since Linux has a default of 4096 PTYs (``kernel.pty.max``), while OS X has a default of 127 and an absolute maximum of 999 (``kern.tty.ptmx_max``). -* `#339 `_: the LXD connection method +* :gh:issue:`339`: the LXD connection method was erroneously executing LXC Classic commands. -* `#345 `_: the SSH connection method +* :gh:issue:`345`: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. -* `#356 `_: if the master Python +* :gh:issue:`356`: if the master Python process does not have :data:`sys.executable` set, the default Python interpreter used for new children on the local machine defaults to ``"/usr/bin/python"``. -* `#366 `_, - `#380 `_: attempts by children to +* :gh:issue:`366`, + :gh:issue:`380`: attempts by children to import :mod:`__main__` where the main program module lacks an execution guard are refused, and an error is logged. This prevents a common and highly confusing error when prototyping new scripts. -* `#371 `_: the LXC connection method +* :gh:pull:`371`: the LXC connection method uses a more compatible method to establish an non-interactive session. Contributed by Brian Candler. -* `af2ded66 `_: add +* :gh:commit:`af2ded66`: add :func:`mitogen.fork.on_fork` to allow non-Mitogen managed process forks to clean up Mitogen resources in the child. -* `d6784242 `_: the setns method +* :gh:commit:`d6784242`: the setns method always resets ``HOME``, ``SHELL``, ``LOGNAME`` and ``USER`` environment variables to an account in the target container, defaulting to ``root``. -* `830966bf `_: the UNIX +* :gh:commit:`830966bf`: the UNIX listener no longer crashes if the peer process disappears in the middle of connection setup. @@ -1108,28 +1083,28 @@ v0.2.2 (2018-07-26) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#291 `_: ``ansible_*_interpreter`` +* :gh:issue:`291`: ``ansible_*_interpreter`` variables are parsed using a restrictive shell-like syntax, supporting a common idiom where ``ansible_python_interpreter`` is set to ``/usr/bin/env python``. -* `#299 `_: fix the ``network_cli`` +* :gh:issue:`299`: fix the ``network_cli`` connection type when the Mitogen strategy is active. Mitogen cannot help network device connections, however it should still be possible to use device connections while Mitogen is active. -* `#301 `_: variables like ``$HOME`` in +* :gh:pull:`301`: variables like ``$HOME`` in the ``remote_tmp`` setting are evaluated correctly. -* `#303 `_: the :ref:`doas` become method +* :gh:pull:`303`: the :ref:`doas` become method is supported. Contributed by `Mike Walker `_. -* `#309 `_: fix a regression to +* :gh:issue:`309`: fix a regression to process environment cleanup, caused by the change in v0.2.1 to run local tasks with the correct environment. -* `#317 `_: respect the verbosity +* :gh:issue:`317`: respect the verbosity setting when writing to Ansible's ``log_path``, if it is enabled. Child log filtering was also incorrect, causing the master to needlessly wake many times. This nets a 3.5% runtime improvement running against the local @@ -1142,32 +1117,32 @@ Mitogen for Ansible Core Library ~~~~~~~~~~~~ -* `#291 `_: the ``python_path`` +* :gh:issue:`291`: the ``python_path`` parameter may specify an argument vector prefix rather than a string program path. -* `#300 `_: the broker could crash on +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to scheduled `kqueue `_ filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. -* `#303 `_: the :ref:`doas` become method +* :gh:pull:`303`: the :ref:`doas` become method is now supported. Contributed by `Mike Walker `_. -* `#307 `_: SSH login banner output +* :gh:issue:`307`: SSH login banner output containing the word 'password' is no longer confused for a password prompt. -* `#319 `_: SSH connections would +* :gh:issue:`319`: SSH connections would fail immediately on Windows Subsystem for Linux, due to use of `TCSAFLUSH` with :func:`termios.tcsetattr`. The flag is omitted if WSL is detected. -* `#320 `_: The OS X poller +* :gh:issue:`320`: The OS X poller could spuriously wake up due to ignoring an error bit set on events returned by the kernel, manifesting as a failure to read from an unrelated descriptor. -* `#342 `_: The ``network_cli`` +* :gh:issue:`342`: The ``network_cli`` connection type would fail due to a missing internal SSH plugin method. * Standard IO forwarding accidentally configured the replacement ``stdout`` and @@ -1213,7 +1188,7 @@ v0.2.1 (2018-07-10) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#297 `_: compatibility: local +* :gh:issue:`297`: compatibility: local actions set their working directory to that of their defining playbook, and inherit a process environment as if they were executed as a subprocess of the forked task worker. diff --git a/docs/conf.py b/docs/conf.py index aa91c8b8..86332cd2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,13 +2,14 @@ import os import sys sys.path.append('..') +sys.path.append('.') import mitogen VERSION = '%s.%s.%s' % mitogen.__version__ author = u'Network Genomics' copyright = u'2019, Network Genomics' exclude_patterns = ['_build', '.venv'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput', 'domainrefs'] html_show_copyright = False html_show_sourcelink = False html_show_sphinx = False @@ -36,6 +37,49 @@ templates_path = ['_templates'] todo_include_todos = False version = VERSION +domainrefs = { + 'gh:commit': { + 'text': '%s', + 'url': 'https://github.com/dw/mitogen/commit/%s', + }, + 'gh:issue': { + 'text': '#%s', + 'url': 'https://github.com/dw/mitogen/issues/%s', + }, + 'gh:pull': { + 'text': '#%s', + 'url': 'https://github.com/dw/mitogen/pull/%s', + }, + 'ans:mod': { + 'text': '%s Module', + 'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html', + }, + 'ans:conn': { + 'text': '%s Connection Plug-in', + 'url': 'https://docs.ansible.com/ansible/latest/plugins/connection/%s.html', + }, + 'freebsd:man2': { + 'text': '%s(2)', + 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + }, + 'linux:man1': { + 'text': '%s(1)', + 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + }, + 'linux:man2': { + 'text': '%s(2)', + 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + }, + 'linux:man3': { + 'text': '%s(3)', + 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + }, + 'linux:man7': { + 'text': '%s(7)', + 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + }, +} + rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s diff --git a/docs/domainrefs.py b/docs/domainrefs.py new file mode 100644 index 00000000..4ff29dc5 --- /dev/null +++ b/docs/domainrefs.py @@ -0,0 +1,41 @@ + +import functools +import re + +import docutils.nodes +import docutils.utils + + +CUSTOM_RE = re.compile('(.*) <(.*)>') + + +def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]): + template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html' + + match = CUSTOM_RE.match(text) + if match: # "custom text " + title = match.group(1) + text = match.group(2) + elif text.startswith('~'): # brief + text = text[1:] + title = config.get('brief', '%s') % ( + docutils.utils.unescape(text), + ) + else: + title = config.get('text', '%s') % ( + docutils.utils.unescape(text), + ) + + node = docutils.nodes.reference( + rawsource=rawtext, + text=title, + refuri=config['url'] % (text,), + **options + ) + + return [node], [] + + +def setup(app): + for name, info in app.config._raw_config['domainrefs'].items(): + app.add_role(name, functools.partial(role, info)) diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 20c4f948..05c097e5 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -434,8 +434,9 @@ also listen on the following handles: Receives `target_id` integer from downstream, verifies a route exists to `target_id` via the stream on which the message was received, removes that - route from its local table, then propagates the message upward towards its - own parent. + route from its local table, triggers the ``disconnect`` signal on any + :class:`mitogen.core.Context` instance in the local process, then + propagates the message upward towards its own parent. .. currentmodule:: mitogen.core .. data:: DETACHING @@ -629,7 +630,8 @@ The `auth_id` field is separate from `src_id` in order to support granting privilege to contexts that do not follow the tree's natural trust chain. This supports cases where siblings are permitted to execute code on one another, or where isolated processes can connect to a listener and communicate with an -already established established tree. +already established established tree, such as where a :mod:`mitogen.unix` +client receives the same privilege as the process it connects to. Differences Between Master And Child Brokers diff --git a/docs/index.rst b/docs/index.rst index 17d183aa..3cd53d32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,8 +27,8 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_, ultimately it is not intended for direct use by consumer software. .. _Salt: https://docs.saltstack.com/en/latest/ -.. _Ansible: http://docs.ansible.com/ -.. _Fabric: http://www.fabfile.org/ +.. _Ansible: https://docs.ansible.com/ +.. _Fabric: https://www.fabfile.org/ The focus is to centralize and perfect the intricate dance required to run Python code safely and efficiently on a remote machine, while **avoiding @@ -132,7 +132,7 @@ any tool such as `py2exe`_ that correctly implement the protocols in PEP-302, allowing truly single file applications to run across multiple machines without further effort. -.. _py2exe: http://www.py2exe.org/ +.. _py2exe: https://www.py2exe.org/ Common sources of import latency and bandwidth consumption are mitigated: diff --git a/mitogen/core.py b/mitogen/core.py index 30c0e948..87388620 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3600,6 +3600,8 @@ class Dispatcher(object): mode, any exception that occurs is recorded, and causes all subsequent calls with the same `chain_id` to fail with the same exception. """ + _service_recv = None + def __repr__(self): return 'Dispatcher' diff --git a/mitogen/fork.py b/mitogen/fork.py index 25d1b5a6..4172e96f 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -73,8 +73,8 @@ def reset_logging_framework(): threads in the parent may have been using the logging package at the moment of fork. - It is not possible to solve this problem in general; see - https://github.com/dw/mitogen/issues/150 for a full discussion. + It is not possible to solve this problem in general; see :gh:issue:`150` + for a full discussion. """ logging._lock = threading.RLock() diff --git a/mitogen/master.py b/mitogen/master.py index 11ef2b00..f9ddf3dd 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -535,18 +535,20 @@ class SysModulesMethod(FinderMethod): Find `fullname` using its :data:`__file__` attribute. """ module = sys.modules.get(fullname) - LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) - if getattr(module, '__name__', None) != fullname: - LOG.debug('sys.modules[%r].__name__ does not match %r, assuming ' - 'this is a hacky module alias and ignoring it', - fullname, fullname) - return - if not isinstance(module, types.ModuleType): LOG.debug('%r: sys.modules[%r] absent or not a regular module', self, fullname) return + LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) + alleged_name = getattr(module, '__name__', None) + if alleged_name != fullname: + LOG.debug('sys.modules[%r].__name__ is incorrect, assuming ' + 'this is a hacky module alias and ignoring it. ' + 'Got %r, module object: %r', + fullname, alleged_name, module) + return + path = _py_filename(getattr(module, '__file__', '')) if not path: return @@ -573,43 +575,57 @@ class SysModulesMethod(FinderMethod): class ParentEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less - insane) parent package. Required for older versions of - :mod:`ansible.compat.six`, :mod:`plumbum.colors`, and Ansible 2.8 - :mod:`ansible.module_utils.distro`. + insane) parent package, and if no insane parents exist, simply use + :mod:`sys.path` to search for it from scratch on the filesystem using the + normal Python lookup mechanism. + + This is required for older versions of :mod:`ansible.compat.six`, + :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and + its submodule :mod:`ansible.module_utils.distro._distro`. + + When some package dynamically replaces itself in :data:`sys.modules`, but + only conditionally according to some program logic, it is possible that + children may attempt to load modules and subpackages from it that can no + longer be resolved by examining a (corrupted) parent. For cases like :mod:`ansible.module_utils.distro`, this must handle cases where a package transmuted itself into a totally unrelated module during - import and vice versa. + import and vice versa, where :data:`sys.modules` is replaced with junk that + makes it impossible to discover the loaded module using the in-memory + module object or any parent package's :data:`__path__`, since they have all + been overwritten. Some men just want to watch the world burn. """ - def find(self, fullname): + def _find_sane_parent(self, fullname): """ - See implementation for a description of how this works. + Iteratively search :data:`sys.modules` for the least indirect parent of + `fullname` that is loaded and contains a :data:`__path__` attribute. + + :return: + `(parent_name, path, modpath)` tuple, where: + + * `modname`: canonical name of the found package, or the empty + string if none is found. + * `search_path`: :data:`__path__` attribute of the least + indirect parent found, or :data:`None` if no indirect parent + was found. + * `modpath`: list of module name components leading from `path` + to the target module. """ - if fullname not in sys.modules: - # Don't attempt this unless a module really exists in sys.modules, - # else we could return junk. - return - - pkgname, _, modname = str_rpartition(to_text(fullname), u'.') - pkg = sys.modules.get(pkgname) - if pkg is None or not hasattr(pkg, '__file__'): - LOG.debug('%r: %r is not a package or lacks __file__ attribute', - self, pkgname) - return - - pkg_path = [os.path.dirname(pkg.__file__)] - try: - fp, path, (suffix, _, kind) = imp.find_module(modname, pkg_path) - except ImportError: - e = sys.exc_info()[1] - LOG.debug('%r: imp.find_module(%r, %r) -> %s', - self, modname, [pkg_path], e) - return None - - if kind == imp.PKG_DIRECTORY: - return self._found_package(fullname, path) - else: - return self._found_module(fullname, path, fp) + path = None + modpath = [] + while True: + pkgname, _, modname = str_rpartition(to_text(fullname), u'.') + modpath.insert(0, modname) + if not pkgname: + return [], None, modpath + + pkg = sys.modules.get(pkgname) + path = getattr(pkg, '__path__', None) + if pkg and path: + return pkgname.split('.'), path, modpath + + LOG.debug('%r: %r lacks __path__ attribute', self, pkgname) + fullname = pkgname def _found_package(self, fullname, path): path = os.path.join(path, '__init__.py') @@ -638,6 +654,47 @@ class ParentEnumerationMethod(FinderMethod): source = source.encode('utf-8') return path, source, is_pkg + def _find_one_component(self, modname, search_path): + try: + #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + return imp.find_module(modname, search_path) + except ImportError: + e = sys.exc_info()[1] + LOG.debug('%r: imp.find_module(%r, %r) -> %s', + self, modname, [search_path], e) + return None + + def find(self, fullname): + """ + See implementation for a description of how this works. + """ + #if fullname not in sys.modules: + # Don't attempt this unless a module really exists in sys.modules, + # else we could return junk. + #return + + fullname = to_text(fullname) + modname, search_path, modpath = self._find_sane_parent(fullname) + while True: + tup = self._find_one_component(modpath.pop(0), search_path) + if tup is None: + return None + + fp, path, (suffix, _, kind) = tup + if modpath: + # Still more components to descent. Result must be a package + if fp: + fp.close() + if kind != imp.PKG_DIRECTORY: + LOG.debug('%r: %r appears to be child of non-package %r', + self, fullname, path) + return None + search_path = [path] + elif kind == imp.PKG_DIRECTORY: + return self._found_package(fullname, path) + else: + return self._found_module(fullname, path, fp) + class ModuleFinder(object): """ diff --git a/mitogen/parent.py b/mitogen/parent.py index 82b4a7d1..9f93571b 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -901,9 +901,9 @@ class CallSpec(object): class PollPoller(mitogen.core.Poller): """ - Poller based on the POSIX poll(2) interface. Not available on some versions - of OS X, otherwise it is the preferred poller for small FD counts, as there - is no setup/teardown/configuration system call overhead. + Poller based on the POSIX :linux:man2:`poll` interface. Not available on + some versions of OS X, otherwise it is the preferred poller for small FD + counts, as there is no setup/teardown/configuration system call overhead. """ SUPPORTED = hasattr(select, 'poll') _repr = 'PollPoller()' @@ -949,7 +949,7 @@ class PollPoller(mitogen.core.Poller): class KqueuePoller(mitogen.core.Poller): """ - Poller based on the FreeBSD/Darwin kqueue(2) interface. + Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface. """ SUPPORTED = hasattr(select, 'kqueue') _repr = 'KqueuePoller()' @@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux epoll(2) interface. + Poller based on the Linux :linux:man2:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' diff --git a/mitogen/service.py b/mitogen/service.py index 168f0140..6bd64eb0 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -55,7 +55,6 @@ except NameError: LOG = logging.getLogger(__name__) -DEFAULT_POOL_SIZE = 16 _pool = None _pool_pid = None #: Serialize pool construction. @@ -80,7 +79,7 @@ def get_or_create_pool(size=None, router=None): global _pool_pid my_pid = os.getpid() - if _pool is None or my_pid != _pool_pid: + if _pool is None or _pool.closed or my_pid != _pool_pid: # Avoid acquiring heavily contended lock if possible. _pool_lock.acquire() try: @@ -88,7 +87,7 @@ def get_or_create_pool(size=None, router=None): _pool = Pool( router, services=[], - size=size or DEFAULT_POOL_SIZE, + size=size or 2, overwrite=True, recv=mitogen.core.Dispatcher._service_recv, ) diff --git a/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py new file mode 100644 index 00000000..f757e54c --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py @@ -0,0 +1,5 @@ +# #590: a subpackage that turns itself into a module from elsewhere on sys.path. +I_AM = "the subpackage that was replaced with a system module" +import sys +import system_distro +sys.modules[__name__] = system_distro diff --git a/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py new file mode 100644 index 00000000..16c32b2a --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py @@ -0,0 +1 @@ +I_AM = "the module inside the replaced subpackage" diff --git a/tests/data/importer/system_distro.py b/tests/data/importer/system_distro.py new file mode 100644 index 00000000..78fb1601 --- /dev/null +++ b/tests/data/importer/system_distro.py @@ -0,0 +1,2 @@ +# #590: a system module that replaces some subpackage +I_AM = "the system module that replaced the subpackage" diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index e61c768f..fc3a17de 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -180,6 +180,7 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): 'pkg_like_ansible.module_utils.distro._distro' ) + # ensure we can resolve the subpackage. path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro') modpath = os.path.join(MODS_DIR, 'pkg_like_ansible/module_utils/distro/__init__.py') @@ -187,6 +188,44 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): self.assertEquals(src, open(modpath, 'rb').read()) self.assertEquals(is_pkg, True) + # ensure we can resolve a child of the subpackage. + path, src, is_pkg = self.call( + 'pkg_like_ansible.module_utils.distro._distro' + ) + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/distro/_distro.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, False) + + def test_ansible_module_utils_system_distro_succeeds(self): + # #590: a package that turns itself into a module. + # #590: a package that turns itself into a module. + import pkg_like_ansible.module_utils.sys_distro as d + self.assertEquals(d.I_AM, "the system module that replaced the subpackage") + self.assertEquals( + sys.modules['pkg_like_ansible.module_utils.sys_distro'].__name__, + 'system_distro' + ) + + # ensure we can resolve the subpackage. + path, src, is_pkg = self.call('pkg_like_ansible.module_utils.sys_distro') + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/sys_distro/__init__.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, True) + + # ensure we can resolve a child of the subpackage. + path, src, is_pkg = self.call( + 'pkg_like_ansible.module_utils.sys_distro._distro' + ) + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/sys_distro/_distro.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, False) + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder diff --git a/tests/responder_test.py b/tests/responder_test.py index 285acd6f..2653589c 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -158,14 +158,15 @@ class BrokenModulesTest(testlib.TestCase): self.assertEquals(1, len(router._async_route.mock_calls)) self.assertEquals(1, responder.get_module_count) - self.assertEquals(0, responder.good_load_module_count) - self.assertEquals(0, responder.good_load_module_size) - self.assertEquals(1, responder.bad_load_module_count) + self.assertEquals(1, responder.good_load_module_count) + self.assertEquals(0, responder.bad_load_module_count) call = router._async_route.mock_calls[0] msg, = call[1] self.assertEquals(mitogen.core.LOAD_MODULE, msg.handle) - self.assertIsInstance(msg.unpickle(), tuple) + + tup = msg.unpickle() + self.assertIsInstance(tup, tuple) class ForwardTest(testlib.RouterMixin, testlib.TestCase): diff --git a/tests/service_test.py b/tests/service_test.py index 438766f7..a3e75e14 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -15,6 +15,13 @@ class MyService(mitogen.service.Service): self._counter += 1 return self._counter, id(self) + @mitogen.service.expose(policy=mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'foo': int + }) + def test_arg_spec(self, foo): + return foo + @mitogen.service.expose(policy=mitogen.service.AllowParents()) def privileged_op(self): return 'privileged!' @@ -24,7 +31,6 @@ class MyService(mitogen.service.Service): return 'unprivileged!' - class MyService2(MyService): """ A uniquely named service that lets us test framework activation and class @@ -36,6 +42,44 @@ def call_service_in(context, service_name, method_name): return context.call_service(service_name, method_name) +class CallTest(testlib.RouterMixin, testlib.TestCase): + def test_local(self): + pool = mitogen.service.get_or_create_pool(router=self.router) + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService, 'privileged_op') + ) + pool.stop() + + def test_remote_bad_arg(self): + c1 = self.router.local() + self.assertRaises( + mitogen.core.CallError, + lambda: mitogen.service.call( + MyService.name(), + 'test_arg_spec', + foo='x', + call_context=c1 + ) + ) + + def test_local_unicode(self): + pool = mitogen.service.get_or_create_pool(router=self.router) + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService.name(), 'privileged_op') + ) + pool.stop() + + def test_remote(self): + c1 = self.router.local() + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService, 'privileged_op', + call_context=c1) + ) + + class ActivationTest(testlib.RouterMixin, testlib.TestCase): def test_parent_can_activate(self): l1 = self.router.local()