diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000..1b9f9dfa --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,44 @@ + +# `.ci` + +This directory contains scripts for Travis CI and (more or less) Azure +Pipelines, but they will also happily run on any Debian-like machine. + +The scripts are usually split into `_install` and `_test` steps. The `_install` +step will damage your machine, the `_test` step will just run the tests the way +CI runs them. + +There is a common library, `ci_lib.py`, which just centralized a bunch of +random macros and also environment parsing. + +Some of the scripts allow you to pass extra flags through to the component +under test, e.g. `../../.ci/ansible_tests.py -vvv` will run with verbose. + +Hack these scripts until your heart is content. There is no pride to be found +here, just necessity. + + +### `ci_lib.run_batches()` + +There are some weird looking functions to extract more paralellism from the +build. The above function takes lists of strings, arranging for the strings in +each list to run in order, but for the lists to run in parallel. That's great +for doing `setup.py install` while pulling a Docker container, for example. + + +### Environment Variables + +* `VER`: Ansible version the `_install` script should install. Default changes + over time. +* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. +* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This + name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` +* `DISTROS`: the `ansible_` tests can run against multiple targets + simultaneously, which speeds things up. This is a space-separated list of + DISTRO names, but additionally, supports: + * `debian-py3`: when generating Ansible inventory file, set + `ansible_python_interpreter` to `python3`, i.e. run a test where the + target interpreter is Python 3. + * `debian*16`: generate 16 Docker containers running Debian. Also works + with -py3. + diff --git a/docs/ansible.rst b/docs/ansible.rst index e45a9f7a..17354755 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -10,9 +10,8 @@ Mitogen, replacing embedded shell invocations with pure-Python equivalents invoked via highly efficient remote procedure calls to persistent interpreters tunnelled over SSH. No changes are required to target hosts. -The extension is approaching stability and real-world usage is encouraged. `Bug -reports`_ are welcome: Ansible is huge, and only wide testing will ensure -soundness. +The extension is stable and real-world use is encouraged. `Bug reports`_ are +welcome: Ansible is huge, and only wide testing will ensure soundness. .. _Ansible: https://www.ansible.com/ diff --git a/docs/changelog.rst b/docs/changelog.rst index 15ff67d7..bcca6088 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -455,6 +455,21 @@ Core Library import :mod:`__main__` on Python 3.4 and newer due to a breaking change in the :mod:`pkgutil` API. The program's main script is now handled specially. +* `#481 `_: the version of `sudo` + that shipped with CentOS 5 replaced itself with the program to be executed, + and therefore did not hold any child PTY open on our behalf. The child + context is updated to preserve any PTY FD in order to avoid the kernel + sending `SIGHUP` early during startup. + +* `#523 `_: the test suite didn't + generate a code coverage report if any test failed. + +* `#524 `_: Python 3.6+ emitted a + :class:`DeprecationWarning` for :func:`mitogen.utils.run_with_router`. + +* `#529 `_: Code coverage of the + test suite was not measured across all Python versions. + * `16ca111e `_: handle OpenSSH 7.5 permission denied prompts when ``~/.ssh/config`` rewrites are present. @@ -475,18 +490,9 @@ Core Library a failure. * `57b652ed `_: a stray import - meant an extra roundtrip and ~20KiB of data was wasted for any context that + meant an extra roundtrip and ~4KiB of data was wasted for any context that imported :mod:`mitogen.parent`. -* `#523 ` : the test suite didn't - generate a code coverage report if any test failed. - -* `#524 ` : Python 3.6+ emitted a - :class:`DeprecationWarning` for :func:`mitogen.utils.run_with_router`. - -* `#529 ` : Code coverage of the - test suite was not measured across all Python versions. - Thanks! ~~~~~~~ @@ -508,6 +514,7 @@ bug reports, testing, features and fixes in this release contributed by `Johan Beisser `_, `Jonathan Rosser `_, `Josh Smift `_, +`Kevin Carter `_, `Mehdi `_, `Michael DeHaan `_, `Michal Medvecky `_, diff --git a/mitogen/core.py b/mitogen/core.py index 3694342c..a48e13ed 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3295,6 +3295,18 @@ class ExternalContext(object): os.close(fd) def _setup_stdio(self): + # #481: when stderr is a TTY due to being started via + # tty_create_child()/hybrid_tty_create_child(), and some privilege + # escalation tool like prehistoric versions of sudo exec this process + # over the top of itself, there is nothing left to keep the slave PTY + # open after we replace our stdio. Therefore if stderr is a TTY, keep + # around a permanent dup() to avoid receiving SIGHUP. + try: + if os.isatty(2): + self.reserve_tty_fd = os.dup(2) + set_cloexec(self.reserve_tty_fd) + except OSError: + pass # When sys.stdout was opened by the runtime, overwriting it will not # close FD 1. However when forking from a child that previously used # fdopen(), overwriting it /will/ close FD 1. So we must swallow the diff --git a/mitogen/master.py b/mitogen/master.py index 427bf731..257fb81b 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -922,6 +922,11 @@ class ModuleResponder(object): fullname, _, _ = str_rpartition(fullname, u'.') stream = self._router.stream_by_id(context.context_id) + if stream is None: + LOG.debug('%r: dropping forward of %s to no longer existent ' + '%r', self, path[0], context) + return + for fullname in reversed(path): self._send_module_and_related(stream, fullname) self._send_forward_module(stream, context, fullname) diff --git a/preamble_size.py b/preamble_size.py index bf3b5950..ead5af85 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -16,6 +16,9 @@ import mitogen.service import mitogen.ssh import mitogen.sudo +import ansible_mitogen.runner +import ansible_mitogen.target + router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) stream = mitogen.ssh.Stream(router, 0, max_message_size=0, hostname='foo') @@ -31,7 +34,7 @@ if '--dump' in sys.argv: print( - ' ' + ' ' ' ' ' Original ' ' ' @@ -42,6 +45,9 @@ print( for mod in ( mitogen.parent, + mitogen.fork, + ansible_mitogen.target, + ansible_mitogen.runner, mitogen.ssh, mitogen.sudo, mitogen.select, @@ -56,7 +62,7 @@ for mod in ( compressed = zlib.compress(minimized, 9) compressed_size = len(compressed) print( - '%-15s' + '%-25s' ' ' '%5i %4.1fKiB' ' ' diff --git a/tests/responder_test.py b/tests/responder_test.py index 15f21976..da4f22d7 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -167,6 +167,19 @@ class BrokenModulesTest(testlib.TestCase): class ForwardTest(testlib.RouterMixin, testlib.TestCase): + def test_forward_to_nonexistent_context(self): + nonexistent = mitogen.core.Context(self.router, 123) + capture = testlib.LogCapturer() + capture.start() + self.broker.defer_sync(lambda: + self.router.responder.forward_modules( + nonexistent, + ['mitogen.core'] + ) + ) + s = capture.stop() + self.assertTrue('dropping forward of' in s) + def test_stats(self): # Forwarding stats broken because forwarding is broken. See #469. c1 = self.router.local()