From b38318dfecef289ed9e38e4e7f996d2f74b7a022 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 23 Jun 2018 23:21:15 +0000 Subject: [PATCH 01/24] issue #275: build for centos 6 too (python2.6) --- tests/build_docker_images.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/build_docker_images.py b/tests/build_docker_images.py index 60ceac10..32e3384b 100755 --- a/tests/build_docker_images.py +++ b/tests/build_docker_images.py @@ -12,7 +12,7 @@ import tempfile DEBIAN_DOCKERFILE = r""" -FROM debian:stable +FROM debian:stretch RUN apt-get update RUN \ apt-get install -y python2.7 openssh-server sudo rsync git strace \ @@ -21,7 +21,18 @@ RUN \ rm -rf /var/cache/apt """ -CENTOS_DOCKERFILE = r""" +CENTOS6_DOCKERFILE = r""" +FROM centos:6 +RUN yum clean all && \ + yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \ + perl-JSON python-virtualenv && \ + yum clean all && \ + groupadd sudo && \ + ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key + +""" + +CENTOS7_DOCKERFILE = r""" FROM centos:7 RUN yum clean all && \ yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \ @@ -49,8 +60,8 @@ RUN \ useradd -s /bin/bash -m mitogen__readonly_homedir && \ useradd -s /bin/bash -m mitogen__slow_user && \ chown -R root: ~mitogen__readonly_homedir && \ - { for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user$i; done; } && \ - { for i in `seq 1 21`; do echo mitogen__user$i:user$i_password | chpasswd; done; } && \ + ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \ + ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \ ( echo 'root:rootpassword' | chpasswd; ) && \ ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \ ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \ @@ -62,7 +73,7 @@ RUN \ ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \ ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \ mkdir ~mitogen__has_sudo_pubkey/.ssh && \ - { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; } + ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; ) COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile @@ -89,8 +100,11 @@ def sh(s, *args): return shlex.split(s) -for (distro, wheel, prefix) in (('debian', 'sudo', DEBIAN_DOCKERFILE), - ('centos', 'wheel', CENTOS_DOCKERFILE)): +for (distro, wheel, prefix) in ( + ('debian', 'sudo', DEBIAN_DOCKERFILE), + ('centos6', 'wheel', CENTOS6_DOCKERFILE), + ('centos7', 'wheel', CENTOS7_DOCKERFILE), + ): mydir = os.path.abspath(os.path.dirname(__file__)) with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp: dockerfile_fp.write(prefix) From 38d69a6ecdb38f1747313e3a25c769fdc4943d1a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 01:55:35 +0100 Subject: [PATCH 02/24] issue #275: tests: drop docker client dep, doesn't run on 2.6. --- dev_requirements.txt | 2 - tests/show_docker_hostname.py | 5 +-- tests/testlib.py | 74 ++++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index c8b6af13..a55be0de 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,8 +4,6 @@ coverage==4.5.1 Django==1.6.11; python_version < '2.7' Django==1.11.5; python_version >= '2.7' # for module_finder_test debops==0.7.2 -https://github.com/docker/docker-py/archive/1.10.6.tar.gz; python_version < '2.7' -docker[tls]==2.5.1; python_version >= '2.7' mock==2.0.0 pytest-catchlog==1.2.2 pytest==3.1.2 diff --git a/tests/show_docker_hostname.py b/tests/show_docker_hostname.py index 1dc1cb98..d4326f92 100644 --- a/tests/show_docker_hostname.py +++ b/tests/show_docker_hostname.py @@ -5,8 +5,5 @@ For use by the Travis scripts, just print out the hostname of the Docker daemon from the environment. """ -import docker import testlib - -docker = docker.from_env(version='auto') -print testlib.get_docker_host(docker) +print testlib.get_docker_host() diff --git a/tests/testlib.py b/tests/testlib.py index 28316ba9..56f2e232 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -5,6 +5,7 @@ import os import random import re import socket +import subprocess import sys import time import urlparse @@ -15,9 +16,6 @@ import mitogen.core import mitogen.master import mitogen.utils -if mitogen.is_master: # TODO: shouldn't be necessary. - import docker - DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') sys.path.append(DATA_DIR) @@ -34,6 +32,19 @@ def data_path(suffix): return path +def subprocess__check_output(*popenargs, **kwargs): + # Missing from 2.6. + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, _ = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd, output=output) + return output + + def wait_for_port( host, port, @@ -162,11 +173,12 @@ class TestCase(unittest2.TestCase): assert 0, '%r did not raise %r' % (func, exc) -def get_docker_host(docker): - if docker.api.base_url == 'http+docker://localunixsocket': +def get_docker_host(): + url = os.environ.get('DOCKER_HOST') + if url in (None, 'http+docker://localunixsocket'): return 'localhost' - parsed = urlparse.urlparse(docker.api.base_url) + parsed = urlparse.urlparse(url) return parsed.netloc.partition(':')[0] @@ -179,29 +191,47 @@ class DockerizedSshDaemon(object): self.image = 'mitogen/%s-test' % (distro,) return self.image - def __init__(self): - self.docker = docker.from_env(version='auto') - self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),) - self.container = self.docker.containers.run( - image=self.get_image(), - detach=True, - privileged=True, - publish_all_ports=True, - ) - self.container.reload() - self.port = (self.container.attrs['NetworkSettings']['Ports'] - ['22/tcp'][0]['HostPort']) + # 22/tcp -> 0.0.0.0:32771 + PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') + port = None + + def _get_container_port(self): + s = subprocess__check_output(['docker', 'port', self.container_name]) + for line in s.splitlines(): + dport, proto, baddr, bport = self.PORT_RE.match(line).groups() + if dport == '22' and proto == 'tcp': + self.port = int(bport) + self.host = self.get_host() + if self.port is None: + raise ValueError('could not find SSH port in: %r' % (s,)) + + def start_container(self): + self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),) + args = [ + 'docker', + 'run', + '--detach', + '--privileged', + '--publish-all', + '--name', self.container_name, + self.get_image() + ] + subprocess__check_output(args) + self._get_container_port() + + def __init__(self): + self.start_container() def get_host(self): - return get_docker_host(self.docker) + return get_docker_host() def wait_for_sshd(self): - wait_for_port(self.get_host(), int(self.port), pattern='OpenSSH') + wait_for_port(self.get_host(), self.port, pattern='OpenSSH') def close(self): - self.container.stop() - self.container.remove() + args = ['docker', 'rm', '-f', self.container_name] + subprocess__check_output(args) class BrokerMixin(object): From cfd28872929b5c7d548190ec870f80714374bc06 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 02:46:36 +0100 Subject: [PATCH 03/24] issue #275: default to 'python' for default remote interpreter. So we get 2.4/2.5/2.6/2.7/3.x. --- mitogen/parent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index aeebc43a..88a6ab46 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -648,7 +648,7 @@ class Stream(mitogen.core.Stream): Base for streams capable of starting new slaves. """ #: The path to the remote Python interpreter. - python_path = 'python2.7' + python_path = 'python' #: Maximum time to wait for a connection attempt. connect_timeout = 30.0 From 4649e11da369cd939f87baa43e9f7628b1838c2e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 00:33:46 +0100 Subject: [PATCH 04/24] issue #275: Travis build matrix from hell. dev_requirements.txt: - drop debops, it's not available for Python2.6 --- .travis.yml | 61 ++++++++++++++++++++++++++++++++++---------- dev_requirements.txt | 1 - 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80c88ee4..e879c748 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,20 +9,53 @@ notifications: language: python cache: pip -python: -- "2.7" - -env: -- MODE=mitogen DISTRO=debian -- MODE=mitogen DISTRO=centos -- MODE=debops_common VER=2.4.3.0 -- MODE=debops_common VER=2.5.1 -# Ansible tests. -- MODE=ansible VER=2.4.3.0 DISTRO=debian -- MODE=ansible VER=2.5.1 DISTRO=centos -- MODE=ansible VER=2.5.1 DISTRO=debian -# Sanity check our tests against vanilla Ansible, they should still pass. -- MODE=ansible VER=2.5.1 DISTRO=debian STRATEGY=linear +matrix: + include: + # Mitogen tests. + # 2.7 -> 2.7 + - python: "2.7" + env: MODE=mitogen DISTRO=debian + # 2.7 -> 2.6 + - python: "2.7" + env: MODE=mitogen DISTRO=centos6 + # 2.6 -> 2.7 + - python: "2.6" + env: MODE=mitogen DISTRO=centos7 + # 2.6 -> 2.6 + - python: "2.6" + env: MODE=mitogen DISTRO=centos6 + + # Debops tests. + # 2.4.3.0; 2.7 -> 2.7 + - python: "2.7" + env: MODE=debops_common VER=2.4.3.0 + # 2.5.5; 2.7 -> 2.7 + - python: "2.7" + env: MODE=debops_common VER=2.5.5 + + # ansible_mitogen tests. + # 2.4.3.0; Debian; 2.7 -> 2.7 + - python: "2.7" + env: MODE=ansible VER=2.4.3.0 DISTRO=debian + # 2.5.5; Debian; 2.7 -> 2.7 + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=debian + # 2.5.5; CentOS; 2.7 -> 2.7 + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=centos7 + # 2.5.5; CentOS; 2.7 -> 2.6 + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=centos6 + # 2.5.5; CentOS; 2.6 -> 2.7 + - python: "2.6" + env: MODE=ansible VER=2.5.5 DISTRO=centos7 + # 2.5.5; CentOS; 2.6 -> 2.6 + - python: "2.6" + env: MODE=ansible VER=2.5.5 DISTRO=centos6 + + # Sanity check our tests against vanilla Ansible, they should pass. + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear install: - pip install -r dev_requirements.txt diff --git a/dev_requirements.txt b/dev_requirements.txt index a55be0de..202f2b9f 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,7 +3,6 @@ ansible==2.5.2 coverage==4.5.1 Django==1.6.11; python_version < '2.7' Django==1.11.5; python_version >= '2.7' # for module_finder_test -debops==0.7.2 mock==2.0.0 pytest-catchlog==1.2.2 pytest==3.1.2 From 1d04a99adba7730e496d855964c2cfc33e929c46 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 04:34:59 +0100 Subject: [PATCH 05/24] issue #275: missing check_output() call --- tests/responder_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/responder_test.py b/tests/responder_test.py index 3f6f66a9..86e6dd7e 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -30,7 +30,7 @@ class GoodModulesTest(testlib.RouterMixin, unittest2.TestCase): # Ensure a program composed of a single script can be imported # successfully. args = [sys.executable, testlib.data_path('self_contained_program.py')] - output = subprocess.check_output(args) + output = testlib.subprocess__check_output(args) self.assertEquals(output, "['__main__', 50]\n") From e0c116a29fab83a8c8742057fe34d6f8ed8e549c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 04:47:18 +0100 Subject: [PATCH 06/24] issue #275: logging package uses classic classes in 2.6. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 9c9a14af..1a854d11 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -278,7 +278,7 @@ class PidfulStreamHandler(logging.StreamHandler): def emit(self, record): if self.open_pid != os.getpid(): self._reopen() - return super(PidfulStreamHandler, self).emit(record) + logging.StreamHandler.emit(self, record) def enable_debug_logging(): From 6d618593f301187c04ee0f93bcac30bcaba766f9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 16:27:34 +0100 Subject: [PATCH 07/24] issue #275: Python 2.6 reports linux as 'linux3'. --- tests/fork_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/fork_test.py b/tests/fork_test.py index 61c0e16d..47160855 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -19,6 +19,8 @@ PLATFORM_TO_PATH = { ('darwin', True): '/usr/lib/libssl.dylib', ('linux2', False): '/usr/lib/libssl.so', ('linux2', True): '/usr/lib/x86_64-linux-gnu/libssl.so', + ('linux3', False): '/usr/lib/libssl.so', + ('linux3', True): '/usr/lib/x86_64-linux-gnu/libssl.so', } c_ssl = ctypes.CDLL(PLATFORM_TO_PATH[sys.platform, IS_64BIT]) From 3b1cc3676cf68038abc60b5d3875c6d4bd2974db Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:01:15 +0100 Subject: [PATCH 08/24] issue #275: ssh_debug_level=3 for tests --- tests/testlib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testlib.py b/tests/testlib.py index 56f2e232..ae2d5504 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -274,6 +274,7 @@ class DockerMixin(RouterMixin): kwargs.setdefault('hostname', self.dockerized_ssh.host) kwargs.setdefault('port', self.dockerized_ssh.port) kwargs.setdefault('check_host_keys', 'ignore') + kwargs.setdefault('ssh_debug_level', '3') return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): From 60ad75f4362e6679beca586738015f7e8f8100eb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:11:52 +0100 Subject: [PATCH 09/24] issue #275: Tidier SSH debug logging. --- mitogen/ssh.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 5c0df8dc..9ba5a914 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -58,8 +58,10 @@ def _filter_debug(stream, it, buf): if not buf.startswith(DEBUG_PREFIXES): return buf while '\n' in buf: + if not buf.startswith(DEBUG_PREFIXES): + return buf line, _, buf = buf.partition('\n') - LOG.debug('%r: received %r', stream, line.rstrip()) + LOG.debug('%r: %s', stream, line.rstrip()) try: buf += next(it) except StopIteration: @@ -77,7 +79,8 @@ def filter_debug(stream, it): for chunk in it: chunk = _filter_debug(stream, it, chunk) if chunk: - yield chunk + for line in chunk.splitlines(): + yield line class PasswordError(mitogen.core.StreamError): @@ -219,7 +222,7 @@ class Stream(mitogen.parent.Stream): for buf in filter_debug(self, it): LOG.debug('%r: received %r', self, buf) - if buf.endswith('EC0\n'): + if buf.endswith('EC0'): self._router.broker.start_receive(self.tty_stream) self._ec0_received() return From e5d02b948b01f1a9b510a6f8cfeecc88ae2bb01b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:19:42 +0100 Subject: [PATCH 10/24] issue #275: travis: run_tests with -vvv --- .travis/mitogen_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/mitogen_tests.sh b/.travis/mitogen_tests.sh index 01e24963..db393d73 100755 --- a/.travis/mitogen_tests.sh +++ b/.travis/mitogen_tests.sh @@ -2,4 +2,4 @@ # Run the Mitogen tests. MITOGEN_TEST_DISTRO="${DISTRO:-debian}" -MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests +MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -vvv From 83617ad1928589ca1e1f596ca18f5755597db044 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:23:10 +0100 Subject: [PATCH 11/24] issue #275: tests: cache virtualenvs too --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e879c748..94a9aff5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,10 @@ notifications: email: false language: python -cache: pip +cache: +- pip +- directories: + - /home/travis/virtualenv matrix: include: From fbd5837cf2427469b554eeb1bdda15fd0829b5dd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:31:50 +0100 Subject: [PATCH 12/24] issue #275: parent: use TIOCSCTTY on Linux too. This appears to be harmless, except for Python 2.6 on Linux/Travis, where for some reason (some stdlib change?) simply opening the TTY is insufficient. --- mitogen/parent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 88a6ab46..4e2b1c60 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -189,8 +189,9 @@ def _acquire_controlling_tty(): # On Linux, the controlling tty becomes the first tty opened by a # process lacking any prior tty. os.close(os.open(os.ttyname(2), os.O_RDWR)) - if sys.platform.startswith('freebsd') or sys.platform == 'darwin': - # On BSD an explicit ioctl is required. + if hasattr(termios, 'TIOCSCTTY'): + # On BSD an explicit ioctl is required. For some inexplicable reason, + # Python 2.6 on Travis also requires it. fcntl.ioctl(2, termios.TIOCSCTTY) From 84fa3ff024df4ed93ff2e1e7d099b3427dee569a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 17:45:05 +0000 Subject: [PATCH 13/24] issue #275: ssh: state machine-ish filter_debug() --- mitogen/ssh.py | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 9ba5a914..c710f990 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -53,21 +53,6 @@ HOSTKEY_FAIL = 'host key verification failed.' DEBUG_PREFIXES = ('debug1:', 'debug2:', 'debug3:') -def _filter_debug(stream, it, buf): - while True: - if not buf.startswith(DEBUG_PREFIXES): - return buf - while '\n' in buf: - if not buf.startswith(DEBUG_PREFIXES): - return buf - line, _, buf = buf.partition('\n') - LOG.debug('%r: %s', stream, line.rstrip()) - try: - buf += next(it) - except StopIteration: - return buf - - def filter_debug(stream, it): """ Read line chunks from it, either yielding them directly, or building up and @@ -76,11 +61,33 @@ def filter_debug(stream, it): This contains the mess of dealing with both line-oriented input, and partial lines such as the password prompt. """ + state = 'start_of_line' + buf = '' for chunk in it: - chunk = _filter_debug(stream, it, chunk) - if chunk: - for line in chunk.splitlines(): - yield line + buf += chunk + while buf: + if state == 'start_of_line': + if len(buf) < 8: + # short read near the buffer limit, block waiting for at + # least 8 bytes so we can discern either a debug line, or + # the minimum desired interesting token from above + # ('password'). + break + elif buf.startswith(DEBUG_PREFIXES): + state = 'in_debug' + else: + state = 'in_plain' + elif state == 'in_debug': + if '\n' not in buf: + break + line, _, buf = buf.partition('\n') + LOG.debug('%r: %s', stream, line.rstrip()) + state = 'start_of_line' + elif state == 'in_plain': + line, nl, buf = buf.partition('\n') + yield line + nl + if nl: + state = 'start_of_line' class PasswordError(mitogen.core.StreamError): @@ -222,7 +229,7 @@ class Stream(mitogen.parent.Stream): for buf in filter_debug(self, it): LOG.debug('%r: received %r', self, buf) - if buf.endswith('EC0'): + if buf.endswith('EC0\n'): self._router.broker.start_receive(self.tty_stream) self._ec0_received() return From d6126a95169e2984208ee092daae93607c20fea9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 18:28:50 +0000 Subject: [PATCH 14/24] issue #275: parent/ssh: centralize EC0_MARKER and change it for ssh.py. Must maintain a minimum buffer length prior to deciding whether we have an interesting token, and 'EC0' is too short for that. --- docs/howitworks.rst | 19 ++++++++++++------- mitogen/parent.py | 12 ++++++++---- mitogen/ssh.py | 10 +++++----- mitogen/su.py | 2 +- mitogen/sudo.py | 2 +- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 0b0ae42d..b14ceab7 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -51,7 +51,7 @@ can be recovered by the bootstrapped process later. It then forks into a new process. After fork, the parent half overwrites its ``stdin`` with the read end of the -pipe, and the child half writes the string ``EC0\n``, then begins reading the +pipe, and the child half writes the string ``MITOGEN0\n``, then begins reading the :py:mod:`zlib`-compressed payload supplied on ``stdin`` by the master, and writing the decompressed result to the write-end of the UNIX pipe. @@ -112,12 +112,17 @@ fetched from the master a second time. Signalling Success ################## -Once the first stage has signalled ``EC0\n``, the master knows it is ready to -receive the compressed bootstrap. After decompressing and writing the bootstrap -source to its parent Python interpreter, the first stage writes the string -``EC1\n`` to ``stdout`` before exiting. The master process waits for this -string before considering bootstrap successful and the child's ``stdio`` ready -to receive messages. +Once the first stage has signalled ``MITO000\n``, the master knows it is ready +to receive the compressed bootstrap. After decompressing and writing the +bootstrap source to its parent Python interpreter, the first stage writes the +string ``MITO001\n`` to ``stdout`` before exiting. The master process waits for +this string before considering bootstrap successful and the child's ``stdio`` +ready to receive messages. + +The signal value is 8 bytes to match the minimum chunk size required to +disambiguate between lines containing an interesting token during SSH password +authentication, a debug message from the SSH client itself, or a message from +the first stage. ExternalContext.main() diff --git a/mitogen/parent.py b/mitogen/parent.py index 4e2b1c60..c999c89a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -790,11 +790,11 @@ class Stream(mitogen.core.Stream): sys.executable += sys.version[:3] os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') - os.write(1,'EC0\n') + os.write(1,'MITO000\n') C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') os.fdopen(W,'w',0).write(C) os.fdopen(w,'w',0).write('PREAMBLE_LEN\n'+C) - os.write(1,'EC1\n') + os.write(1,'MITO001\n') def get_boot_command(self): source = inspect.getsource(self._first_stage) @@ -870,13 +870,17 @@ class Stream(mitogen.core.Stream): self._reap_child() raise + #: For ssh.py, this must be at least max(len('password'), len('debug1:')) + EC0_MARKER = 'MITO000\n' + EC1_MARKER = 'MITO001\n' + def _ec0_received(self): LOG.debug('%r._ec0_received()', self) write_all(self.transmit_side.fd, self.get_preamble()) - discard_until(self.receive_side.fd, 'EC1\n', self.connect_deadline) + discard_until(self.receive_side.fd, 'MITO001\n', self.connect_deadline) def _connect_bootstrap(self, extra_fd): - discard_until(self.receive_side.fd, 'EC0\n', self.connect_deadline) + discard_until(self.receive_side.fd, 'MITO000\n', self.connect_deadline) self._ec0_received() diff --git a/mitogen/ssh.py b/mitogen/ssh.py index c710f990..161b5323 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -68,10 +68,10 @@ def filter_debug(stream, it): while buf: if state == 'start_of_line': if len(buf) < 8: - # short read near the buffer limit, block waiting for at - # least 8 bytes so we can discern either a debug line, or - # the minimum desired interesting token from above - # ('password'). + # short read near buffer limit, block awaiting at least 8 + # bytes so we can discern a debug line, or the minimum + # interesting token from above or the bootstrap + # ('password', 'MITO000\n'). break elif buf.startswith(DEBUG_PREFIXES): state = 'in_debug' @@ -229,7 +229,7 @@ class Stream(mitogen.parent.Stream): for buf in filter_debug(self, it): LOG.debug('%r: received %r', self, buf) - if buf.endswith('EC0\n'): + if buf.endswith(self.EC0_MARKER): self._router.broker.start_receive(self.tty_stream) self._ec0_received() return diff --git a/mitogen/su.py b/mitogen/su.py index 2cc3406b..ce81eed5 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -97,7 +97,7 @@ class Stream(mitogen.parent.Stream): for buf in it: LOG.debug('%r: received %r', self, buf) - if buf.endswith('EC0\n'): + if buf.endswith(self.EC0_MARKER): self._ec0_received() return if any(s in buf.lower() for s in self.incorrect_prompts): diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 5d2911fc..32287f9f 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -168,7 +168,7 @@ class Stream(mitogen.parent.Stream): for buf in it: LOG.debug('%r: received %r', self, buf) - if buf.endswith('EC0\n'): + if buf.endswith(self.EC0_MARKER): self._ec0_received() return elif PASSWORD_PROMPT in buf.lower(): From 6e0883f3692392123680274f69d27b202939a9ad Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 18:30:16 +0000 Subject: [PATCH 15/24] issue #275: tests: fix bug in 2.6 compat check_output(), ignore it for >2.6. --- tests/testlib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/testlib.py b/tests/testlib.py index ae2d5504..46868f6d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -41,9 +41,12 @@ def subprocess__check_output(*popenargs, **kwargs): cmd = kwargs.get("args") if cmd is None: cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd, output=output) + raise subprocess.CalledProcessError(retcode, cmd) return output +if hasattr(subprocess, 'check_output'): + subprocess__check_output = subprocess.check_output + def wait_for_port( host, From 7b84a2c2e4c6c3cd8962131fc80cbcca1e544b96 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 18:47:29 +0000 Subject: [PATCH 16/24] issue #275: tests: use same EC0_MARKER as parent.py --- tests/first_stage_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index d868f8b5..1698a435 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -35,7 +35,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): ) stdout, stderr = proc.communicate() self.assertEquals(0, proc.returncode) - self.assertEquals("EC0\n", stdout) + self.assertEquals(mitogen.parent.Stream.EC0_MARKER, stdout) self.assertIn("Error -5 while decompressing data: incomplete or truncated stream", stderr) From 4be8afa3d3cb31a934211c822e5a972477c2b29b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 20:10:32 +0100 Subject: [PATCH 17/24] issue #275: tests: fix test_simple for 2.6. --- tests/module_finder_test.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 1c77bdee..c2e64cfb 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,4 +1,5 @@ import inspect +import sys import unittest2 @@ -190,19 +191,24 @@ class FindRelatedTest(testlib.TestCase): def call(self, fullname): return self.klass().find_related(fullname) + SIMPLE_EXPECT = set([ + 'mitogen', + 'mitogen.compat', + 'mitogen.compat.collections', + 'mitogen.compat.functools', + 'mitogen.core', + 'mitogen.master', + 'mitogen.minify', + 'mitogen.parent', + ]) + + if sys.version_info < (2, 7): + SIMPLE_EXPECT.add('mitogen.compat.tokenize') + def test_simple(self): import mitogen.fakessh related = self.call('mitogen.fakessh') - self.assertEquals(related, [ - 'mitogen', - 'mitogen.compat', - 'mitogen.compat.collections', - 'mitogen.compat.functools', - 'mitogen.core', - 'mitogen.master', - 'mitogen.minify', - 'mitogen.parent', - ]) + self.assertEquals(set(related), self.SIMPLE_EXPECT) def test_django_pkg(self): import django From cec564654ec5aebc250d4f2b3c32a7f2c9ff898c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 21:12:26 +0100 Subject: [PATCH 18/24] issue #275: tests: fix module_finder_test for 2.6. --- dev_requirements.txt | 4 +- tests/module_finder_test.py | 133 ++++++++++-------------------------- 2 files changed, 38 insertions(+), 99 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 202f2b9f..36e458f3 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,9 +1,9 @@ -r docs/docs-requirements.txt ansible==2.5.2 coverage==4.5.1 -Django==1.6.11; python_version < '2.7' -Django==1.11.5; python_version >= '2.7' # for module_finder_test +Django==1.6.11 # Last version supporting 2.6. mock==2.0.0 +pytz==2012d # Last 2.6-compat version. pytest-catchlog==1.2.2 pytest==3.1.2 PyYAML==3.11; python_version < '2.7' diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index c2e64cfb..d4db641f 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,4 +1,5 @@ import inspect +import os import sys import unittest2 @@ -143,13 +144,6 @@ class FindRelatedImportsTest(testlib.TestCase): 'mitogen.parent', ]) - def test_django_pkg(self): - import django - related = self.call('django') - self.assertEquals(related, [ - 'django.utils.version', - ]) - def test_django_db(self): import django.db related = self.call('django.db') @@ -158,6 +152,7 @@ class FindRelatedImportsTest(testlib.TestCase): 'django.core', 'django.core.signals', 'django.db.utils', + 'django.utils.functional', ]) def test_django_db_models(self): @@ -175,10 +170,9 @@ class FindRelatedImportsTest(testlib.TestCase): 'django.db.models.expressions', 'django.db.models.fields', 'django.db.models.fields.files', - 'django.db.models.fields.proxy', 'django.db.models.fields.related', - 'django.db.models.indexes', - 'django.db.models.lookups', + 'django.db.models.fields.subclassing', + 'django.db.models.loading', 'django.db.models.manager', 'django.db.models.query', 'django.db.models.signals', @@ -210,14 +204,27 @@ class FindRelatedTest(testlib.TestCase): related = self.call('mitogen.fakessh') self.assertEquals(set(related), self.SIMPLE_EXPECT) - def test_django_pkg(self): - import django - related = self.call('django') - self.assertEquals(related, [ - 'django.utils', - 'django.utils.lru_cache', - 'django.utils.version', - ]) + +class DjangoFindRelatedTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + maxDiff = None + + def call(self, fullname): + return self.klass().find_related(fullname) + + WEBPROJECT_PATH = testlib.data_path('webproject') + + @classmethod + def setUpClass(cls): + super(DjangoFindRelatedTest, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoFindRelatedTest, cls).tearDownClass() def test_django_db(self): import django.db @@ -232,17 +239,14 @@ class FindRelatedTest(testlib.TestCase): 'django.db.utils', 'django.dispatch', 'django.dispatch.dispatcher', - 'django.dispatch.weakref_backports', + 'django.dispatch.saferef', 'django.utils', 'django.utils._os', - 'django.utils.deprecation', 'django.utils.encoding', 'django.utils.functional', - 'django.utils.inspect', - 'django.utils.lru_cache', + 'django.utils.importlib', 'django.utils.module_loading', 'django.utils.six', - 'django.utils.version', ]) def test_django_db_models(self): @@ -250,31 +254,9 @@ class FindRelatedTest(testlib.TestCase): related = self.call('django.db.models') self.assertEquals(related, [ 'django', - 'django.apps', - 'django.apps.config', - 'django.apps.registry', 'django.conf', 'django.conf.global_settings', 'django.core', - 'django.core.cache', - 'django.core.cache.backends', - 'django.core.cache.backends.base', - 'django.core.checks', - 'django.core.checks.caches', - 'django.core.checks.compatibility', - 'django.core.checks.compatibility.django_1_10', - 'django.core.checks.compatibility.django_1_8_0', - 'django.core.checks.database', - 'django.core.checks.messages', - 'django.core.checks.model_checks', - 'django.core.checks.registry', - 'django.core.checks.security', - 'django.core.checks.security.base', - 'django.core.checks.security.csrf', - 'django.core.checks.security.sessions', - 'django.core.checks.templates', - 'django.core.checks.urls', - 'django.core.checks.utils', 'django.core.exceptions', 'django.core.files', 'django.core.files.base', @@ -287,7 +269,8 @@ class FindRelatedTest(testlib.TestCase): 'django.core.validators', 'django.db', 'django.db.backends', - 'django.db.backends.utils', + 'django.db.backends.signals', + 'django.db.backends.util', 'django.db.models.aggregates', 'django.db.models.base', 'django.db.models.constants', @@ -297,88 +280,44 @@ class FindRelatedTest(testlib.TestCase): 'django.db.models.fields.files', 'django.db.models.fields.proxy', 'django.db.models.fields.related', - 'django.db.models.fields.related_descriptors', - 'django.db.models.fields.related_lookups', - 'django.db.models.fields.reverse_related', - 'django.db.models.functions', - 'django.db.models.functions.base', - 'django.db.models.functions.datetime', - 'django.db.models.indexes', - 'django.db.models.lookups', + 'django.db.models.fields.subclassing', + 'django.db.models.loading', 'django.db.models.manager', 'django.db.models.options', 'django.db.models.query', 'django.db.models.query_utils', + 'django.db.models.related', 'django.db.models.signals', 'django.db.models.sql', - 'django.db.models.sql.constants', - 'django.db.models.sql.datastructures', - 'django.db.models.sql.query', - 'django.db.models.sql.subqueries', - 'django.db.models.sql.where', - 'django.db.models.utils', 'django.db.transaction', 'django.db.utils', 'django.dispatch', 'django.dispatch.dispatcher', - 'django.dispatch.weakref_backports', + 'django.dispatch.saferef', 'django.forms', - 'django.forms.boundfield', - 'django.forms.fields', - 'django.forms.forms', - 'django.forms.formsets', - 'django.forms.models', - 'django.forms.renderers', - 'django.forms.utils', - 'django.forms.widgets', - 'django.template', - 'django.template.backends', - 'django.template.backends.base', - 'django.template.backends.django', - 'django.template.backends.jinja2', - 'django.template.base', - 'django.template.context', - 'django.template.engine', - 'django.template.exceptions', - 'django.template.library', - 'django.template.loader', - 'django.template.utils', - 'django.templatetags', - 'django.templatetags.static', 'django.utils', 'django.utils._os', 'django.utils.crypto', 'django.utils.datastructures', - 'django.utils.dateformat', 'django.utils.dateparse', - 'django.utils.dates', - 'django.utils.datetime_safe', - 'django.utils.deconstruct', 'django.utils.decorators', 'django.utils.deprecation', - 'django.utils.duration', 'django.utils.encoding', - 'django.utils.formats', 'django.utils.functional', - 'django.utils.html', - 'django.utils.html_parser', - 'django.utils.http', - 'django.utils.inspect', + 'django.utils.importlib', 'django.utils.ipv6', 'django.utils.itercompat', - 'django.utils.lru_cache', 'django.utils.module_loading', - 'django.utils.numberformat', 'django.utils.safestring', 'django.utils.six', 'django.utils.text', 'django.utils.timezone', 'django.utils.translation', 'django.utils.tree', - 'django.utils.version', + 'django.utils.tzinfo', + 'pkg_resources', 'pytz', 'pytz.exceptions', - 'pytz.lazy', 'pytz.tzfile', 'pytz.tzinfo', ]) From 1cb084061c19e79e04a8e4efabd5c6a9d975286a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 21:43:58 +0100 Subject: [PATCH 19/24] issue #275: Pin paramiko to a v2.6-compatible version. --- dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev_requirements.txt b/dev_requirements.txt index 36e458f3..ece4c7ac 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,6 +4,7 @@ coverage==4.5.1 Django==1.6.11 # Last version supporting 2.6. mock==2.0.0 pytz==2012d # Last 2.6-compat version. +paramiko==2.3.1 # Last 2.6-compat version. pytest-catchlog==1.2.2 pytest==3.1.2 PyYAML==3.11; python_version < '2.7' From b7eb96d116706aec495f053447fc9f39cc2932bf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 22:07:05 +0100 Subject: [PATCH 20/24] issue #275: tests: don't explicitly specify interpreter path. --- .travis/ansible_tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index b5162ae0..bd1af85a 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -45,7 +45,6 @@ echo \ target \ ansible_host=$DOCKER_HOSTNAME \ ansible_port=2201 \ - ansible_python_interpreter=/usr/bin/python2.7 \ ansible_user=mitogen__has_sudo_nopw \ ansible_password=has_sudo_nopw_password \ >> ${TMPDIR}/hosts From 4f57c59b7ea3077fd36dc38ede150498df15daf8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 22:43:11 +0100 Subject: [PATCH 21/24] issue #275: Don't run virtualnv test on 2.6. --- .../lib/modules/custom_python_detect_environment.py | 1 + .../regression/issue_152__virtualenv_python_fails.yml | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 8f369e86..d4795422 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -13,6 +13,7 @@ import sys def main(): module = AnsibleModule(argument_spec={}) module.exit_json( + python_version=sys.version[:3], argv=sys.argv, __file__=__file__, argv_types=[str(type(s)) for s in sys.argv], diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index c17b3dd5..0234a0ef 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -2,20 +2,26 @@ any_errors_fatal: true hosts: test-targets tasks: + - custom_python_detect_environment: + register: lout - # Can't use pip module because you can't fricking just create a virtualenv, - # must call it directly. + # Can't use pip module because it can't create virtualenvs, must call it + # directly. - shell: virtualenv /tmp/issue_152_virtualenv + when: lout.python_version != '2.6' - custom_python_detect_environment: vars: ansible_python_interpreter: /tmp/issue_152_virtualenv/bin/python register: out + when: lout.python_version != '2.6' - assert: that: - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" + when: lout.python_version != '2.6' - file: path: /tmp/issue_152_virtualenv state: absent + when: lout.python_version != '2.6' From fb8bad934b465317d522300d9df248959e460c95 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 24 Jun 2018 23:02:25 +0100 Subject: [PATCH 22/24] issue #275: Don't use -U in ansible_tests.sh -- forces paramiko upgrade --- .travis/ansible_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index bd1af85a..ffe775fe 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -36,7 +36,7 @@ echo travis_fold:end:docker_setup echo travis_fold:start:job_setup -pip install -U ansible=="${ANSIBLE_VERSION}" +pip install ansible=="${ANSIBLE_VERSION}" cd ${TRAVIS_BUILD_DIR}/tests/ansible chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key From 3a304b2458493bff3fcbd39b13e718e79de3da0a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 25 Jun 2018 00:41:24 +0100 Subject: [PATCH 23/24] issue #275: su: add CentOS 6 style su failure --- mitogen/su.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitogen/su.py b/mitogen/su.py index ce81eed5..dd340554 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -58,6 +58,7 @@ class Stream(mitogen.parent.Stream): incorrect_prompts = ( 'su: sorry', # BSD 'su: authentication failure', # Linux + 'su: incorrect password', # CentOS 6 ) def construct(self, username=None, password=None, su_path=None, From 8b398e797eb42e1814fc56c277af1a0f6bf39280 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 25 Jun 2018 01:30:23 +0100 Subject: [PATCH 24/24] issue #275: bump ansible to 2.5.5 --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index ece4c7ac..faa7dab9 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ -r docs/docs-requirements.txt -ansible==2.5.2 +ansible==2.5.5 coverage==4.5.1 Django==1.6.11 # Last version supporting 2.6. mock==2.0.0