Merge pull request #287 from dw/issue275

Issue275

.travis.yml

@@ -7,22 +7,58 @@ notifications:
   email: false
 
 language: python
-cache: pip
-
-python:
-- "2.7"
-
-env:
-- MODE=mitogen DISTRO=debian
-- MODE=mitogen DISTRO=centos
-- MODE=debops_common VER=2.4.3.0
-- MODE=debops_common VER=2.5.1
-# Ansible tests.
-- MODE=ansible VER=2.4.3.0 DISTRO=debian
-- MODE=ansible VER=2.5.1 DISTRO=centos
-- MODE=ansible VER=2.5.1 DISTRO=debian
-# Sanity check our tests against vanilla Ansible, they should still pass.
-- MODE=ansible VER=2.5.1 DISTRO=debian STRATEGY=linear
+cache:
+- pip
+- directories:
+  - /home/travis/virtualenv
+
+matrix:
+  include:
+    # Mitogen tests.
+    # 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=mitogen DISTRO=debian
+    # 2.7 -> 2.6
+    - python: "2.7"
+      env: MODE=mitogen DISTRO=centos6
+    # 2.6 -> 2.7
+    - python: "2.6"
+      env: MODE=mitogen DISTRO=centos7
+    # 2.6 -> 2.6
+    - python: "2.6"
+      env: MODE=mitogen DISTRO=centos6
+
+    # Debops tests.
+    # 2.4.3.0; 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=debops_common VER=2.4.3.0
+    # 2.5.5; 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=debops_common VER=2.5.5
+
+    # ansible_mitogen tests.
+    # 2.4.3.0; Debian; 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=ansible VER=2.4.3.0 DISTRO=debian
+    # 2.5.5; Debian; 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=ansible VER=2.5.5 DISTRO=debian
+    # 2.5.5; CentOS; 2.7 -> 2.7
+    - python: "2.7"
+      env: MODE=ansible VER=2.5.5 DISTRO=centos7
+    # 2.5.5; CentOS; 2.7 -> 2.6
+    - python: "2.7"
+      env: MODE=ansible VER=2.5.5 DISTRO=centos6
+    # 2.5.5; CentOS; 2.6 -> 2.7
+    - python: "2.6"
+      env: MODE=ansible VER=2.5.5 DISTRO=centos7
+    # 2.5.5; CentOS; 2.6 -> 2.6
+    - python: "2.6"
+      env: MODE=ansible VER=2.5.5 DISTRO=centos6
+
+    # Sanity check our tests against vanilla Ansible, they should pass.
+    - python: "2.7"
+      env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear
 
 install:
 - pip install -r dev_requirements.txt

.travis/ansible_tests.sh

@@ -36,7 +36,7 @@ echo travis_fold:end:docker_setup
 
 
 echo travis_fold:start:job_setup
-pip install -U ansible=="${ANSIBLE_VERSION}"
+pip install ansible=="${ANSIBLE_VERSION}"
 cd ${TRAVIS_BUILD_DIR}/tests/ansible
 
 chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
@@ -45,7 +45,6 @@ echo \
     target \
     ansible_host=$DOCKER_HOSTNAME \
     ansible_port=2201 \
-    ansible_python_interpreter=/usr/bin/python2.7 \
     ansible_user=mitogen__has_sudo_nopw \
     ansible_password=has_sudo_nopw_password \
     >> ${TMPDIR}/hosts

.travis/mitogen_tests.sh

@@ -2,4 +2,4 @@
 # Run the Mitogen tests.
 
 MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
-MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests
+MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -vvv

dev_requirements.txt

@@ -1,12 +1,10 @@
 -r docs/docs-requirements.txt
-ansible==2.5.2
+ansible==2.5.5
 coverage==4.5.1
-Django==1.6.11; python_version < '2.7'
-Django==1.11.5; python_version >= '2.7' # for module_finder_test
-debops==0.7.2
-https://github.com/docker/docker-py/archive/1.10.6.tar.gz; python_version < '2.7'
-docker[tls]==2.5.1; python_version >= '2.7'
+Django==1.6.11  # Last version supporting 2.6.
 mock==2.0.0
+pytz==2012d  # Last 2.6-compat version.
+paramiko==2.3.1  # Last 2.6-compat version.
 pytest-catchlog==1.2.2
 pytest==3.1.2
 PyYAML==3.11; python_version < '2.7'

docs/howitworks.rst

@@ -51,7 +51,7 @@ can be recovered by the bootstrapped process later. It then forks into a new
 process.
 
 After fork, the parent half overwrites its ``stdin`` with the read end of the
-pipe, and the child half writes the string ``EC0\n``, then begins reading the
+pipe, and the child half writes the string ``MITOGEN0\n``, then begins reading the
 :py:mod:`zlib`-compressed payload supplied on ``stdin`` by the master, and
 writing the decompressed result to the write-end of the UNIX pipe.
 
@@ -112,12 +112,17 @@ fetched from the master a second time.
 Signalling Success
 ##################
 
-Once the first stage has signalled ``EC0\n``, the master knows it is ready to
-receive the compressed bootstrap. After decompressing and writing the bootstrap
-source to its parent Python interpreter, the first stage writes the string
-``EC1\n`` to ``stdout`` before exiting. The master process waits for this
-string before considering bootstrap successful and the child's ``stdio`` ready
-to receive messages.
+Once the first stage has signalled ``MITO000\n``, the master knows it is ready
+to receive the compressed bootstrap. After decompressing and writing the
+bootstrap source to its parent Python interpreter, the first stage writes the
+string ``MITO001\n`` to ``stdout`` before exiting. The master process waits for
+this string before considering bootstrap successful and the child's ``stdio``
+ready to receive messages.
+
+The signal value is 8 bytes to match the minimum chunk size required to
+disambiguate between lines containing an interesting token during SSH password
+authentication, a debug message from the SSH client itself, or a message from
+the first stage.
 
 
 ExternalContext.main()

mitogen/core.py

@@ -278,7 +278,7 @@ class PidfulStreamHandler(logging.StreamHandler):
     def emit(self, record):
         if self.open_pid != os.getpid():
             self._reopen()
-        return super(PidfulStreamHandler, self).emit(record)
+        logging.StreamHandler.emit(self, record)
 
 
 def enable_debug_logging():

mitogen/parent.py

@@ -189,8 +189,9 @@ def _acquire_controlling_tty():
         # On Linux, the controlling tty becomes the first tty opened by a
         # process lacking any prior tty.
         os.close(os.open(os.ttyname(2), os.O_RDWR))
-    if sys.platform.startswith('freebsd') or sys.platform == 'darwin':
-        # On BSD an explicit ioctl is required.
+    if hasattr(termios, 'TIOCSCTTY'):
+        # On BSD an explicit ioctl is required. For some inexplicable reason,
+        # Python 2.6 on Travis also requires it.
         fcntl.ioctl(2, termios.TIOCSCTTY)
 
 
@@ -648,7 +649,7 @@ class Stream(mitogen.core.Stream):
     Base for streams capable of starting new slaves.
     """
     #: The path to the remote Python interpreter.
-    python_path = 'python2.7'
+    python_path = 'python'
 
     #: Maximum time to wait for a connection attempt.
     connect_timeout = 30.0
@@ -789,11 +790,11 @@ class Stream(mitogen.core.Stream):
                 sys.executable += sys.version[:3]
             os.environ['ARGV0']=sys.executable
             os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
-        os.write(1,'EC0\n')
+        os.write(1,'MITO000\n')
         C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
         os.fdopen(W,'w',0).write(C)
         os.fdopen(w,'w',0).write('PREAMBLE_LEN\n'+C)
-        os.write(1,'EC1\n')
+        os.write(1,'MITO001\n')
 
     def get_boot_command(self):
         source = inspect.getsource(self._first_stage)
@@ -869,13 +870,17 @@ class Stream(mitogen.core.Stream):
             self._reap_child()
             raise
 
+    #: For ssh.py, this must be at least max(len('password'), len('debug1:'))
+    EC0_MARKER = 'MITO000\n'
+    EC1_MARKER = 'MITO001\n'
+
     def _ec0_received(self):
         LOG.debug('%r._ec0_received()', self)
         write_all(self.transmit_side.fd, self.get_preamble())
-        discard_until(self.receive_side.fd, 'EC1\n', self.connect_deadline)
+        discard_until(self.receive_side.fd, 'MITO001\n', self.connect_deadline)
 
     def _connect_bootstrap(self, extra_fd):
-        discard_until(self.receive_side.fd, 'EC0\n', self.connect_deadline)
+        discard_until(self.receive_side.fd, 'MITO000\n', self.connect_deadline)
         self._ec0_received()
 
 

mitogen/ssh.py

@@ -53,19 +53,6 @@ HOSTKEY_FAIL = 'host key verification failed.'
 DEBUG_PREFIXES = ('debug1:', 'debug2:', 'debug3:')
 
 
-def _filter_debug(stream, it, buf):
-    while True:
-        if not buf.startswith(DEBUG_PREFIXES):
-            return buf
-        while '\n' in buf:
-            line, _, buf = buf.partition('\n')
-            LOG.debug('%r: received %r', stream, line.rstrip())
-        try:
-            buf += next(it)
-        except StopIteration:
-            return buf
-
-
 def filter_debug(stream, it):
     """
     Read line chunks from it, either yielding them directly, or building up and
@@ -74,10 +61,33 @@ def filter_debug(stream, it):
     This contains the mess of dealing with both line-oriented input, and partial
     lines such as the password prompt.
     """
+    state = 'start_of_line'
+    buf = ''
     for chunk in it:
-        chunk = _filter_debug(stream, it, chunk)
-        if chunk:
-            yield chunk
+        buf += chunk
+        while buf:
+            if state == 'start_of_line':
+                if len(buf) < 8:
+                    # short read near buffer limit, block awaiting at least 8
+                    # bytes so we can discern a debug line, or the minimum
+                    # interesting token from above or the bootstrap
+                    # ('password', 'MITO000\n').
+                    break
+                elif buf.startswith(DEBUG_PREFIXES):
+                    state = 'in_debug'
+                else:
+                    state = 'in_plain'
+            elif state == 'in_debug':
+                if '\n' not in buf:
+                    break
+                line, _, buf = buf.partition('\n')
+                LOG.debug('%r: %s', stream, line.rstrip())
+                state = 'start_of_line'
+            elif state == 'in_plain':
+                line, nl, buf = buf.partition('\n')
+                yield line + nl
+                if nl:
+                    state = 'start_of_line'
 
 
 class PasswordError(mitogen.core.StreamError):
@@ -219,7 +229,7 @@ class Stream(mitogen.parent.Stream):
 
         for buf in filter_debug(self, it):
             LOG.debug('%r: received %r', self, buf)
-            if buf.endswith('EC0\n'):
+            if buf.endswith(self.EC0_MARKER):
                 self._router.broker.start_receive(self.tty_stream)
                 self._ec0_received()
                 return

mitogen/su.py

@@ -58,6 +58,7 @@ class Stream(mitogen.parent.Stream):
     incorrect_prompts = (
         'su: sorry',                    # BSD
         'su: authentication failure',   # Linux
+        'su: incorrect password',       # CentOS 6
     )
 
     def construct(self, username=None, password=None, su_path=None,
@@ -97,7 +98,7 @@ class Stream(mitogen.parent.Stream):
 
         for buf in it:
             LOG.debug('%r: received %r', self, buf)
-            if buf.endswith('EC0\n'):
+            if buf.endswith(self.EC0_MARKER):
                 self._ec0_received()
                 return
             if any(s in buf.lower() for s in self.incorrect_prompts):

mitogen/sudo.py

@@ -168,7 +168,7 @@ class Stream(mitogen.parent.Stream):
 
         for buf in it:
             LOG.debug('%r: received %r', self, buf)
-            if buf.endswith('EC0\n'):
+            if buf.endswith(self.EC0_MARKER):
                 self._ec0_received()
                 return
             elif PASSWORD_PROMPT in buf.lower():

tests/ansible/lib/modules/custom_python_detect_environment.py

@@ -13,6 +13,7 @@ import sys
 def main():
     module = AnsibleModule(argument_spec={})
     module.exit_json(
+        python_version=sys.version[:3],
         argv=sys.argv,
         __file__=__file__,
         argv_types=[str(type(s)) for s in sys.argv],

tests/ansible/regression/issue_152__virtualenv_python_fails.yml

@@ -2,20 +2,26 @@
   any_errors_fatal: true
   hosts: test-targets
   tasks:
+    - custom_python_detect_environment:
+      register: lout
 
-    # Can't use pip module because you can't fricking just create a virtualenv,
-    # must call it directly.
+    # Can't use pip module because it can't create virtualenvs, must call it
+    # directly.
     - shell: virtualenv /tmp/issue_152_virtualenv
+      when: lout.python_version != '2.6'
 
     - custom_python_detect_environment:
       vars:
         ansible_python_interpreter: /tmp/issue_152_virtualenv/bin/python
       register: out
+      when: lout.python_version != '2.6'
 
     - assert:
         that:
           - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python"
+      when: lout.python_version != '2.6'
 
     - file:
         path: /tmp/issue_152_virtualenv
         state: absent
+      when: lout.python_version != '2.6'

tests/build_docker_images.py

@@ -12,7 +12,7 @@ import tempfile
 
 
 DEBIAN_DOCKERFILE = r"""
-FROM debian:stable
+FROM debian:stretch
 RUN apt-get update
 RUN \
     apt-get install -y python2.7 openssh-server sudo rsync git strace \
@@ -21,7 +21,18 @@ RUN \
     rm -rf /var/cache/apt
 """
 
-CENTOS_DOCKERFILE = r"""
+CENTOS6_DOCKERFILE = r"""
+FROM centos:6
+RUN yum clean all && \
+    yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \
+                      perl-JSON python-virtualenv && \
+    yum clean all && \
+    groupadd sudo && \
+    ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
+
+"""
+
+CENTOS7_DOCKERFILE = r"""
 FROM centos:7
 RUN yum clean all && \
     yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \
@@ -49,8 +60,8 @@ RUN \
     useradd -s /bin/bash -m mitogen__readonly_homedir && \
     useradd -s /bin/bash -m mitogen__slow_user && \
     chown -R root: ~mitogen__readonly_homedir && \
-    { for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user$i; done; } && \
-    { for i in `seq 1 21`; do echo mitogen__user$i:user$i_password | chpasswd; done; } && \
+    ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \
+    ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \
     ( echo 'root:rootpassword' | chpasswd; ) && \
     ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \
     ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \
@@ -62,7 +73,7 @@ RUN \
     ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \
     ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \
     mkdir ~mitogen__has_sudo_pubkey/.ssh && \
-    { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; }
+    ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; )
 
 COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys
 COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile
@@ -89,8 +100,11 @@ def sh(s, *args):
     return shlex.split(s)
 
 
-for (distro, wheel, prefix) in (('debian', 'sudo', DEBIAN_DOCKERFILE),
-                                ('centos', 'wheel', CENTOS_DOCKERFILE)):
+for (distro, wheel, prefix) in (
+        ('debian', 'sudo', DEBIAN_DOCKERFILE),
+        ('centos6', 'wheel', CENTOS6_DOCKERFILE),
+        ('centos7', 'wheel', CENTOS7_DOCKERFILE),
+    ):
     mydir = os.path.abspath(os.path.dirname(__file__))
     with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp:
         dockerfile_fp.write(prefix)

tests/first_stage_test.py

@@ -35,7 +35,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase):
         )
         stdout, stderr = proc.communicate()
         self.assertEquals(0, proc.returncode)
-        self.assertEquals("EC0\n", stdout)
+        self.assertEquals(mitogen.parent.Stream.EC0_MARKER, stdout)
         self.assertIn("Error -5 while decompressing data: incomplete or truncated stream", stderr)
 
 

tests/fork_test.py

@@ -19,6 +19,8 @@ PLATFORM_TO_PATH = {
     ('darwin', True): '/usr/lib/libssl.dylib',
     ('linux2', False): '/usr/lib/libssl.so',
     ('linux2', True): '/usr/lib/x86_64-linux-gnu/libssl.so',
+    ('linux3', False): '/usr/lib/libssl.so',
+    ('linux3', True): '/usr/lib/x86_64-linux-gnu/libssl.so',
 }
 
 c_ssl = ctypes.CDLL(PLATFORM_TO_PATH[sys.platform, IS_64BIT])

tests/module_finder_test.py

@@ -1,4 +1,6 @@
 import inspect
+import os
+import sys
 
 import unittest2
 
@@ -142,13 +144,6 @@ class FindRelatedImportsTest(testlib.TestCase):
             'mitogen.parent',
         ])
 
-    def test_django_pkg(self):
-        import django
-        related = self.call('django')
-        self.assertEquals(related, [
-            'django.utils.version',
-        ])
-
     def test_django_db(self):
         import django.db
         related = self.call('django.db')
@@ -157,6 +152,7 @@ class FindRelatedImportsTest(testlib.TestCase):
             'django.core',
             'django.core.signals',
             'django.db.utils',
+            'django.utils.functional',
         ])
 
     def test_django_db_models(self):
@@ -174,10 +170,9 @@ class FindRelatedImportsTest(testlib.TestCase):
             'django.db.models.expressions',
             'django.db.models.fields',
             'django.db.models.fields.files',
-            'django.db.models.fields.proxy',
             'django.db.models.fields.related',
-            'django.db.models.indexes',
-            'django.db.models.lookups',
+            'django.db.models.fields.subclassing',
+            'django.db.models.loading',
             'django.db.models.manager',
             'django.db.models.query',
             'django.db.models.signals',
@@ -190,28 +185,46 @@ class FindRelatedTest(testlib.TestCase):
     def call(self, fullname):
         return self.klass().find_related(fullname)
 
+    SIMPLE_EXPECT = set([
+        'mitogen',
+        'mitogen.compat',
+        'mitogen.compat.collections',
+        'mitogen.compat.functools',
+        'mitogen.core',
+        'mitogen.master',
+        'mitogen.minify',
+        'mitogen.parent',
+    ])
+
+    if sys.version_info < (2, 7):
+        SIMPLE_EXPECT.add('mitogen.compat.tokenize')
+
     def test_simple(self):
         import mitogen.fakessh
         related = self.call('mitogen.fakessh')
-        self.assertEquals(related, [
-            'mitogen',
-            'mitogen.compat',
-            'mitogen.compat.collections',
-            'mitogen.compat.functools',
-            'mitogen.core',
-            'mitogen.master',
-            'mitogen.minify',
-            'mitogen.parent',
-        ])
+        self.assertEquals(set(related), self.SIMPLE_EXPECT)
 
-    def test_django_pkg(self):
-        import django
-        related = self.call('django')
-        self.assertEquals(related, [
-            'django.utils',
-            'django.utils.lru_cache',
-            'django.utils.version',
-        ])
+
+class DjangoFindRelatedTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+    maxDiff = None
+
+    def call(self, fullname):
+        return self.klass().find_related(fullname)
+
+    WEBPROJECT_PATH = testlib.data_path('webproject')
+
+    @classmethod
+    def setUpClass(cls):
+        super(DjangoFindRelatedTest, cls).setUpClass()
+        sys.path.append(cls.WEBPROJECT_PATH)
+        os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings'
+
+    @classmethod
+    def tearDownClass(cls):
+        sys.path.remove(cls.WEBPROJECT_PATH)
+        del os.environ['DJANGO_SETTINGS_MODULE']
+        super(DjangoFindRelatedTest, cls).tearDownClass()
 
     def test_django_db(self):
         import django.db
@@ -226,17 +239,14 @@ class FindRelatedTest(testlib.TestCase):
             'django.db.utils',
             'django.dispatch',
             'django.dispatch.dispatcher',
-            'django.dispatch.weakref_backports',
+            'django.dispatch.saferef',
             'django.utils',
             'django.utils._os',
-            'django.utils.deprecation',
             'django.utils.encoding',
             'django.utils.functional',
-            'django.utils.inspect',
-            'django.utils.lru_cache',
+            'django.utils.importlib',
             'django.utils.module_loading',
             'django.utils.six',
-            'django.utils.version',
         ])
 
     def test_django_db_models(self):
@@ -244,31 +254,9 @@ class FindRelatedTest(testlib.TestCase):
         related = self.call('django.db.models')
         self.assertEquals(related, [
             'django',
-            'django.apps',
-            'django.apps.config',
-            'django.apps.registry',
             'django.conf',
             'django.conf.global_settings',
             'django.core',
-            'django.core.cache',
-            'django.core.cache.backends',
-            'django.core.cache.backends.base',
-            'django.core.checks',
-            'django.core.checks.caches',
-            'django.core.checks.compatibility',
-            'django.core.checks.compatibility.django_1_10',
-            'django.core.checks.compatibility.django_1_8_0',
-            'django.core.checks.database',
-            'django.core.checks.messages',
-            'django.core.checks.model_checks',
-            'django.core.checks.registry',
-            'django.core.checks.security',
-            'django.core.checks.security.base',
-            'django.core.checks.security.csrf',
-            'django.core.checks.security.sessions',
-            'django.core.checks.templates',
-            'django.core.checks.urls',
-            'django.core.checks.utils',
             'django.core.exceptions',
             'django.core.files',
             'django.core.files.base',
@@ -281,7 +269,8 @@ class FindRelatedTest(testlib.TestCase):
             'django.core.validators',
             'django.db',
             'django.db.backends',
-            'django.db.backends.utils',
+            'django.db.backends.signals',
+            'django.db.backends.util',
             'django.db.models.aggregates',
             'django.db.models.base',
             'django.db.models.constants',
@@ -291,88 +280,44 @@ class FindRelatedTest(testlib.TestCase):
             'django.db.models.fields.files',
             'django.db.models.fields.proxy',
             'django.db.models.fields.related',
-            'django.db.models.fields.related_descriptors',
-            'django.db.models.fields.related_lookups',
-            'django.db.models.fields.reverse_related',
-            'django.db.models.functions',
-            'django.db.models.functions.base',
-            'django.db.models.functions.datetime',
-            'django.db.models.indexes',
-            'django.db.models.lookups',
+            'django.db.models.fields.subclassing',
+            'django.db.models.loading',
             'django.db.models.manager',
             'django.db.models.options',
             'django.db.models.query',
             'django.db.models.query_utils',
+            'django.db.models.related',
             'django.db.models.signals',
             'django.db.models.sql',
-            'django.db.models.sql.constants',
-            'django.db.models.sql.datastructures',
-            'django.db.models.sql.query',
-            'django.db.models.sql.subqueries',
-            'django.db.models.sql.where',
-            'django.db.models.utils',
             'django.db.transaction',
             'django.db.utils',
             'django.dispatch',
             'django.dispatch.dispatcher',
-            'django.dispatch.weakref_backports',
+            'django.dispatch.saferef',
             'django.forms',
-            'django.forms.boundfield',
-            'django.forms.fields',
-            'django.forms.forms',
-            'django.forms.formsets',
-            'django.forms.models',
-            'django.forms.renderers',
-            'django.forms.utils',
-            'django.forms.widgets',
-            'django.template',
-            'django.template.backends',
-            'django.template.backends.base',
-            'django.template.backends.django',
-            'django.template.backends.jinja2',
-            'django.template.base',
-            'django.template.context',
-            'django.template.engine',
-            'django.template.exceptions',
-            'django.template.library',
-            'django.template.loader',
-            'django.template.utils',
-            'django.templatetags',
-            'django.templatetags.static',
             'django.utils',
             'django.utils._os',
             'django.utils.crypto',
             'django.utils.datastructures',
-            'django.utils.dateformat',
             'django.utils.dateparse',
-            'django.utils.dates',
-            'django.utils.datetime_safe',
-            'django.utils.deconstruct',
             'django.utils.decorators',
             'django.utils.deprecation',
-            'django.utils.duration',
             'django.utils.encoding',
-            'django.utils.formats',
             'django.utils.functional',
-            'django.utils.html',
-            'django.utils.html_parser',
-            'django.utils.http',
-            'django.utils.inspect',
+            'django.utils.importlib',
             'django.utils.ipv6',
             'django.utils.itercompat',
-            'django.utils.lru_cache',
             'django.utils.module_loading',
-            'django.utils.numberformat',
             'django.utils.safestring',
             'django.utils.six',
             'django.utils.text',
             'django.utils.timezone',
             'django.utils.translation',
             'django.utils.tree',
-            'django.utils.version',
+            'django.utils.tzinfo',
+            'pkg_resources',
             'pytz',
             'pytz.exceptions',
-            'pytz.lazy',
             'pytz.tzfile',
             'pytz.tzinfo',
         ])

tests/responder_test.py

@@ -30,7 +30,7 @@ class GoodModulesTest(testlib.RouterMixin, unittest2.TestCase):
         # Ensure a program composed of a single script can be imported
         # successfully.
         args = [sys.executable, testlib.data_path('self_contained_program.py')]
-        output = subprocess.check_output(args)
+        output = testlib.subprocess__check_output(args)
         self.assertEquals(output, "['__main__', 50]\n")
 
 

tests/show_docker_hostname.py

@@ -5,8 +5,5 @@ For use by the Travis scripts, just print out the hostname of the Docker
 daemon from the environment.
 """
 
-import docker
 import testlib
-
-docker = docker.from_env(version='auto')
-print testlib.get_docker_host(docker)
+print testlib.get_docker_host()

tests/testlib.py

@@ -5,6 +5,7 @@ import os
 import random
 import re
 import socket
+import subprocess
 import sys
 import time
 import urlparse
@@ -15,9 +16,6 @@ import mitogen.core
 import mitogen.master
 import mitogen.utils
 
-if mitogen.is_master:  # TODO: shouldn't be necessary.
-    import docker
-
 
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
 sys.path.append(DATA_DIR)
@@ -34,6 +32,22 @@ def data_path(suffix):
     return path
 
 
+def subprocess__check_output(*popenargs, **kwargs):
+    # Missing from 2.6.
+    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    output, _ = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        raise subprocess.CalledProcessError(retcode, cmd)
+    return output
+
+if hasattr(subprocess, 'check_output'):
+    subprocess__check_output = subprocess.check_output
+
+
 def wait_for_port(
         host,
         port,
@@ -162,11 +176,12 @@ class TestCase(unittest2.TestCase):
         assert 0, '%r did not raise %r' % (func, exc)
 
 
-def get_docker_host(docker):
-    if docker.api.base_url == 'http+docker://localunixsocket':
+def get_docker_host():
+    url = os.environ.get('DOCKER_HOST')
+    if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
 
-    parsed = urlparse.urlparse(docker.api.base_url)
+    parsed = urlparse.urlparse(url)
     return parsed.netloc.partition(':')[0]
 
 
@@ -179,29 +194,47 @@ class DockerizedSshDaemon(object):
             self.image = 'mitogen/%s-test' % (distro,)
         return self.image
 
-    def __init__(self):
-        self.docker = docker.from_env(version='auto')
-        self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),)
-        self.container = self.docker.containers.run(
-            image=self.get_image(),
-            detach=True,
-            privileged=True,
-            publish_all_ports=True,
-        )
-        self.container.reload()
-        self.port = (self.container.attrs['NetworkSettings']['Ports']
-                                         ['22/tcp'][0]['HostPort'])
+    # 22/tcp -> 0.0.0.0:32771
+    PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')
+    port = None
+
+    def _get_container_port(self):
+        s = subprocess__check_output(['docker', 'port', self.container_name])
+        for line in s.splitlines():
+            dport, proto, baddr, bport = self.PORT_RE.match(line).groups()
+            if dport == '22' and proto == 'tcp':
+                self.port = int(bport)
+
         self.host = self.get_host()
+        if self.port is None:
+            raise ValueError('could not find SSH port in: %r' % (s,))
+
+    def start_container(self):
+        self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),)
+        args = [
+            'docker',
+            'run',
+            '--detach',
+            '--privileged',
+            '--publish-all',
+            '--name', self.container_name,
+            self.get_image()
+        ]
+        subprocess__check_output(args)
+        self._get_container_port()
+
+    def __init__(self):
+        self.start_container()
 
     def get_host(self):
-        return get_docker_host(self.docker)
+        return get_docker_host()
 
     def wait_for_sshd(self):
-        wait_for_port(self.get_host(), int(self.port), pattern='OpenSSH')
+        wait_for_port(self.get_host(), self.port, pattern='OpenSSH')
 
     def close(self):
-        self.container.stop()
-        self.container.remove()
+        args = ['docker', 'rm', '-f', self.container_name]
+        subprocess__check_output(args)
 
 
 class BrokerMixin(object):
@@ -244,6 +277,7 @@ class DockerMixin(RouterMixin):
         kwargs.setdefault('hostname', self.dockerized_ssh.host)
         kwargs.setdefault('port', self.dockerized_ssh.port)
         kwargs.setdefault('check_host_keys', 'ignore')
+        kwargs.setdefault('ssh_debug_level', '3')
         return self.router.ssh(**kwargs)
 
     def docker_ssh_any(self, **kwargs):