Merge remote-tracking branch 'origin/dmw'

- issue #421
- Azure Pipelines

.ci/ansible_tests.py

@@ -41,6 +41,7 @@ with ci_lib.Fold('job_setup'):
     run("mkdir %s", HOSTS_DIR)
     run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR)
 
+    docker_hostname = ci_lib.get_docker_hostname()
     with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp:
         fp.write('[test-targets]\n')
         for i, distro in enumerate(ci_lib.DISTROS):
@@ -51,7 +52,7 @@ with ci_lib.Fold('job_setup'):
                      "ansible_password=has_sudo_nopw_password"
                      "\n" % (
                 distro,
-                ci_lib.DOCKER_HOSTNAME,
+                docker_hostname,
                 BASE_PORT + i,
             ))
 

.ci/azure-pipelines.yml

@@ -0,0 +1,86 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+jobs:
+
+- job: 'MitogenTests'
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  strategy:
+    matrix:
+      Mitogen27Debian_27:
+        python.version: '2.7'
+        MODE: mitogen
+        DISTRO: debian
+
+      MitogenPy27CentOS6_26:
+        python.version: '2.7'
+        MODE: mitogen
+        DISTRO: centos6
+
+      #Py26CentOS7:
+        #python.version: '2.7'
+        #MODE: mitogen
+        #DISTRO: centos6
+
+      Mitogen36CentOS6_26:
+        python.version: '3.6'
+        MODE: mitogen
+        DISTRO: centos6
+
+      DebOps_2460_27_27:
+        python.version: '2.7'
+        MODE: debops_common
+        VER: 2.4.6.0
+
+      DebOps_262_36_27:
+        python.version: '3.6'
+        MODE: debops_common
+        VER: 2.6.2
+
+      Ansible_2460_26:
+        python.version: '2.7'
+        MODE: ansible
+        VER: 2.4.6.0
+
+      Ansible_262_26:
+        python.version: '2.7'
+        MODE: ansible
+        VER: 2.6.2
+
+      Ansible_2460_36:
+        python.version: '3.6'
+        MODE: ansible
+        VER: 2.4.6.0
+
+      Ansible_262_36:
+        python.version: '3.6'
+        MODE: ansible
+        VER: 2.6.2
+
+      Vanilla_262_27:
+        python.version: '2.7'
+        MODE: ansible
+        VER: 2.6.2
+        DISTROS: debian
+        STRATEGY: linear
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      architecture: 'x64'
+
+  - script: .ci/prep_azure.py
+    displayName: "Install requirements."
+
+  - script: |
+      export TRAVIS_BUILD_DIR=`pwd`
+      if [ -f ".ci/$(MODE)_tests.sh" ]; then
+        .ci/$(MODE)_tests.sh;
+      else
+        .ci/$(MODE)_tests.py;
+      fi
+    displayName: Run tests.

.ci/ci_lib.py

@@ -10,8 +10,6 @@ import shlex
 import shutil
 import tempfile
 
-import os
-os.system('curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/machine-type')
 
 #
 # check_output() monkeypatch cutpasted from testlib.py
@@ -37,20 +35,21 @@ if not hasattr(subprocess, 'check_output'):
 
 # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars.
 
-proc = subprocess.Popen(
+if sys.platform.startswith('linux'):
-    args=['stdbuf', '-oL', 'cat'],
+    proc = subprocess.Popen(
-    stdin=subprocess.PIPE
+        args=['stdbuf', '-oL', 'cat'],
-)
+        stdin=subprocess.PIPE
+    )
 
-os.dup2(proc.stdin.fileno(), 1)
+    os.dup2(proc.stdin.fileno(), 1)
-os.dup2(proc.stdin.fileno(), 2)
+    os.dup2(proc.stdin.fileno(), 2)
 
-def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc):
+    def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc):
-    stdout.close()
+        stdout.close()
-    stderr.close()
+        stderr.close()
-    proc.terminate()
+        proc.terminate()
 
-atexit.register(cleanup_travis_junk)
+    atexit.register(cleanup_travis_junk)
 
 # -----------------
 
@@ -113,10 +112,11 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
     GIT_ROOT
 )
 
-DOCKER_HOSTNAME = subprocess.check_output([
+def get_docker_hostname():
-    sys.executable,
+    return subprocess.check_output([
-    os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'),
+        sys.executable,
-]).decode().strip()
+        os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'),
+    ]).decode().strip()
 
 # SSH passes these through to the container when run interactively, causing
 # stdout to get messed up with libc warnings.

.ci/mitogen_tests.sh

@@ -2,4 +2,4 @@
 # Run the Mitogen tests.
 
 MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
-MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -vvv
+MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v

.ci/prep_azure.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# Run preparation steps in parallel.
+
+import subprocess
+import ci_lib
+
+subprocess.check_call(
+    'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
+    shell=True,
+)
+
+procs = [
+    subprocess.Popen(
+        'pip install -r dev_requirements.txt 2>&1 | cat',
+        shell=True,
+    ),
+    subprocess.Popen(
+        """
+        sudo add-apt-repository ppa:deadsnakes/ppa && \
+        ( sudo apt-get update 2>&1 | cat ) && \
+        sudo apt-get -y install \
+            python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat
+        """,
+        shell=True,
+    )
+]
+
+procs += [
+    subprocess.Popen(
+        'docker pull mitogen/%s-test 2>&1 | cat' % (distro,),
+        shell=True
+    )
+    for distro in ci_lib.DISTROS
+]
+
+assert [proc.wait() for proc in procs] == [0] * len(procs)

.travis.yml

@@ -19,10 +19,10 @@ install:
 
 script:
 - |
-  if [ -f "${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh" ]; then
+  if [ -f "${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh" ]; then
-    ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh;
+    ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh;
   else
-    ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.py;
+    ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.py;
   fi
 
 

azure-pipelines.yml

@@ -1,55 +0,0 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-jobs:
-
-- job: 'Test'
-  pool:
-    vmImage: 'Ubuntu 16.04'
-  strategy:
-    matrix:
-      Python27:
-        python.version: '2.7'
-      Python35:
-        python.version: '3.5'
-      Python36:
-        python.version: '3.6'
-      Python37:
-        python.version: '3.7'
-    maxParallel: 4
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install -r requirements.txt
-    displayName: 'Install dependencies'
-
-  - script: |
-      pip install pytest
-      pytest tests --doctest-modules --junitxml=junit/test-results.xml
-    displayName: 'pytest'
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '**/test-results.xml'
-      testRunTitle: 'Python $(python.version)'
-    condition: succeededOrFailed()
-
-- job: 'Publish'
-  dependsOn: 'Test'
-  pool:
-    vmImage: 'Ubuntu 16.04'
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.x'
-      architecture: 'x64'
-
-  - script: python setup.py sdist
-    displayName: 'Build sdist'

docs/changelog.rst

@@ -159,7 +159,7 @@ Fixes
 
 * `#334 <https://github.com/dw/mitogen/issues/334>`_: the SSH method
   tilde-expands private key paths using Ansible's logic. Previously the path
-  was passed unmodified to SSH, which expanded it using :func:`os.getpwent`.
+  was passed unmodified to SSH, which expanded it using :func:`pwd.getpwnam`.
   This differs from :func:`os.path.expanduser`, which uses the ``HOME``
   environment variable if it is set, causing behaviour to diverge when Ansible
   was invoked across user accounts via ``sudo``.
@@ -180,8 +180,8 @@ Fixes
   recurrence.
 
 * `#409 <https://github.com/dw/mitogen/issues/409>`_: the LXC and LXD methods
-  support ``mitogen_lxc_path`` and ``mitogen_lxc_attach`` variables to control
+  support ``mitogen_lxc_path`` and ``mitogen_lxc_attach_path`` variables to
-  the location of third pary utilities.
+  control the location of third pary utilities.
 
 * `#410 <https://github.com/dw/mitogen/issues/410>`_: the sudo method supports
   the SELinux ``--type`` and ``--role`` options.
@@ -245,6 +245,10 @@ Core Library
   execution of its :keyword:`finally` block was delayed on Python 3. Now
   callers explicitly close the generator when finished.
 
+* `#421 <https://github.com/dw/mitogen/issues/421>`_: the fork method could
+  fail to start if :data:`sys.stdout` was opened in block buffered mode, and
+  buffered data was pending in the parent prior to fork.
+
 * `16ca111e <https://github.com/dw/mitogen/commit/16ca111e>`_: handle OpenSSH
   7.5 permission denied prompts when ``~/.ssh/config`` rewrites are present.
 

mitogen/core.py

@@ -2808,28 +2808,32 @@ class ExternalContext(object):
         mitogen.parent_ids = self.config['parent_ids'][:]
         mitogen.parent_id = mitogen.parent_ids[0]
 
-    def _setup_stdio(self):
+    def _nullify_stdio(self):
-        # We must open this prior to closing stdout, otherwise it will recycle
+        """
-        # a standard handle, the dup2() will not error, and on closing it, we
+        Open /dev/null to replace stdin, and stdout/stderr temporarily. In case
-        # lose a standrd handle, causing later code to again recycle a standard
+        of odd startup, assume we may be allocated a standard handle.
-        # handle.
+        """
-        fp = open('/dev/null')
+        fd = os.open('/dev/null', os.O_RDWR)
+        try:
+            for stdfd in (0, 1, 2):
+                if fd != stdfd:
+                    os.dup2(fd, stdfd)
+        finally:
+            if fd not in (0, 1, 2):
+                os.close(fd)
 
+    def _setup_stdio(self):
         # When sys.stdout was opened by the runtime, overwriting it will not
-        # cause close to be called. However when forking from a child that
+        # close FD 1. However when forking from a child that previously used
-        # previously used fdopen, overwriting it /will/ cause close to be
+        # fdopen(), overwriting it /will/ close FD 1. So we must swallow the
-        # called. So we must explicitly close it before IoLogger overwrites the
+        # close before IoLogger overwrites FD 1, otherwise its new FD 1 will be
-        # file descriptor, otherwise the assignment below will cause stdout to
+        # clobbered. Additionally, stdout must be replaced with /dev/null prior
-        # be closed.
+        # to stdout.close(), since if block buffering was active in the parent,
+        # any pre-fork buffered data will be flushed on close(), corrupting the
+        # connection to the parent.
+        self._nullify_stdio()
         sys.stdout.close()
-        sys.stdout = None
+        self._nullify_stdio()
-
-        try:
-            os.dup2(fp.fileno(), 0)
-            os.dup2(fp.fileno(), 1)
-            os.dup2(fp.fileno(), 2)
-        finally:
-            fp.close()
 
         self.stdout_log = IoLogger(self.broker, 'stdout', 1)
         self.stderr_log = IoLogger(self.broker, 'stderr', 2)

tests/ansible/integration/connection/_put_file.yml

@@ -19,4 +19,4 @@
 - assert:
     that:
     - original.stat.checksum == copied.stat.checksum
-    - original.stat.mtime|int == copied.stat.mtime|int
+    - (not is_mitogen) or (original.stat.mtime|int == copied.stat.mtime|int)

tests/testlib.py

@@ -282,11 +282,10 @@ class TestCase(unittest2.TestCase):
         'mitogen.master.join_thread_async'
     ])
 
-    @classmethod
+    def _teardown_check_threads(self):
-    def _teardown_check_threads(cls):
         counts = {}
         for thread in threading.enumerate():
-            assert thread.name in cls.ALLOWED_THREADS, \
+            assert thread.name in self.ALLOWED_THREADS, \
                 'Found thread %r still running after tests.' % (thread.name,)
             counts[thread.name] = counts.get(thread.name, 0) + 1
 
@@ -294,20 +293,18 @@ class TestCase(unittest2.TestCase):
             assert counts[name] == 1, \
                 'Found %d copies of thread %r running after tests.' % (name,)
 
-    @classmethod
+    def _teardown_check_fds(self):
-    def _teardown_check_fds(cls):
         mitogen.core.Latch._on_fork()
-        if get_fd_count() != cls._fd_count_before:
+        if get_fd_count() != self._fd_count_before:
             import os; os.system('lsof -p %s' % (os.getpid(),))
             assert 0, "%s leaked FDs. Count before: %s, after: %s" % (
-                cls, cls._fd_count_before, get_fd_count(),
+                self, self._fd_count_before, get_fd_count(),
             )
 
-    @classmethod
+    def tearDown(self):
-    def tearDownClass(cls):
+        self._teardown_check_threads()
-        super(TestCase, cls).tearDownClass()
+        self._teardown_check_fds()
-        cls._teardown_check_threads()
+        super(TestCase, self).tearDown()
-        cls._teardown_check_fds()
 
     def assertRaises(self, exc, func, *args, **kwargs):
         """Like regular assertRaises, except return the exception that was