Merge remote-tracking branch 'origin/dmw'

- 3.x target test job support
- new 2.x->3.x Mitogen job
- 3.x runner regression
- 3.x importer regression
- #109
- #391

.ci/ansible_install.py

@@ -14,7 +14,7 @@ batches = [
 ]
 
 batches.extend(
-    ['docker pull mitogen/%s-test' % (distro,)]
+    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
     for distro in ci_lib.DISTROS
 )
 

.ci/ansible_tests.py

@@ -32,12 +32,14 @@ with ci_lib.Fold('job_setup'):
     run("mkdir %s", HOSTS_DIR)
     run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR)
 
-    with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp:
+    inventory_path = os.path.join(HOSTS_DIR, 'target')
+    with open(inventory_path, 'w') as fp:
         fp.write('[test-targets]\n')
         fp.writelines(
             "%(name)s "
             "ansible_host=%(hostname)s "
             "ansible_port=%(port)s "
+            "ansible_python_interpreter=%(python_path)s "
             "ansible_user=mitogen__has_sudo_nopw "
             "ansible_password=has_sudo_nopw_password"
             "\n"
@@ -45,8 +47,8 @@ with ci_lib.Fold('job_setup'):
             for container in containers
         )
 
-    # Build the binaries.
-    # run("make -C %s", TESTS_DIR)
+    ci_lib.dump_file(inventory_path)
+
     if not ci_lib.exists_in_path('sshpass'):
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")

.ci/ci_lib.py

@@ -142,14 +142,26 @@ def get_docker_hostname():
     return parsed.netloc.partition(':')[0]
 
 
+def image_for_distro(distro):
+    return 'mitogen/%s-test' % (distro.partition('-')[0],)
+
+
 def make_containers():
     docker_hostname = get_docker_hostname()
+    firstbit = lambda s: (s+'-').split('-')[0]
+    secondbit = lambda s: (s+'-').split('-')[1]
+
     return [
         {
-            "distro": distro,
+            "distro": firstbit(distro),
             "name": "target-%s-%s" % (distro, i),
             "hostname": docker_hostname,
             "port": BASE_PORT + i,
+            "python_path": (
+                '/usr/bin/python3'
+                if secondbit(distro) == 'py3'
+                else '/usr/bin/python'
+            )
         }
         for i, distro in enumerate(DISTROS, 1)
     ]
@@ -173,6 +185,16 @@ def start_containers(containers):
     return containers
 
 
+def dump_file(path):
+    print()
+    print('--- %s ---' % (path,))
+    print()
+    with open(path, 'r') as fp:
+        print(fp.read().rstrip())
+    print('---')
+    print()
+
+
 # SSH passes these through to the container when run interactively, causing
 # stdout to get messed up with libc warnings.
 os.environ.pop('LANG', None)

.ci/debops_common_install.py

@@ -13,6 +13,6 @@ ci_lib.run_batches([
         'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
     ],
     [
-        'docker pull mitogen/debian-test',
+        'docker pull %s' % (ci_lib.image_for_distro('debian'),),
     ],
 ])

.ci/debops_common_tests.py

@@ -55,6 +55,7 @@ with ci_lib.Fold('job_setup'):
             '%(name)s '
             'ansible_host=%(hostname)s '
             'ansible_port=%(port)d '
+            'ansible_python_interpreter=%(python_path)s '
             '\n'
             % container
             for container in containers

.ci/mitogen_install.py

@@ -8,7 +8,7 @@ batches = [
         'pip install -r tests/requirements.txt',
     ],
     [
-        'docker pull mitogen/%s-test' % (ci_lib.DISTRO,),
+        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ]
 ]
 

.ci/prep_azure.py

@@ -15,7 +15,7 @@ batches.append([
 ])
 
 batches.extend(
-    ['docker pull mitogen/%s-test' % (distro,)]
+    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
     for distro in ci_lib.DISTROS
 )
 

.travis.yml

@@ -32,6 +32,9 @@ matrix:
     # 2.6 -> 2.7
     - python: "2.6"
       env: MODE=mitogen DISTRO=centos7
+    # 2.6 -> 3.5
+    - python: "2.6"
+      env: MODE=mitogen DISTRO=debian-py3
     # 3.6 -> 2.6
     - python: "3.6"
       env: MODE=mitogen DISTRO=centos6

ansible_mitogen/runner.py

@@ -255,7 +255,7 @@ class Runner(object):
         self.service_context = service_context
         self.econtext = econtext
         self.detach = detach
-        self.args = json.loads(json_args)
+        self.args = json.loads(mitogen.core.to_text(json_args))
         self.good_temp_dir = good_temp_dir
         self.extra_env = extra_env
         self.env = env

docs/changelog.rst

@@ -176,6 +176,11 @@ Fixes
   print a useful hint on failure, as no useful error is normally logged to the
   console by these tools.
 
+* `#391 <https://github.com/dw/mitogen/issues/391>`_: file transfer from 2.x
+  controllers to 3.x targets was broken due to a regression caused by
+  refactoring, and compounded by `#426
+  <https://github.com/dw/mitogen/issues/426>`_.
+
 * `#400 <https://github.com/dw/mitogen/issues/400>`_: work around a threading
   bug in the AWX display callback when running with high verbosity setting.
 
@@ -210,6 +215,12 @@ Core Library
   receivers to wake with :class:`mitogen.core.ChannelError`, even when one
   participant is not a parent of the other.
 
+* `#109 <https://github.com/dw/mitogen/issues/109>`_,
+  `57504ba6 <https://github.com/dw/mitogen/commit/57504ba6>`_: newer Python 3
+  releases explicitly populate :data:`sys.meta_path` with importer internals,
+  causing Mitogen to install itself at the end of the importer chain rather
+  than the front.
+
 * `#387 <https://github.com/dw/mitogen/issues/387>`_,
   `#413 <https://github.com/dw/mitogen/issues/413>`_: dead messages include an
   optional reason in their body. This is used to cause

mitogen/core.py

@@ -2791,7 +2791,7 @@ class ExternalContext(object):
 
         self.importer = importer
         self.router.importer = importer
-        sys.meta_path.append(self.importer)
+        sys.meta_path.insert(0, self.importer)
 
     def _setup_package(self):
         global mitogen

mitogen/service.py

@@ -787,12 +787,12 @@ class FileService(Service):
             raise IOError('%r is not a regular file.' % (path,))
 
         return {
-            'size': st.st_size,
-            'mode': st.st_mode,
-            'owner': self._name_or_none(pwd.getpwuid, 0, 'pw_name'),
-            'group': self._name_or_none(grp.getgrgid, 0, 'gr_name'),
-            'mtime': st.st_mtime,
-            'atime': st.st_atime,
+            u'size': st.st_size,
+            u'mode': st.st_mode,
+            u'owner': self._name_or_none(pwd.getpwuid, 0, 'pw_name'),
+            u'group': self._name_or_none(grp.getgrgid, 0, 'gr_name'),
+            u'mtime': st.st_mtime,
+            u'atime': st.st_atime,
         }
 
     def on_shutdown(self):

tests/ansible/bench/file_transfer.yml

@@ -5,11 +5,11 @@
   tasks:
 
   - name: Make 32MiB file
-    connection: local
+    delegate_to: localhost
     shell: openssl rand 33554432 > /tmp/bigfile.in
 
   - name: Make 320MiB file
-    connection: local
+    delegate_to: localhost
     shell: >
       cat
       /tmp/bigfile.in
@@ -47,21 +47,21 @@
     file:
       path: "{{item}}"
       state: absent
-    connection: local
+    delegate_to: localhost
     become: true
     with_items:
     - /tmp/bigfile.out
     - /tmp/bigbigfile.out
 
   - name: Copy 32MiB file via localhost sudo
-    connection: local
+    delegate_to: localhost
     become: true
     copy:
       src: /tmp/bigfile.in
       dest: /tmp/bigfile.out
 
   - name: Copy 320MiB file via localhost sudo
-    connection: local
+    delegate_to: localhost
     become: true
     copy:
       src: /tmp/bigbigfile.in

tests/data/plain_old_module.py

@@ -12,7 +12,7 @@ class MyError(Exception):
 
 def get_sentinel_value():
     # Some proof we're even talking to the mitogen-test Docker image
-    return open('/etc/sentinel').read().decode()
+    return open('/etc/sentinel', 'rb').read().decode()
 
 
 def add(x, y):

tests/parent_test.py

@@ -233,6 +233,7 @@ class IterReadTest(testlib.TestCase):
     func = staticmethod(mitogen.parent.iter_read)
 
     def make_proc(self):
+        # I produce text every 100ms.
         args = [testlib.data_path('iter_read_generator.py')]
         proc = subprocess.Popen(args, stdout=subprocess.PIPE)
         mitogen.core.set_nonblock(proc.stdout.fileno())
@@ -267,18 +268,22 @@ class IterReadTest(testlib.TestCase):
 
     def test_deadline_exceeded_during_call(self):
         proc = self.make_proc()
-        reader = self.func([proc.stdout.fileno()], time.time() + 0.4)
+        deadline = time.time() + 0.4
+
+        reader = self.func([proc.stdout.fileno()], deadline)
         try:
             got = []
             try:
                 for chunk in reader:
-                    got.append(chunk)
+                    if time.time() > (deadline + 1.0):
                         assert 0, 'TimeoutError not raised'
+                    got.append(chunk)
             except mitogen.core.TimeoutError:
                 # Give a little wiggle room in case of imperfect scheduling.
                 # Ideal number should be 9.
-                self.assertLess(3, len(got))
-                self.assertLess(len(got), 5)
+                self.assertLess(deadline, time.time())
+                self.assertLess(1, len(got))
+                self.assertLess(len(got), 20)
         finally:
             proc.terminate()
             proc.stdout.close()

tests/testlib.py

@@ -331,13 +331,17 @@ def get_docker_host():
 
 
 class DockerizedSshDaemon(object):
-    image = None
+    distro, _, _py3 = (
+        os.environ.get('MITOGEN_TEST_DISTRO', 'debian')
+        .partition('-')
+    )
 
-    def get_image(self):
-        if not self.image:
-            distro = os.environ.get('MITOGEN_TEST_DISTRO', 'debian')
-            self.image = 'mitogen/%s-test' % (distro,)
-        return self.image
+    python_path = (
+        '/usr/bin/python3'
+        if _py3 == 'py3'
+        else '/usr/bin/python'
+    )
+    image = 'mitogen/%s-test' % (distro,)
 
     # 22/tcp -> 0.0.0.0:32771
     PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')
@@ -363,7 +367,7 @@ class DockerizedSshDaemon(object):
             '--privileged',
             '--publish-all',
             '--name', self.container_name,
-            self.get_image()
+            self.image,
         ]
         subprocess__check_output(args)
         self._get_container_port()
@@ -423,6 +427,7 @@ class DockerMixin(RouterMixin):
         kwargs.setdefault('port', self.dockerized_ssh.port)
         kwargs.setdefault('check_host_keys', 'ignore')
         kwargs.setdefault('ssh_debug_level', 3)
+        kwargs.setdefault('python_path', self.dockerized_ssh.python_path)
         return self.router.ssh(**kwargs)
 
     def docker_ssh_any(self, **kwargs):