From f1661abe4e99c72b0efcc6f274e752cf67febc19 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:06:03 +0000 Subject: [PATCH 01/15] tests: make IterReadTest a little more robust --- tests/parent_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/parent_test.py b/tests/parent_test.py index 0bd8079c..ec33415b 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -233,6 +233,7 @@ class IterReadTest(testlib.TestCase): func = staticmethod(mitogen.parent.iter_read) def make_proc(self): + # I produce text every 100ms. args = [testlib.data_path('iter_read_generator.py')] proc = subprocess.Popen(args, stdout=subprocess.PIPE) mitogen.core.set_nonblock(proc.stdout.fileno()) @@ -267,18 +268,22 @@ class IterReadTest(testlib.TestCase): def test_deadline_exceeded_during_call(self): proc = self.make_proc() - reader = self.func([proc.stdout.fileno()], time.time() + 0.4) + deadline = time.time() + 0.4 + + reader = self.func([proc.stdout.fileno()], deadline) try: got = [] try: for chunk in reader: + if time.time() > (deadline + 1.0): + assert 0, 'TimeoutError not raised' got.append(chunk) - assert 0, 'TimeoutError not raised' except mitogen.core.TimeoutError: # Give a little wiggle room in case of imperfect scheduling. # Ideal number should be 9. - self.assertLess(3, len(got)) - self.assertLess(len(got), 5) + self.assertLess(deadline, time.time()) + self.assertLess(1, len(got)) + self.assertLess(len(got), 20) finally: proc.terminate() proc.stdout.close() From 3206d59c8767853004f087c7426deaff40c666f0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:13:05 +0000 Subject: [PATCH 02/15] issue #426: teach DockerMixin to allow selecting interpreter --- tests/testlib.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index b9a1bec5..19f6ed86 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -331,13 +331,17 @@ def get_docker_host(): class DockerizedSshDaemon(object): - image = None - - def get_image(self): - if not self.image: - distro = os.environ.get('MITOGEN_TEST_DISTRO', 'debian') - self.image = 'mitogen/%s-test' % (distro,) - return self.image + distro, _, _py3 = ( + os.environ.get('MITOGEN_TEST_DISTRO', 'debian') + .partition('-') + ) + + python_path = ( + '/usr/bin/python3' + if _py3 == 'py3' + else '/usr/bin/python' + ) + image = 'mitogen/%s-test' % (distro,) # 22/tcp -> 0.0.0.0:32771 PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') @@ -363,7 +367,7 @@ class DockerizedSshDaemon(object): '--privileged', '--publish-all', '--name', self.container_name, - self.get_image() + self.image, ] subprocess__check_output(args) self._get_container_port() @@ -423,6 +427,7 @@ class DockerMixin(RouterMixin): kwargs.setdefault('port', self.dockerized_ssh.port) kwargs.setdefault('check_host_keys', 'ignore') kwargs.setdefault('ssh_debug_level', 3) + kwargs.setdefault('python_path', self.dockerized_ssh.python_path) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): From 934d8ac13905199703e310ccc7ad5812f124c135 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:13:22 +0000 Subject: [PATCH 03/15] issue #426: fix 2->3 issue in plain_old_module. --- tests/data/plain_old_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/plain_old_module.py b/tests/data/plain_old_module.py index 49294464..608f27a5 100755 --- a/tests/data/plain_old_module.py +++ b/tests/data/plain_old_module.py @@ -12,7 +12,7 @@ class MyError(Exception): def get_sentinel_value(): # Some proof we're even talking to the mitogen-test Docker image - return open('/etc/sentinel').read().decode() + return open('/etc/sentinel', 'rb').read().decode() def add(x, y): From c84f36e809ec5f890f5905686194bc7786af4397 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:33:55 +0000 Subject: [PATCH 04/15] issue #426: teach .ci/ Docker pulls to ignore -py3 image suffix. --- .ci/ansible_install.py | 2 +- .ci/ci_lib.py | 4 ++++ .ci/debops_common_install.py | 2 +- .ci/mitogen_install.py | 2 +- .ci/prep_azure.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 167a9cb1..b2da2702 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -14,7 +14,7 @@ batches = [ ] batches.extend( - ['docker pull mitogen/%s-test' % (distro,)] + ['docker pull %s' % (ci_lib.image_for_distro(distro),)] for distro in ci_lib.DISTROS ) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 77cc30a2..7126887b 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -142,6 +142,10 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] +def image_for_distro(distro): + return 'mitogen/%s-test' % (distro.partition('-')[0],) + + def make_containers(): docker_hostname = get_docker_hostname() return [ diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 8830eaf6..32241449 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -13,6 +13,6 @@ ci_lib.run_batches([ 'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION, ], [ - 'docker pull mitogen/debian-test', + 'docker pull %s' % (ci_lib.image_for_distro('debian'),), ], ]) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 4cc06c04..9afbe67d 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -8,7 +8,7 @@ batches = [ 'pip install -r tests/requirements.txt', ], [ - 'docker pull mitogen/%s-test' % (ci_lib.DISTRO,), + 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ] ] diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 85c19947..10126df2 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -15,7 +15,7 @@ batches.append([ ]) batches.extend( - ['docker pull mitogen/%s-test' % (distro,)] + ['docker pull %s' % (ci_lib.image_for_distro(distro),)] for distro in ci_lib.DISTROS ) From 8f03060e0c2c6c1d6be05bce9d2038d3b8b50838 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:14:10 +0000 Subject: [PATCH 05/15] issue #426: enable a 2->3 Mitogen job. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a0d1f924..0092b10c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,9 @@ matrix: # 2.6 -> 2.7 - python: "2.6" env: MODE=mitogen DISTRO=centos7 + # 2.6 -> 3.5 + - python: "2.6" + env: MODE=mitogen DISTRO=debian-py3 # 3.6 -> 2.6 - python: "3.6" env: MODE=mitogen DISTRO=centos6 From 8df895a8ac457676a7e8e232fe21a0eb3e17a49f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:40:37 +0000 Subject: [PATCH 06/15] issue #426: make ansible_tests dump inventory. --- .ci/ansible_tests.py | 4 ++-- .ci/ci_lib.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index bae95902..3161196d 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -45,8 +45,8 @@ with ci_lib.Fold('job_setup'): for container in containers ) - # Build the binaries. - # run("make -C %s", TESTS_DIR) + ci_lib.dump_file(inventory_path) + if not ci_lib.exists_in_path('sshpass'): run("sudo apt-get update") run("sudo apt-get install -y sshpass") diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 7126887b..f52e4cd4 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -177,6 +177,16 @@ def start_containers(containers): return containers +def dump_file(path): + print() + print('--- %s ---' % (path,)) + print() + with open(path, 'r') as fp: + print(fp.read().rstrip()) + print('---') + print() + + # SSH passes these through to the container when run interactively, causing # stdout to get messed up with libc warnings. os.environ.pop('LANG', None) From e7bb5c1ee06aaeaef745a1eb0e65052c2a82ad3f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:41:47 +0000 Subject: [PATCH 07/15] issue #426: teach make_containers() to parse -py3 DISTRO suffix --- .ci/ci_lib.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f52e4cd4..bec700bd 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -148,12 +148,20 @@ def image_for_distro(distro): def make_containers(): docker_hostname = get_docker_hostname() + firstbit = lambda s: (s+'-').split('-')[0] + secondbit = lambda s: (s+'-').split('-')[1] + return [ { - "distro": distro, + "distro": firstbit(distro), "name": "target-%s-%s" % (distro, i), "hostname": docker_hostname, "port": BASE_PORT + i, + "python_path": ( + '/usr/bin/python3' + if secondbit(distro) == 'py3' + else '/usr/bin/python' + ) } for i, distro in enumerate(DISTROS, 1) ] From e12c963279c22a8ef9e3100dfbac405ffcb4dc42 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 00:52:20 +0000 Subject: [PATCH 08/15] issue #426: make ansible_tests use python_path --- .ci/ansible_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 3161196d..ae433bd9 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -32,12 +32,14 @@ with ci_lib.Fold('job_setup'): run("mkdir %s", HOSTS_DIR) run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR) - with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: + inventory_path = os.path.join(HOSTS_DIR, 'target') + with open(inventory_path, 'w') as fp: fp.write('[test-targets]\n') fp.writelines( "%(name)s " "ansible_host=%(hostname)s " "ansible_port=%(port)s " + "ansible_python_interpreter=%(python_path)s " "ansible_user=mitogen__has_sudo_nopw " "ansible_password=has_sudo_nopw_password" "\n" From 4db2168f83a5efd637861ad0174423b21c635444 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 01:16:18 +0000 Subject: [PATCH 09/15] issue #426: teach debops_common_tests to use py3 prefix --- .ci/debops_common_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 04fbb938..8e9f2953 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -55,6 +55,7 @@ with ci_lib.Fold('job_setup'): '%(name)s ' 'ansible_host=%(hostname)s ' 'ansible_port=%(port)d ' + 'ansible_python_interpreter=%(python_path)s ' '\n' % container for container in containers From 57504ba6ec1d42972c3732013bb573e4c93b3315 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 01:58:30 +0000 Subject: [PATCH 10/15] issue #109: core: meta_path regression in newer Pythons Python at some point (at least since https://bugs.python.org/issue14605) began populating sys.meta_path with its internal importer classes, meaning that interpreters no longer start with an empty sys.meta_path. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index cb64fc92..7f00da05 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2791,7 +2791,7 @@ class ExternalContext(object): self.importer = importer self.router.importer = importer - sys.meta_path.append(self.importer) + sys.meta_path.insert(0, self.importer) def _setup_package(self): global mitogen From 1756cea65bcc444c87c1b0157afbaaed1b6685dc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 02:12:25 +0000 Subject: [PATCH 11/15] issue #109: update Changelog. --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99b30214..308fe4b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -210,6 +210,12 @@ Core Library receivers to wake with :class:`mitogen.core.ChannelError`, even when one participant is not a parent of the other. +* `#109 `_, + `57504ba6 `_: newer Python 3 + releases explicitly populate :data:`sys.meta_path` with importer internals, + causing Mitogen to install itself at the end of the importer chain rather + than the front. + * `#387 `_, `#413 `_: dead messages include an optional reason in their body. This is used to cause From 15ddecdb583bbc7d2e7d1f002b0c30c9a22f805d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 02:20:34 +0000 Subject: [PATCH 12/15] issue #391: FileService metadata dict keys must be Unicode. This is a regression since moving FileService from a 2.6-compatible file with unicode_literals set, to a <2.5-compatible file. --- mitogen/service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mitogen/service.py b/mitogen/service.py index 0bafc899..e2b752fe 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -787,12 +787,12 @@ class FileService(Service): raise IOError('%r is not a regular file.' % (path,)) return { - 'size': st.st_size, - 'mode': st.st_mode, - 'owner': self._name_or_none(pwd.getpwuid, 0, 'pw_name'), - 'group': self._name_or_none(grp.getgrgid, 0, 'gr_name'), - 'mtime': st.st_mtime, - 'atime': st.st_atime, + u'size': st.st_size, + u'mode': st.st_mode, + u'owner': self._name_or_none(pwd.getpwuid, 0, 'pw_name'), + u'group': self._name_or_none(grp.getgrgid, 0, 'gr_name'), + u'mtime': st.st_mtime, + u'atime': st.st_atime, } def on_shutdown(self): From 4553039ed2ce46a5fcdf9a80b3fb5ea828e17c6b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 02:23:53 +0000 Subject: [PATCH 13/15] docs: update Changelog; closes #391. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 308fe4b3..140e86cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -176,6 +176,11 @@ Fixes print a useful hint on failure, as no useful error is normally logged to the console by these tools. +* `#391 `_: file transfer from 2.x + controllers to 3.x targets was broken due to a regression caused by + refactoring, and compounded by `#426 + `_. + * `#400 `_: work around a threading bug in the AWX display callback when running with high verbosity setting. From 44d6ca771a5e851705957a1dc46c0e5d89fb1a0a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 02:26:52 +0000 Subject: [PATCH 14/15] issue #426: fix local/delegate_to issue --- tests/ansible/bench/file_transfer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index d0ac727d..2ca46f1c 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -5,11 +5,11 @@ tasks: - name: Make 32MiB file - connection: local + delegate_to: localhost shell: openssl rand 33554432 > /tmp/bigfile.in - name: Make 320MiB file - connection: local + delegate_to: localhost shell: > cat /tmp/bigfile.in @@ -47,21 +47,21 @@ file: path: "{{item}}" state: absent - connection: local + delegate_to: localhost become: true with_items: - /tmp/bigfile.out - /tmp/bigbigfile.out - name: Copy 32MiB file via localhost sudo - connection: local + delegate_to: localhost become: true copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out - name: Copy 320MiB file via localhost sudo - connection: local + delegate_to: localhost become: true copy: src: /tmp/bigbigfile.in From 16911c9464af72510d66d86e3f359dc771e1ed01 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 7 Nov 2018 02:28:06 +0000 Subject: [PATCH 15/15] ansible: fix 3.x TypeError regression. --- ansible_mitogen/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 45bb5f0b..5ab0236c 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -255,7 +255,7 @@ class Runner(object): self.service_context = service_context self.econtext = econtext self.detach = detach - self.args = json.loads(json_args) + self.args = json.loads(mitogen.core.to_text(json_args)) self.good_temp_dir = good_temp_dir self.extra_env = extra_env self.env = env