From cb595b30d4d4a2eaac856aa32099ac0188c013e3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 17 Apr 2018 17:35:19 +0100 Subject: [PATCH 01/50] issue #16: rearrange Travis matrix and add Python 3.6 target for core tests --- .travis.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94a9aff5..85bbc1e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,21 @@ notifications: email: false language: python + cache: - pip - directories: - /home/travis/virtualenv +install: +- pip install -r dev_requirements.txt + +script: +- ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh + +services: + - docker + matrix: include: # Mitogen tests. @@ -27,6 +37,9 @@ matrix: # 2.6 -> 2.6 - python: "2.6" env: MODE=mitogen DISTRO=centos6 + # 3.6 -> 2.7 + - python: "3.6" + env: MODE=mitogen DISTRO=debian # Debops tests. # 2.4.3.0; 2.7 -> 2.7 @@ -35,6 +48,9 @@ matrix: # 2.5.5; 2.7 -> 2.7 - python: "2.7" env: MODE=debops_common VER=2.5.5 + # 2.5.5; 3.6 -> 2.7 + - python: "3.6" + env: MODE=debops_common VER=2.5.5 # ansible_mitogen tests. # 2.4.3.0; Debian; 2.7 -> 2.7 @@ -55,16 +71,10 @@ matrix: # 2.5.5; CentOS; 2.6 -> 2.6 - python: "2.6" env: MODE=ansible VER=2.5.5 DISTRO=centos6 + # 2.5.5; Debian; 3.6 -> 2.7 + - python: "3.6" + env: MODE=ansible VER=2.5.5 DISTRO=centos6 # Sanity check our tests against vanilla Ansible, they should pass. - python: "2.7" env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear - -install: -- pip install -r dev_requirements.txt - -script: -- ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh - -services: - - docker From 410016ff479b717b2066cb3e1c673658d1ab43fe Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 17 Apr 2018 17:40:45 +0100 Subject: [PATCH 02/50] Initial Python 3.x port work. * ansible: use unicode_literals everywhere since it only needs to be compatible back to 2.6. * compat/collections.py: delete this entirely and rip out the parts of functools that require it. * Introduce serializable Kwargs dict subclass that translates keys to Unicode on instantiation. * enable_debug_logging() must set _v/_vv globals. * cStringIO does not exist in 3.x. * Treat IOLogger and LogForwarder input as latin-1. * Avoid ResourceWarnings in first stage by explicitly closing fps. * Fix preamble_size.py syntax errors. --- .travis.yml | 48 ++--- ansible_mitogen/connection.py | 10 +- ansible_mitogen/mixins.py | 10 +- ansible_mitogen/module_finder.py | 2 + ansible_mitogen/planner.py | 28 +-- ansible_mitogen/process.py | 11 +- ansible_mitogen/runner.py | 91 +++++---- ansible_mitogen/services.py | 27 ++- ansible_mitogen/target.py | 21 +- docs/api.rst | 10 +- docs/examples.rst | 2 +- docs/getting_started.rst | 12 +- docs/index.rst | 8 +- examples/mitop.py | 4 +- examples/service/client.py | 2 +- mitogen/__init__.py | 2 +- mitogen/compat/collections.py | 117 ------------ mitogen/compat/functools.py | 19 +- mitogen/core.py | 179 ++++++++++++------ mitogen/docker.py | 2 +- mitogen/fakessh.py | 8 +- mitogen/fork.py | 4 +- mitogen/jail.py | 2 +- mitogen/lxc.py | 2 +- mitogen/master.py | 88 +++++++-- mitogen/minify.py | 11 +- mitogen/parent.py | 145 +++++++++----- mitogen/service.py | 39 ++-- mitogen/setns.py | 2 +- mitogen/ssh.py | 25 +-- mitogen/su.py | 15 +- mitogen/sudo.py | 10 +- mitogen/unix.py | 6 +- mitogen/utils.py | 13 +- preamble_size.py | 6 +- setup.py | 1 - tests/ansible/compare_output_test.py | 4 +- .../integration/action/transfer_data.yml | 4 +- .../library/uses_custom_known_hosts.py | 4 +- .../runner/custom_python_new_style_module.yml | 2 +- tests/ansible/lib/callback/nice_stdout.py | 14 +- .../custom_python_detect_environment.py | 3 +- .../modules/custom_python_json_args_module.py | 10 +- ...om_python_new_style_missing_interpreter.py | 10 +- .../modules/custom_python_new_style_module.py | 14 +- .../modules/custom_python_want_json_module.py | 12 +- tests/bench/local.py | 12 +- tests/bench/megatime.py | 8 +- tests/call_error_test.py | 39 ++-- tests/call_function_test.py | 14 +- .../data/module_finder_testmod/regular_mod.py | 2 +- tests/data/self_contained_program.py | 2 +- tests/data/six_brokenpkg/__init__.py | 4 +- tests/first_stage_test.py | 4 +- tests/fork_test.py | 4 + tests/id_allocation_test.py | 2 +- tests/importer_test.py | 7 +- tests/latch_test.py | 15 +- tests/module_finder_test.py | 1 - tests/nested_test.py | 1 - tests/receiver_test.py | 2 +- tests/router_test.py | 6 +- tests/show_docker_hostname.py | 2 +- tests/soak/latch.py | 4 +- tests/ssh_test.py | 14 +- tests/testlib.py | 42 ++-- tests/two_three_compat_test.py | 33 ++++ 67 files changed, 722 insertions(+), 555 deletions(-) delete mode 100644 mitogen/compat/collections.py create mode 100644 tests/two_three_compat_test.py diff --git a/.travis.yml b/.travis.yml index 85bbc1e4..2696c437 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,52 +29,52 @@ matrix: - python: "2.7" env: MODE=mitogen DISTRO=debian # 2.7 -> 2.6 - - python: "2.7" - env: MODE=mitogen DISTRO=centos6 + #- python: "2.7" + #env: MODE=mitogen DISTRO=centos6 # 2.6 -> 2.7 - - python: "2.6" - env: MODE=mitogen DISTRO=centos7 + #- python: "2.6" + #env: MODE=mitogen DISTRO=centos7 # 2.6 -> 2.6 - - python: "2.6" - env: MODE=mitogen DISTRO=centos6 + #- python: "2.6" + #env: MODE=mitogen DISTRO=centos6 # 3.6 -> 2.7 - python: "3.6" env: MODE=mitogen DISTRO=debian # Debops tests. # 2.4.3.0; 2.7 -> 2.7 - - python: "2.7" - env: MODE=debops_common VER=2.4.3.0 + #- python: "2.7" + #env: MODE=debops_common VER=2.4.3.0 # 2.5.5; 2.7 -> 2.7 - - python: "2.7" - env: MODE=debops_common VER=2.5.5 + #- python: "2.7" + #env: MODE=debops_common VER=2.5.5 # 2.5.5; 3.6 -> 2.7 - - python: "3.6" - env: MODE=debops_common VER=2.5.5 + #- python: "3.6" + #env: MODE=debops_common VER=2.5.5 # ansible_mitogen tests. # 2.4.3.0; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.4.3.0 DISTRO=debian + #- python: "2.7" + #env: MODE=ansible VER=2.4.3.0 DISTRO=debian # 2.5.5; Debian; 2.7 -> 2.7 - python: "2.7" env: MODE=ansible VER=2.5.5 DISTRO=debian # 2.5.5; CentOS; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 + #- python: "2.7" + #env: MODE=ansible VER=2.5.5 DISTRO=centos7 # 2.5.5; CentOS; 2.7 -> 2.6 - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 + #- python: "2.7" + #env: MODE=ansible VER=2.5.5 DISTRO=centos6 # 2.5.5; CentOS; 2.6 -> 2.7 - - python: "2.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 + #- python: "2.6" + #env: MODE=ansible VER=2.5.5 DISTRO=centos7 # 2.5.5; CentOS; 2.6 -> 2.6 - - python: "2.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 + #- python: "2.6" + #env: MODE=ansible VER=2.5.5 DISTRO=centos6 # 2.5.5; Debian; 3.6 -> 2.7 - python: "3.6" env: MODE=ansible VER=2.5.5 DISTRO=centos6 # Sanity check our tests against vanilla Ansible, they should pass. - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear + #- python: "2.7" + #env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 7c3f59c5..8ceb0dee 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -27,6 +27,8 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +from __future__ import unicode_literals + import logging import os import shlex @@ -239,7 +241,7 @@ def config_from_play_context(transport, inventory_name, connection): 'timeout': connection._play_context.timeout, 'ansible_ssh_timeout': connection.ansible_ssh_timeout, 'ssh_args': [ - term + mitogen.core.to_text(term) for s in ( getattr(connection._play_context, 'ssh_args', ''), getattr(connection._play_context, 'ssh_common_args', ''), @@ -249,7 +251,7 @@ def config_from_play_context(transport, inventory_name, connection): ], 'become_exe': connection._play_context.become_exe, 'sudo_args': [ - term + mitogen.core.to_text(term) for s in ( connection._play_context.sudo_flags, connection._play_context.become_flags @@ -351,7 +353,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): def __init__(self, play_context, new_stdin, **kwargs): assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( 'Mitogen connection types may only be instantiated ' - 'while the "mitogen" strategy is active.' + 'while the "mitogen" strategy is active.' ) super(Connection, self).__init__(play_context, new_stdin) @@ -528,7 +530,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): return self.call_async(func, *args, **kwargs).get().unpickle() finally: LOG.debug('Call took %d ms: %s%r', 1000 * (time.time() - t0), - func.func_name, args) + func.__name__, args) def create_fork_child(self): """ diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index efa0bd5a..22d9c445 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -27,13 +27,17 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import -import commands import logging import os import pwd import shutil import traceback +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + from ansible.module_utils._text import to_bytes from ansible.parsing.utils.jsonify import jsonify @@ -321,7 +325,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): ansible_mitogen.planner.Invocation( action=self, connection=self._connection, - module_name=mitogen.utils.cast(module_name), + module_name=mitogen.core.to_text(module_name), module_args=mitogen.utils.cast(module_args), task_vars=task_vars, templar=self._templar, @@ -369,7 +373,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if executable is None: # executable defaults to False executable = self._play_context.executable if executable: - cmd = executable + ' -c ' + commands.mkarg(cmd) + cmd = executable + ' -c ' + shlex_quote(cmd) rc, stdout, stderr = self._connection.exec_command( cmd=cmd, diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 79e1882c..b9b62b73 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -27,6 +27,8 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +from __future__ import unicode_literals + import collections import imp import os diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 4c3ef876..007974ad 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -35,6 +35,8 @@ files/modules known missing. """ from __future__ import absolute_import +from __future__ import unicode_literals + import json import logging import os @@ -43,6 +45,7 @@ import random from ansible.executor import module_common import ansible.errors import ansible.module_utils +import mitogen.core try: from ansible.plugins.loader import module_loader @@ -71,11 +74,11 @@ def parse_script_interpreter(source): """ # Linux requires first 2 bytes with no whitespace, pretty sure it's the # same everywhere. See binfmt_script.c. - if not source.startswith('#!'): + if not source.startswith(b'#!'): return None, None # Find terminating newline. Assume last byte of binprm_buf if absent. - nl = source.find('\n', 0, 128) + nl = source.find(b'\n', 0, 128) if nl == -1: nl = min(128, len(source)) @@ -83,8 +86,8 @@ def parse_script_interpreter(source): # bits just contains the interpreter filename. bits = source[2:nl].strip().split(None, 1) if len(bits) == 1: - return bits[0], None - return bits[0], bits[1] + return mitogen.core.to_text(bits[0]), None + return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1]) class Invocation(object): @@ -176,9 +179,11 @@ class Planner(object): # named by `runner_name`. } """ - kwargs.setdefault('emulate_tty', True) - kwargs.setdefault('service_context', self._inv.connection.parent) - return kwargs + new = dict((mitogen.core.UnicodeType(k), kwargs[k]) + for k in kwargs) + new.setdefault('emulate_tty', True) + new.setdefault('service_context', self._inv.connection.parent) + return new def __repr__(self): return '%s()' % (type(self).__name__,) @@ -202,7 +207,7 @@ class BinaryPlanner(Planner): runner_name=self.runner_name, module=self._inv.module_name, path=self._inv.module_path, - args=self._inv.module_args, + json_args=json.dumps(self._inv.module_args), env=self._inv.env, **kwargs ) @@ -264,7 +269,7 @@ class WantJsonPlanner(ScriptPlanner): runner_name = 'WantJsonRunner' def detect(self): - return 'WANT_JSON' in self._inv.module_source + return b'WANT_JSON' in self._inv.module_source class NewStylePlanner(ScriptPlanner): @@ -274,9 +279,10 @@ class NewStylePlanner(ScriptPlanner): preprocessing the module. """ runner_name = 'NewStyleRunner' + marker = b'from ansible.module_utils.' def detect(self): - return 'from ansible.module_utils.' in self._inv.module_source + return self.marker in self._inv.module_source def _get_interpreter(self): return None, None @@ -394,7 +400,7 @@ def get_module_data(name): path = module_loader.find_plugin(name, '') with open(path, 'rb') as fp: source = fp.read() - return path, source + return mitogen.core.to_text(path), source def _propagate_deps(invocation, planner, context): diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index ff637299..69e70571 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -46,6 +46,8 @@ import mitogen.utils import ansible_mitogen.logging import ansible_mitogen.services +from mitogen.core import b + LOG = logging.getLogger(__name__) @@ -103,8 +105,8 @@ class MuxProcess(object): cls.unix_listener_path = mitogen.unix.make_socket_path() cls.worker_sock, cls.child_sock = socket.socketpair() - mitogen.core.set_cloexec(cls.worker_sock) - mitogen.core.set_cloexec(cls.child_sock) + mitogen.core.set_cloexec(cls.worker_sock.fileno()) + mitogen.core.set_cloexec(cls.child_sock.fileno()) cls.child_pid = os.fork() ansible_mitogen.logging.setup() @@ -129,7 +131,8 @@ class MuxProcess(object): self._setup_services() # Let the parent know our listening socket is ready. - mitogen.core.io_op(self.child_sock.send, '1') + mitogen.core.io_op(self.child_sock.send, b('1')) + self.child_sock.send(b('1')) # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.child_sock.recv, 1) @@ -178,7 +181,7 @@ class MuxProcess(object): self.pool.stop(join=False) try: os.unlink(self.listener.path) - except OSError, e: + except OSError as e: # Prevent a shutdown race with the parent process. if e.args[0] != errno.ENOENT: raise diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 1ae6a907..5cd1f032 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -36,7 +36,8 @@ how to build arguments for it, preseed related data, etc. """ from __future__ import absolute_import -import cStringIO +from __future__ import unicode_literals + import ctypes import errno import imp @@ -50,6 +51,11 @@ import types import mitogen.core import ansible_mitogen.target # TODO: circular import +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + try: from shlex import quote as shlex_quote except ImportError: @@ -71,6 +77,7 @@ for symbol in 'res_init', '__res_init': except AttributeError: pass +iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -78,7 +85,7 @@ def utf8(s): """ Coerce an object to bytes if it is Unicode. """ - if isinstance(s, unicode): + if isinstance(s, mitogen.core.UnicodeType): s = s.encode('utf-8') return s @@ -110,9 +117,13 @@ class Runner(object): :param mitogen.core.Context service_context: Context to which we should direct FileService calls. For now, always the connection multiplexer process on the controller. - :param dict args: + :param str json_args: Ansible module arguments. A mixture of user and internal keys created by :meth:`ansible.plugins.action.ActionBase._execute_module`. + + This is passed as a string rather than a dict in order to mimic the + implicit bytes/str conversion behaviour of a 2.x controller running + against a 3.x target. :param dict env: Additional environment variables to set during the run. @@ -123,16 +134,13 @@ class Runner(object): When :data:`True`, indicate the runner should detach the context from its parent after setup has completed successfully. """ - def __init__(self, module, service_context, args=None, env=None, + def __init__(self, module, service_context, json_args, env=None, econtext=None, detach=False): - if args is None: - args = {} - - self.module = utf8(module) + self.module = module self.service_context = service_context self.econtext = econtext self.detach = detach - self.args = args + self.args = json.loads(json_args) self.env = env def setup(self): @@ -236,7 +244,7 @@ class ModuleUtilsImporter(object): def load_module(self, fullname): path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) - code = compile(source, path, 'exec') + code = compile(source, path, 'exec', 0, 1) mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = "master:%s" % (path,) mod.__loader__ = self @@ -254,7 +262,7 @@ class TemporaryEnvironment(object): def __init__(self, env=None): self.original = os.environ.copy() self.env = env or {} - os.environ.update((k, str(v)) for k, v in self.env.iteritems()) + os.environ.update((k, str(v)) for k, v in iteritems(self.env)) def revert(self): os.environ.clear() @@ -278,14 +286,11 @@ class NewStyleStdio(object): self.original_stdout = sys.stdout self.original_stderr = sys.stderr self.original_stdin = sys.stdin - sys.stdout = cStringIO.StringIO() - sys.stderr = cStringIO.StringIO() - ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({ - 'ANSIBLE_MODULE_ARGS': args - }) - sys.stdin = cStringIO.StringIO( - ansible.module_utils.basic._ANSIBLE_ARGS - ) + sys.stdout = StringIO() + sys.stderr = StringIO() + encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded) + sys.stdin = StringIO(mitogen.core.to_text(encoded)) def revert(self): sys.stdout = self.original_stdout @@ -309,7 +314,7 @@ class ProgramRunner(Runner): def __init__(self, path, emulate_tty=None, **kwargs): super(ProgramRunner, self).__init__(**kwargs) self.emulate_tty = emulate_tty - self.path = utf8(path) + self.path = path def setup(self): super(ProgramRunner, self).setup() @@ -368,7 +373,7 @@ class ProgramRunner(Runner): args=self._get_program_args(), emulate_tty=self.emulate_tty, ) - except Exception, e: + except Exception as e: LOG.exception('While running %s', self._get_program_args()) return { 'rc': 1, @@ -378,8 +383,8 @@ class ProgramRunner(Runner): return { 'rc': rc, - 'stdout': stdout, - 'stderr': stderr + 'stdout': mitogen.core.to_text(stdout), + 'stderr': mitogen.core.to_text(stderr), } @@ -398,7 +403,7 @@ class ArgsFileRunner(Runner): suffix='-args', dir=ansible_mitogen.target.temp_dir, ) - self.args_fp.write(self._get_args_contents()) + self.args_fp.write(utf8(self._get_args_contents())) self.args_fp.flush() reopen_readonly(self.program_fp) @@ -449,17 +454,17 @@ class ScriptRunner(ProgramRunner): if not self.interpreter: return s - shebang = '#!' + utf8(self.interpreter) + shebang = b'#!' + utf8(self.interpreter) if self.interpreter_arg: - shebang += ' ' + utf8(self.interpreter_arg) + shebang += b' ' + utf8(self.interpreter_arg) new = [shebang] if os.path.basename(self.interpreter).startswith('python'): new.append(self.b_ENCODING_STRING) - _, _, rest = s.partition('\n') + _, _, rest = s.partition(b'\n') new.append(rest) - return '\n'.join(new) + return b'\n'.join(new) class NewStyleRunner(ScriptRunner): @@ -533,15 +538,20 @@ class NewStyleRunner(ScriptRunner): except KeyError: return self._code_by_path.setdefault(self.path, compile( source=self.source, - filename=self.path, + filename="master:" + self.path, mode='exec', dont_inherit=True, )) + if mitogen.core.PY3: + main_module_name = '__main__' + else: + main_module_name = b'__main__' + def _run(self): code = self._get_code() - mod = types.ModuleType('__main__') + mod = types.ModuleType(self.main_module_name) mod.__package__ = None # Some Ansible modules use __file__ to find the Ansiballz temporary # directory. We must provide some temporary path in __file__, but we @@ -551,25 +561,28 @@ class NewStyleRunner(ScriptRunner): ansible_mitogen.target.temp_dir, 'ansible_module_' + os.path.basename(self.path), ) - e = None + exc = None try: - exec code in vars(mod) - except SystemExit, e: - pass + if mitogen.core.PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + except SystemExit as e: + exc = e return { - 'rc': e[0] if e else 2, - 'stdout': sys.stdout.getvalue(), - 'stderr': sys.stderr.getvalue(), + 'rc': exc.args[0] if exc else 2, + 'stdout': mitogen.core.to_text(sys.stdout.getvalue()), + 'stderr': mitogen.core.to_text(sys.stderr.getvalue()), } class JsonArgsRunner(ScriptRunner): - JSON_ARGS = '<>' + JSON_ARGS = b'<>' def _get_args_contents(self): - return json.dumps(self.args) + return json.dumps(self.args).encode() def _rewrite_source(self, s): return ( diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 0331efd2..ad28ce48 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -38,6 +38,8 @@ when a child has completed a job. """ from __future__ import absolute_import +from __future__ import unicode_literals + import logging import os import os.path @@ -53,6 +55,20 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) +if sys.version_info[0] == 3: + def reraise(tp, value, tb): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value +else: + exec( + "def reraise(tp, value, tb=None):\n" + " raise tp, value, tb\n" + ) + + class Error(Exception): pass @@ -118,7 +134,7 @@ class ContextService(mitogen.service.Service): while stack: obj = stack.pop() if isinstance(obj, dict): - stack.extend(sorted(obj.iteritems())) + stack.extend(sorted(obj.items())) elif isinstance(obj, (list, tuple)): stack.extend(obj) else: @@ -351,8 +367,7 @@ class ContextService(mitogen.service.Service): try: result = self._wait_or_start(spec, via=via).get() if isinstance(result, tuple): # exc_info() - e1, e2, e3 = result - raise e1, e2, e3 + reraise(*result) via = result['context'] except mitogen.core.StreamError as e: return { @@ -390,10 +405,10 @@ class ModuleDepService(mitogen.service.Service): @mitogen.service.expose(policy=mitogen.service.AllowParents()) @mitogen.service.arg_spec({ - 'module_name': basestring, - 'module_path': basestring, + 'module_name': mitogen.core.UnicodeType, + 'module_path': mitogen.core.FsPathTypes, 'search_path': tuple, - 'builtin_path': basestring, + 'builtin_path': mitogen.core.FsPathTypes, 'context': mitogen.core.Context, }) def scan(self, module_name, module_path, search_path, builtin_path, context): diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 9bf239d3..85cd67c0 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -32,6 +32,8 @@ for file transfer, module execution and sundry bits like changing file modes. """ from __future__ import absolute_import +from __future__ import unicode_literals + import errno import grp import json @@ -147,7 +149,7 @@ def prune_tree(path): try: os.unlink(path) return - except OSError, e: + except OSError as e: if not (os.path.isdir(path) and e.args[0] in (errno.EPERM, errno.EISDIR)): LOG.error('prune_tree(%r): %s', path, e) @@ -157,7 +159,7 @@ def prune_tree(path): # Ensure write access for readonly directories. Ignore error in case # path is on a weird filesystem (e.g. vfat). os.chmod(path, int('0700', 8)) - except OSError, e: + except OSError as e: LOG.warning('prune_tree(%r): %s', path, e) try: @@ -165,7 +167,7 @@ def prune_tree(path): if name not in ('.', '..'): prune_tree(os.path.join(path, name)) os.rmdir(path) - except OSError, e: + except OSError as e: LOG.error('prune_tree(%r): %s', path, e) @@ -229,7 +231,7 @@ def init_child(econtext): return { 'fork_context': _fork_parent, - 'home_dir': os.path.expanduser('~'), + 'home_dir': mitogen.core.to_text(os.path.expanduser('~')), } @@ -319,12 +321,7 @@ class AsyncRunner(object): 'econtext': self.econtext, 'emulate_tty': False, }) - - dct = run_module(kwargs) - if mitogen.core.PY3: - for key in 'stdout', 'stderr': - dct[key] = dct[key].decode('utf-8', 'surrogateescape') - return dct + return run_module(kwargs) def _parse_result(self, dct): filtered, warnings = ( @@ -465,7 +462,7 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): stdout, stderr = proc.communicate(in_data) if emulate_tty: - stdout = stdout.replace('\n', '\r\n') + stdout = stdout.replace(b'\n', b'\r\n') return proc.returncode, stdout, stderr or '' @@ -481,7 +478,7 @@ def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): :return: (return code, stdout bytes, stderr bytes) """ - assert isinstance(cmd, basestring) + assert isinstance(cmd, mitogen.core.UnicodeType) return exec_args( args=[get_user_shell(), '-c', cmd], in_data=in_data, diff --git a/docs/api.rst b/docs/api.rst index 3df12c0b..febaeb91 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -848,9 +848,9 @@ Context Class try: # Prints output once it is received. msg = recv.get() - print msg.unpickle() + print(msg.unpickle()) except mitogen.core.CallError, e: - print 'Call failed:', str(e) + print('Call failed:', str(e)) Asynchronous calls may be dispatched in parallel to multiple contexts and consumed as they complete using @@ -1038,11 +1038,11 @@ Select Class recvs = [c.call_async(long_running_operation) for c in contexts] for msg in mitogen.select.Select(recvs): - print 'Got %s from %s' % (msg, msg.receiver) + print('Got %s from %s' % (msg, msg.receiver)) total += msg.unpickle() # Iteration ends when last Receiver yields a result. - print 'Received total %s from %s receivers' % (total, len(recvs)) + print('Received total %s from %s receivers' % (total, len(recvs))) :py:class:`Select` may drive a long-running scheduler: @@ -1069,7 +1069,7 @@ Select Class ] for msg in mitogen.select.Select(selects): - print msg.unpickle() + print(msg.unpickle()) .. py:classmethod:: all (it) diff --git a/docs/examples.rst b/docs/examples.rst index 30594601..e3414c80 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -193,7 +193,7 @@ nested.py: context = None for x in range(1, 11): - print 'Connect local%d via %s' % (x, context) + print('Connect local%d via %s' % (x, context)) context = router.local(via=context, name='local%d' % x) context.call(os.system, 'pstree -s python -s mitogen') diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0947e040..c3fac279 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -225,13 +225,13 @@ caller until the return value is available or an exception is raised:: >>> import os >>> # Returns the current time. - >>> print 'Time in remote context:', local.call(time.time) + >>> print('Time in remote context:', local.call(time.time)) >>> try: ... # Raises OSError. ... local.call(os.chdir, '/nonexistent') ... except mitogen.core.CallError, e: - ... print 'Call failed:', str(e) + ... print('Call failed:', str(e)) It is a simple wrapper around the more flexible :meth:`Context.call_async`, which immediately returns a :class:`Receiver ` wired up @@ -242,7 +242,7 @@ is called to block the thread until the result arrives:: >>> call = local.call_async(time.time) >>> msg = call.get() - >>> print msg.unpickle() + >>> print(msg.unpickle()) 1507292737.75547 @@ -259,12 +259,12 @@ We must therefore continue by writing our code as a script:: import mitogen.utils def my_first_function(): - print 'Hello from remote context!' + print('Hello from remote context!') return 123 def main(router): local = router.local() - print local.call(my_first_function) + print(local.call(my_first_function)) if __name__ == '__main__': mitogen.utils.log_to_file(main) @@ -292,7 +292,7 @@ without the need for writing asynchronous code:: calls = [context.call(my_func) for context in contexts] for msg in mitogen.select.Select(calls): - print 'Reply from %s: %s' % (recv.context, data) + print('Reply from %s: %s' % (recv.context, data)) Running Code That May Hang diff --git a/docs/index.rst b/docs/index.rst index 62158736..3e5271bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -135,7 +135,7 @@ configuration. # myapp/__init__.py, myapp/mypkg/__init__.py, and myapp/mypkg/mymodule.py # are transferred automatically. - print context.call(myapp.mymodule.my_function) + print(context.call(myapp.mymodule.my_function)) As the forwarder reuses the import mechanism, it should integrate cleanly with any tool such as `py2exe`_ that correctly implement the protocols in PEP-302, @@ -323,10 +323,10 @@ available. total = 0 for msg in Select(c.call_async(usage, '/tmp') for c in contexts): usage = msg.unpickle() - print 'Context %s /tmp usage: %d' % (recv.context, usage) + print('Context %s /tmp usage: %d' % (recv.context, usage)) total += usage - print 'Total /tmp usage across all contexts: %d' % (total,) + print('Total /tmp usage across all contexts: %d' % (total,)) Single File Programs @@ -361,7 +361,7 @@ usual into the slave process. def main(broker): if len(sys.argv) != 2: - print __doc__ + print(__doc__) sys.exit(1) context = mitogen.ssh.connect(broker, sys.argv[1]) diff --git a/examples/mitop.py b/examples/mitop.py index 1cfd92dd..49eea76e 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -182,7 +182,7 @@ def main(router): """ argv = sys.argv[1:] if not len(argv): - print 'mitop: Need a list of SSH hosts to connect to.' + print('mitop: Need a list of SSH hosts to connect to.') sys.exit(1) delay = 2.0 @@ -193,7 +193,7 @@ def main(router): # connection, a Receiver to accept messages from the host, and finally # start child_main() on the host to pump messages into the receiver. for hostname in argv: - print 'Starting on', hostname + print('Starting on', hostname) host = Host() host.name = hostname diff --git a/examples/service/client.py b/examples/service/client.py index 929016c1..e2d78fc0 100644 --- a/examples/service/client.py +++ b/examples/service/client.py @@ -14,4 +14,4 @@ mitogen.utils.log_to_file() router, parent = mitogen.unix.connect('/tmp/mitosock') with router: - print mitogen.service.call(parent, CONNECT_BY_ID, {}) + print(mitogen.service.call(parent, CONNECT_BY_ID, {})) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 96164d9d..070cf69e 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -92,7 +92,7 @@ def main(log_level='INFO', profiling=False): @mitogen.main() def main(router): z = router.ssh(hostname='k3') - print z.call(get_url, 'https://example.org/') + print(z.call(get_url, 'https://example.org/'))))) """ diff --git a/mitogen/compat/collections.py b/mitogen/compat/collections.py deleted file mode 100644 index 1b1fff15..00000000 --- a/mitogen/compat/collections.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Selected backports from Python stdlib collections module -""" - -__all__ = [ - 'namedtuple', -] - -from operator import itemgetter as _itemgetter -from keyword import iskeyword as _iskeyword -import sys as _sys - -try: - all([]) -except NameError: - def all(iterable): - for element in iterable: - if not element: - return False - return True - -def namedtuple(typename, field_names, verbose=False): - """Returns a new subclass of tuple with named fields. - - >>> Point = namedtuple('Point', 'x y') - >>> Point.__doc__ # docstring for the new class - 'Point(x, y)' - >>> p = Point(11, y=22) # instantiate with positional args or keywords - >>> p[0] + p[1] # indexable like a plain tuple - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessable by name - 33 - >>> d = p._asdict() # convert to a dictionary - >>> d['x'] - 11 - >>> Point(**d) # convert from a dictionary - Point(x=11, y=22) - >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields - Point(x=100, y=22) - - """ - - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. - if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) - for name in (typename,) + field_names: - if not all(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) - if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) - if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() - for name in field_names: - if name.startswith('_'): - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: - raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) - - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - dicttxt = ', '.join('%r: t[%d]' % (name, pos) for pos, name in enumerate(field_names)) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(t): - 'Return a new dict which maps field names to their values' - return {%(dicttxt)s} \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += ' %s = _property(_itemgetter(%d))\n' % (name, i) - if verbose: - print template - - # Execute the template string in a temporary namespace and - # support tracing utilities by setting a value for frame.f_globals['__name__'] - namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, - _property=property, _tuple=tuple) - try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) - result = namespace[typename] - - # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where - # sys._getframe is not defined (Jython for example). - if hasattr(_sys, '_getframe'): - result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') - - return result diff --git a/mitogen/compat/functools.py b/mitogen/compat/functools.py index ef7659f7..6ca75292 100644 --- a/mitogen/compat/functools.py +++ b/mitogen/compat/functools.py @@ -11,10 +11,6 @@ __all__ = [ 'lru_cache', ] -try: - from collections import namedtuple -except ImportError: - from mitogen.compat.collections import namedtuple from threading import RLock @@ -102,8 +98,6 @@ def partial(func, *args, **keywords): ### LRU Cache function decorator ################################################################################ -_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) - class _HashedSeq(list): """ This class guarantees that hash() will be called no more than once per element. This is important because the lru_cache() will hash @@ -170,12 +164,12 @@ def lru_cache(maxsize=128, typed=False): raise TypeError('Expected maxsize to be an integer or None') def decorating_function(user_function): - wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + wrapper = _lru_cache_wrapper(user_function, maxsize, typed) return update_wrapper(wrapper, user_function) return decorating_function -def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): +def _lru_cache_wrapper(user_function, maxsize, typed): # Constants shared by all lru cache instances: sentinel = object() # unique object used to signal cache misses make_key = _make_key # build a key from the function arguments @@ -277,14 +271,6 @@ def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): lock.release() return result - def cache_info(): - """Report cache statistics""" - lock.acquire() - try: - return _CacheInfo(hits, misses, maxsize, cache.__len__()) - finally: - lock.release() - def cache_clear(): """Clear the cache and cache statistics""" lock.acquire() @@ -298,6 +284,5 @@ def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): finally: lock.release() - wrapper.cache_info = cache_info wrapper.cache_clear = cache_clear return wrapper diff --git a/mitogen/core.py b/mitogen/core.py index 1a854d11..db3d4be6 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -33,6 +33,7 @@ bootstrap implementation sent to every new slave context. """ import collections +import encodings.latin_1 import errno import fcntl import imp @@ -54,9 +55,9 @@ import zlib select = __import__('select') try: - import cPickle + import cPickle as pickle except ImportError: - import pickle as cPickle + import pickle try: from cStringIO import StringIO as BytesIO @@ -72,6 +73,8 @@ LOG = logging.getLogger('mitogen') IOLOG = logging.getLogger('mitogen.io') IOLOG.setLevel(logging.INFO) +LATIN1_CODEC = encodings.latin_1.Codec() + _v = False _vv = False @@ -95,14 +98,24 @@ except NameError: PY3 = sys.version_info > (3,) if PY3: - b = lambda s: s.encode('latin-1') + b = str.encode BytesType = bytes - UnicodeType = unicode + UnicodeType = str + FsPathTypes = (str,) + BufferType = lambda buf, start: memoryview(buf)[start:] + long = int else: b = str BytesType = str + FsPathTypes = (str, unicode) + BufferType = buffer UnicodeType = unicode +AnyTextType = (BytesType, UnicodeType) + +if sys.version_info < (2, 5): + next = lambda it: it.next() + CHUNK_SIZE = 131072 _tls = threading.local() @@ -121,6 +134,8 @@ class Error(Exception): def __init__(self, fmt=None, *args): if args: fmt %= args + if fmt and not isinstance(fmt, UnicodeType): + fmt = fmt.decode('utf-8') Exception.__init__(self, fmt) @@ -140,13 +155,31 @@ class Secret(UnicodeType): def __repr__(self): return '[secret]' - def __str__(self): - return UnicodeType(self) + if not PY3: + # TODO: what is this needed for in 2.x? + def __str__(self): + return UnicodeType(self) def __reduce__(self): return (Secret, (UnicodeType(self),)) +class Kwargs(dict): + if PY3: + def __init__(self, dct): + for k, v in dct.items(): + if type(k) is bytes: + self[k.decode()] = v + else: + self[k] = v + + def __repr__(self): + return 'Kwargs(%s)' % (dict.__repr__(self),) + + def __reduce__(self): + return (Kwargs, (dict(self),)) + + class CallError(Error): def __init__(self, fmt=None, *args): if not isinstance(fmt, BaseException): @@ -162,11 +195,11 @@ class CallError(Error): Error.__init__(self, fmt) def __reduce__(self): - return (_unpickle_call_error, (self[0],)) + return (_unpickle_call_error, (self.args[0],)) def _unpickle_call_error(s): - if not (type(s) is str and len(s) < 10000): + if not (type(s) is UnicodeType and len(s) < 10000): raise TypeError('cannot unpickle CallError: bad input') inst = CallError.__new__(CallError) Exception.__init__(inst, s) @@ -186,6 +219,14 @@ class TimeoutError(Error): pass +def to_text(o): + if isinstance(o, UnicodeType): + return UnicodeType(o) + if isinstance(o, BytesType): + return o.decode('utf-8') + return UnicodeType(o) + + def has_parent_authority(msg, _stream=None): return (msg.auth_id == mitogen.context_id or msg.auth_id in mitogen.parent_ids) @@ -247,9 +288,9 @@ def io_op(func, *args): except (select.error, OSError, IOError): e = sys.exc_info()[1] _vv and IOLOG.debug('io_op(%r) -> OSError: %s', func, e) - if e[0] == errno.EINTR: + if e.args[0] == errno.EINTR: continue - if e[0] in (errno.EIO, errno.ECONNRESET, errno.EPIPE): + if e.args[0] in (errno.EIO, errno.ECONNRESET, errno.EPIPE): return None, True raise @@ -326,13 +367,25 @@ def import_module(modname): return __import__(modname, None, None, ['']) +if PY3: + # In 3.x Unpickler is a class exposing find_class as an overridable, but it + # cannot be overridden without subclassing. + class _Unpickler(pickle.Unpickler): + def find_class(self, module, func): + return self.find_global(module, func) +else: + # In 2.x Unpickler is a function exposing a writeable find_global + # attribute. + _Unpickler = pickle.Unpickler + + class Message(object): dst_id = None src_id = None auth_id = None handle = None reply_to = None - data = '' + data = b('') _unpickled = object() router = None @@ -342,7 +395,7 @@ class Message(object): self.src_id = mitogen.context_id self.auth_id = mitogen.context_id vars(self).update(kwargs) - assert isinstance(self.data, str) + assert isinstance(self.data, BytesType) def _unpickle_context(self, context_id, name): return _unpickle_context(self.router, context_id, name) @@ -350,6 +403,10 @@ class Message(object): def _unpickle_sender(self, context_id, dst_handle): return _unpickle_sender(self.router, context_id, dst_handle) + def _unpickle_bytes(self, s, encoding): + s, n = LATIN1_CODEC.encode(s) + return s + def _find_global(self, module, func): """Return the class implementing `module_name.class_name` or raise `StreamError` if the module is not whitelisted.""" @@ -364,6 +421,10 @@ class Message(object): return Blob elif func == 'Secret': return Secret + elif func == 'Kwargs': + return Kwargs + elif module == '_codecs' and func == 'encode': + return self._unpickle_bytes raise StreamError('cannot unpickle %r/%r', module, func) @property @@ -378,10 +439,10 @@ class Message(object): def pickled(cls, obj, **kwargs): self = cls(**kwargs) try: - self.data = cPickle.dumps(obj, protocol=2) - except cPickle.PicklingError: + self.data = pickle.dumps(obj, protocol=2) + except pickle.PicklingError: e = sys.exc_info()[1] - self.data = cPickle.dumps(CallError(e), protocol=2) + self.data = pickle.dumps(CallError(e), protocol=2) return self def reply(self, msg, router=None, **kwargs): @@ -395,6 +456,11 @@ class Message(object): else: LOG.debug('Message.reply(): discarding due to zero handle: %r', msg) + if PY3: + UNPICKLER_KWARGS = {'encoding': 'bytes'} + else: + UNPICKLER_KWARGS = {} + def unpickle(self, throw=True, throw_dead=True): """Deserialize `data` into an object.""" _vv and IOLOG.debug('%r.unpickle()', self) @@ -404,12 +470,8 @@ class Message(object): obj = self._unpickled if obj is Message._unpickled: fp = BytesIO(self.data) - unpickler = cPickle.Unpickler(fp) - try: - unpickler.find_global = self._find_global - except AttributeError: - unpickler.find_class = self._find_global - + unpickler = _Unpickler(fp, **self.UNPICKLER_KWARGS) + unpickler.find_global = self._find_global try: # Must occur off the broker thread. obj = unpickler.load() @@ -513,12 +575,10 @@ class Receiver(object): def __iter__(self): while True: - try: - msg = self.get() - msg.unpickle() # Cause .remote_msg to be thrown. - yield msg - except ChannelError: + msg = self.get() + if msg.is_dead: return + yield msg class Channel(Sender, Receiver): @@ -573,6 +633,8 @@ class Importer(object): # but very unlikely to trigger a bug report. 'org', ] + if PY3: + self.blacklist += ['cStringIO'] # Presence of an entry in this map indicates in-flight GET_MODULE. self._callbacks = {} @@ -619,8 +681,6 @@ class Importer(object): return None _tls.running = True - # TODO: hack: this is papering over a bug elsewhere. - fullname = fullname.rstrip('.') try: pkgname, dot, _ = fullname.rpartition('.') _vv and IOLOG.debug('%r.find_module(%r)', self, fullname) @@ -707,7 +767,9 @@ class Importer(object): else: _v and LOG.debug('_request_module(%r): new request', fullname) self._callbacks[fullname] = [callback] - self._context.send(Message(data=fullname, handle=GET_MODULE)) + self._context.send( + Message(data=b(fullname), handle=GET_MODULE) + ) finally: self._lock.release() @@ -715,9 +777,8 @@ class Importer(object): callback() def load_module(self, fullname): + fullname = to_text(fullname) _v and LOG.debug('Importer.load_module(%r)', fullname) - # TODO: hack: this is papering over a bug elsewhere. - fullname = fullname.rstrip('.') self._refuse_imports(fullname) event = threading.Event() @@ -739,14 +800,12 @@ class Importer(object): else: mod.__package__ = fullname.rpartition('.')[0] or None - # TODO: monster hack: work around modules now being imported as their - # actual name, so when Ansible "apt.py" tries to "import apt", it gets - # itself. Instead force absolute imports during compilation. - flags = 0 - if fullname.startswith('ansible'): - flags = 0x4000 + if mod.__package__ and not PY3: + # 2.x requires __package__ to be exactly a string. + mod.__package__ = mod.__package__.encode() + source = self.get_source(fullname) - code = compile(source, mod.__file__, 'exec', flags, True) + code = compile(source, mod.__file__, 'exec', 0, 1) if PY3: exec(code, vars(mod)) else: @@ -755,11 +814,14 @@ class Importer(object): def get_filename(self, fullname): if fullname in self._cache: - return 'master:' + self._cache[fullname][2] + return u'master:' + self._cache[fullname][2] def get_source(self, fullname): if fullname in self._cache: - return zlib.decompress(self._cache[fullname][3]) + source = zlib.decompress(self._cache[fullname][3]) + if PY3: + return to_text(source) + return source class LogHandler(logging.Handler): @@ -777,7 +839,7 @@ class LogHandler(logging.Handler): try: msg = self.format(rec) encoded = '%s\x00%s\x00%s' % (rec.name, rec.levelno, msg) - if isinstance(encoded, unicode): + if isinstance(encoded, UnicodeType): # Logging package emits both :( encoded = encoded.encode('utf-8') self.context.send(Message(data=encoded, handle=FORWARD_LOG)) @@ -865,7 +927,7 @@ class Stream(BasicStream): def __init__(self, router, remote_id, **kwargs): self._router = router self.remote_id = remote_id - self.name = 'default' + self.name = u'default' self.sent_modules = set(['mitogen', 'mitogen.core']) self.construct(**kwargs) self._input_buf = collections.deque() @@ -935,7 +997,7 @@ class Stream(BasicStream): prev_start = start start = 0 - msg.data = ''.join(bits) + msg.data = b('').join(bits) self._input_buf.appendleft(buf[prev_start+len(bit):]) self._input_buf_len -= total_len self._router._async_route(msg, self) @@ -956,7 +1018,7 @@ class Stream(BasicStream): self.on_disconnect(broker) return elif written != len(buf): - self._output_buf.appendleft(buffer(buf, written)) + self._output_buf.appendleft(BufferType(buf, written)) _vv and IOLOG.debug('%r.on_transmit() -> len %d', self, written) self._output_buf_len -= written @@ -1002,7 +1064,10 @@ class Context(object): self.name = name def __reduce__(self): - return _unpickle_context, (self.context_id, self.name) + name = self.name + if name and not isinstance(name, UnicodeType): + name = UnicodeType(name, 'utf-8') + return _unpickle_context, (self.context_id, name) def on_disconnect(self): _v and LOG.debug('%r.on_disconnect()', self) @@ -1023,9 +1088,11 @@ class Context(object): def call_service_async(self, service_name, method_name, **kwargs): _v and LOG.debug('%r.call_service_async(%r, %r, %r)', self, service_name, method_name, kwargs) - if not isinstance(service_name, basestring): + if isinstance(service_name, BytesType): + service_name = service_name.encode('utf-8') + elif not isinstance(service_name, UnicodeType): service_name = service_name.name() # Service.name() - tup = (service_name, method_name, kwargs) + tup = (service_name, to_text(method_name), Kwargs(kwargs)) msg = Message.pickled(tup, handle=CALL_SERVICE) return self.send_async(msg) @@ -1055,7 +1122,7 @@ def _unpickle_context(router, context_id, name): if not (isinstance(router, Router) and isinstance(context_id, (int, long)) and context_id >= 0 and ( (name is None) or - (isinstance(name, basestring) and len(name) < 100)) + (isinstance(name, UnicodeType) and len(name) < 100)) ): raise TypeError('cannot unpickle Context: bad input') return router.context_class(router, context_id, name) @@ -1193,7 +1260,7 @@ class Latch(object): if i >= self._waking: raise e or TimeoutError() self._waking -= 1 - if rsock.recv(2) != '\x7f': + if rsock.recv(2) != b('\x7f'): raise LatchError('internal error: received >1 wakeups') if e: raise e @@ -1223,10 +1290,10 @@ class Latch(object): def _wake(self, sock): try: - os.write(sock.fileno(), '\x7f') + os.write(sock.fileno(), b('\x7f')) except OSError: e = sys.exc_info()[1] - if e[0] != errno.EBADF: + if e.args[0] != errno.EBADF: raise def __repr__(self): @@ -1319,7 +1386,7 @@ class Waker(BasicStream): self.transmit_side.write(b(' ')) except OSError: e = sys.exc_info()[1] - if e[0] != errno.EBADF: + if e.args[0] != errno.EBADF: raise @@ -1363,7 +1430,7 @@ class IoLogger(BasicStream): if not buf: return self.on_disconnect(broker) - self._buf += buf + self._buf += buf.decode('latin1') self._log_lines() @@ -1420,7 +1487,7 @@ class Router(object): def add_handler(self, fn, handle=None, persist=True, policy=None, respondent=None): - handle = handle or self._last_handle.next() + handle = handle or next(self._last_handle) _vv and IOLOG.debug('%r.add_handler(%r, %r, %r)', self, fn, handle, persist) if respondent: @@ -1761,12 +1828,12 @@ class ExternalContext(object): else: core_src_fd = self.config.get('core_src_fd', 101) if core_src_fd: - fp = os.fdopen(core_src_fd, 'r', 1) + fp = os.fdopen(core_src_fd, 'rb', 1) try: core_size = int(fp.readline()) core_src = fp.read(core_size) # Strip "ExternalContext.main()" call from last line. - core_src = '\n'.join(core_src.splitlines()[:-1]) + core_src = b('\n').join(core_src.splitlines()[:-1]) finally: fp.close() else: diff --git a/mitogen/docker.py b/mitogen/docker.py index 148c132b..38ee9d4e 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -59,7 +59,7 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'docker.' + (self.container or self.image) + self.name = u'docker.' + (self.container or self.image) def get_boot_command(self): args = ['--interactive'] diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index d3950b64..5667bcad 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -299,7 +299,7 @@ def exit(): def die(msg, *args): if args: msg %= args - print msg + sys.stderr.write('%s\n' % (msg,)) exit() @@ -419,12 +419,12 @@ def run(dest, router, args, deadline=None, econtext=None): context_id = router.allocate_id() fakessh = mitogen.parent.Context(router, context_id) - fakessh.name = 'fakessh.%d' % (context_id,) + fakessh.name = u'fakessh.%d' % (context_id,) sock1, sock2 = socket.socketpair() stream = mitogen.core.Stream(router, context_id) - stream.name = 'fakessh' + stream.name = u'fakessh' stream.accept(sock1.fileno(), sock1.fileno()) router.register(fakessh, stream) @@ -445,7 +445,7 @@ def run(dest, router, args, deadline=None, econtext=None): finally: fp.close() - os.chmod(ssh_path, 0755) + os.chmod(ssh_path, int('0755', 8)) env = os.environ.copy() env.update({ 'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')), diff --git a/mitogen/fork.py b/mitogen/fork.py index dd6008db..12bb7dfa 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -45,7 +45,7 @@ def fixup_prngs(): Add 256 bits of /dev/urandom to OpenSSL's PRNG in the child, and re-seed the random package with the same data. """ - s = os.urandom(256 / 8) + s = os.urandom(256 // 8) random.seed(s) if 'ssl' in sys.modules: sys.modules['ssl'].RAND_add(s, 75.0) @@ -112,7 +112,7 @@ class Stream(mitogen.parent.Stream): if isinstance(responder, mitogen.parent.ModuleForwarder): self.importer = responder.importer - name_prefix = 'fork' + name_prefix = u'fork' def start_child(self): parentfp, childfp = mitogen.parent.create_socketpair() diff --git a/mitogen/jail.py b/mitogen/jail.py index ed04da00..726b60e0 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -54,7 +54,7 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'jail.' + self.container + self.name = u'jail.' + self.container def get_boot_command(self): bits = [self.jexec_path] diff --git a/mitogen/lxc.py b/mitogen/lxc.py index eb8ad173..4d6c21db 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -56,7 +56,7 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'lxc.' + self.container + self.name = u'lxc.' + self.container def get_boot_command(self): bits = [ diff --git a/mitogen/master.py b/mitogen/master.py index b024cdd1..25bf8e83 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -56,8 +56,14 @@ import mitogen import mitogen.core import mitogen.minify import mitogen.parent -from mitogen.core import IOLOG + +from mitogen.core import b +from mitogen.core import to_text from mitogen.core import LOG +from mitogen.core import IOLOG + +imap = getattr(itertools, 'imap', map) +izip = getattr(itertools, 'izip', zip) RLOG = logging.getLogger('mitogen.ctx') @@ -79,7 +85,7 @@ def _stdlib_paths(): def get_child_modules(path): it = pkgutil.iter_modules([os.path.dirname(path)]) - return [name for _, name, _ in it] + return [to_text(name) for _, name, _ in it] def get_core_source(): @@ -115,8 +121,12 @@ def scan_code_imports(co): `modname`. """ # Yield `(op, oparg)` tuples from the code object `co`. - ordit = itertools.imap(ord, co.co_code) - nextb = ordit.next + if mitogen.core.PY3: + ordit = iter(co.co_code) + nextb = ordit.__next__ + else: + ordit = imap(ord, co.co_code) + nextb = ordit.next opit = ((c, (None if c < dis.HAVE_ARGUMENT else @@ -131,7 +141,7 @@ def scan_code_imports(co): except StopIteration: return - for oparg1, oparg2, (op3, arg3) in itertools.izip(opit, opit2, opit3): + for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3): if op3 == IMPORT_NAME: op2, arg2 = oparg2 op1, arg1 = oparg1 @@ -227,7 +237,7 @@ class LogForwarder(object): name = '%s.%s' % (RLOG.name, context.name) self._cache[msg.src_id] = logger = logging.getLogger(name) - name, level_s, s = msg.data.split('\x00', 2) + name, level_s, s = msg.data.decode('latin1').split('\x00', 2) logger.log(int(level_s), '%s: %s', name, s, extra={ 'mitogen_message': s, 'mitogen_context': self._router.context_by_id(msg.src_id), @@ -305,9 +315,18 @@ class ModuleFinder(object): def _get_module_via_pkgutil(self, fullname): """Attempt to fetch source code via pkgutil. In an ideal world, this would be the only required implementation of get_module().""" - loader = pkgutil.find_loader(fullname) - IOLOG.debug('pkgutil._get_module_via_pkgutil(%r) -> %r', - fullname, loader) + try: + # Pre-'import spec' this returned None, in Python3.6 it raises + # ImportError. + loader = pkgutil.find_loader(fullname) + except ImportError: + e = sys.exc_info()[1] + LOG.debug('%r._get_module_via_pkgutil(%r): %s', + self, fullname, e) + return None + + IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r', + self, fullname, loader) if not loader: return @@ -318,13 +337,21 @@ class ModuleFinder(object): except AttributeError: return - if path is not None and source is not None: - return path, source, is_pkg + if path is None or source is None: + return + + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + + return path, source, is_pkg def _get_module_via_sys_modules(self, fullname): """Attempt to fetch source code via sys.modules. This is specifically to support __main__, but it may catch a few more cases.""" module = sys.modules.get(fullname) + LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if not isinstance(module, types.ModuleType): LOG.debug('sys.modules[%r] absent or not a regular module', fullname) @@ -344,6 +371,11 @@ class ModuleFinder(object): raise source = '\n' + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + return path, source, is_pkg get_module_methods = [_get_module_via_pkgutil, @@ -483,7 +515,7 @@ class ModuleResponder(object): def __repr__(self): return 'ModuleResponder(%r)' % (self._router,) - MAIN_RE = re.compile(r'^if\s+__name__\s*==\s*.__main__.\s*:', re.M) + MAIN_RE = re.compile(b(r'^if\s+__name__\s*==\s*.__main__.\s*:'), re.M) def whitelist_prefix(self, fullname): if self.whitelist == ['']: @@ -502,6 +534,9 @@ class ModuleResponder(object): return src[:match.start()] return src + def _make_negative_response(self, fullname): + return (fullname, None, None, None, ()) + def _build_tuple(self, fullname): if mitogen.core.is_blacklisted_import(self, fullname): raise ImportError('blacklisted') @@ -512,13 +547,10 @@ class ModuleResponder(object): path, source, is_pkg = self._finder.get_module_source(fullname) if source is None: LOG.error('_build_tuple(%r): could not locate source', fullname) - tup = fullname, None, None, None, () + tup = self._make_negative_response(fullname) self._cache[fullname] = tup return tup - if source is None: - raise ImportError('could not find %r' % (fullname,)) - if is_pkg: pkg_present = get_child_modules(path) LOG.debug('_build_tuple(%r, %r) -> %r', @@ -528,14 +560,20 @@ class ModuleResponder(object): if fullname == '__main__': source = self.neutralize_main(source) - compressed = zlib.compress(source, 9) + compressed = mitogen.core.Blob(zlib.compress(source, 9)) related = [ - name + to_text(name) for name in self._finder.find_related(fullname) if not mitogen.core.is_blacklisted_import(self, name) ] # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - tup = fullname, pkg_present, path, compressed, related + tup = ( + to_text(fullname), + pkg_present, + to_text(path), + compressed, + related + ) self._cache[fullname] = tup return tup @@ -563,6 +601,14 @@ class ModuleResponder(object): def _send_module_and_related(self, stream, fullname): try: tup = self._build_tuple(fullname) + if tup[2] and is_stdlib_path(tup[2]): + # Prevent loading of 2.x<->3.x stdlib modules! This costs one + # RTT per hit, so a client-side solution is also required. + LOG.warning('%r: refusing to serve stdlib module %r', + self, fullname) + self._send_module_load_failed(stream, fullname) + return + for name in tup[4]: # related parent, _, _ = name.partition('.') if parent != fullname and parent not in stream.sent_modules: @@ -581,7 +627,7 @@ class ModuleResponder(object): LOG.debug('%r._on_get_module(%r)', self, msg.data) stream = self._router.stream_by_id(msg.src_id) - fullname = msg.data + fullname = msg.data.decode() if fullname in stream.sent_modules: LOG.warning('_on_get_module(): dup request for %r from %r', fullname, stream) @@ -592,7 +638,7 @@ class ModuleResponder(object): if stream.remote_id != context.context_id: stream.send( mitogen.core.Message( - data='%s\x00%s' % (context.context_id, fullname), + data=b('%s\x00%s' % (context.context_id, fullname)), handle=mitogen.core.FORWARD_MODULE, dst_id=stream.remote_id, ) diff --git a/mitogen/minify.py b/mitogen/minify.py index 1d6f8d11..13b0d364 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -28,11 +28,12 @@ import sys - try: - from cStringIO import StringIO as BytesIO + from cStringIO import StringIO except ImportError: - from io import BytesIO + from io import StringIO + +import mitogen.core if sys.version_info < (2, 7, 11): from mitogen.compat import tokenize @@ -49,7 +50,9 @@ except ImportError: def minimize_source(source): """Remove most comments and docstrings from Python source code. """ - tokens = tokenize.generate_tokens(BytesIO(source).readline) + if not isinstance(source, mitogen.core.UnicodeType): + source = source.decode('utf-8') + tokens = tokenize.generate_tokens(StringIO(source).readline) tokens = strip_comments(tokens) tokens = strip_docstrings(tokens) tokens = reindent(tokens) diff --git a/mitogen/parent.py b/mitogen/parent.py index c999c89a..d1175838 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -32,6 +32,7 @@ sent to any child context that is due to become a parent, due to recursive connection. """ +import codecs import errno import fcntl import getpass @@ -52,11 +53,25 @@ import zlib # Absolute imports for <2.5. select = __import__('select') +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +try: + from functools import lru_cache +except ImportError: + from mitogen.compat.functools import lru_cache + import mitogen.core +from mitogen.core import b from mitogen.core import LOG from mitogen.core import IOLOG +if mitogen.core.PY3: + xrange = range + try: SC_OPEN_MAX = os.sysconf('SC_OPEN_MAX') except: @@ -281,9 +296,13 @@ def write_all(fd, s, deadline=None): if timeout == 0: raise mitogen.core.TimeoutError('write timed out') + if mitogen.core.PY3: + window = memoryview(s)[written:] + else: + window = buffer(s, written) + for fd in poller.poll(timeout): - n, disconnected = mitogen.core.io_op( - os.write, fd, buffer(s, written)) + n, disconnected = mitogen.core.io_op(os.write, fd, window) if disconnected: raise mitogen.core.StreamError('EOF on stream during write') @@ -321,7 +340,7 @@ def iter_read(fds, deadline=None): if not poller.readers: raise mitogen.core.StreamError( 'EOF on stream; last 300 bytes received: %r' % - (''.join(bits)[-300:],) + (b('').join(bits)[-300:],) ) raise mitogen.core.TimeoutError('read timed out') @@ -380,14 +399,18 @@ def upgrade_router(econtext): def make_call_msg(fn, *args, **kwargs): if isinstance(fn, types.MethodType) and \ isinstance(fn.im_self, (type, types.ClassType)): - klass = fn.im_self.__name__ + klass = mitogen.core.to_text(fn.im_self.__name__) else: klass = None - return mitogen.core.Message.pickled( - (fn.__module__, klass, fn.__name__, args, kwargs), - handle=mitogen.core.CALL_FUNCTION, + tup = ( + mitogen.core.to_text(fn.__module__), + klass, + mitogen.core.to_text(fn.__name__), + args, + mitogen.core.Kwargs(kwargs) ) + return mitogen.core.Message.pickled(tup, handle=mitogen.core.CALL_FUNCTION) def stream_by_method_name(name): @@ -395,9 +418,9 @@ def stream_by_method_name(name): Given the name of a Mitogen connection method, import its implementation module and return its Stream subclass. """ - if name == 'local': - name = 'parent' - module = mitogen.core.import_module('mitogen.' + name) + if name == u'local': + name = u'parent' + module = mitogen.core.import_module(u'mitogen.' + name) return module.Stream @@ -412,18 +435,18 @@ def _proxy_connect(name, method_name, kwargs, econtext): ) except mitogen.core.StreamError: return { - 'id': None, - 'name': None, - 'msg': 'error occurred on host %s: %s' % ( + u'id': None, + u'name': None, + u'msg': 'error occurred on host %s: %s' % ( socket.gethostname(), sys.exc_info()[1], ), } return { - 'id': context.context_id, - 'name': context.name, - 'msg': None, + u'id': context.context_id, + u'name': context.name, + u'msg': None, } @@ -790,11 +813,15 @@ class Stream(mitogen.core.Stream): sys.executable += sys.version[:3] os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') - os.write(1,'MITO000\n') + os.write(1,'MITO000\n'.encode()) C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') - os.fdopen(W,'w',0).write(C) - os.fdopen(w,'w',0).write('PREAMBLE_LEN\n'+C) - os.write(1,'MITO001\n') + wfp=os.fdopen(W,'wb',0) + wfp.write(C) + Wfp=os.fdopen(w,'wb',0) + Wfp.write('PREAMBLE_LEN\n'.encode()+C) + wfp.close() + Wfp.close() + os.write(1,'MITO001\n'.encode()) def get_boot_command(self): source = inspect.getsource(self._first_stage) @@ -806,7 +833,8 @@ class Stream(mitogen.core.Stream): str(len(preamble_compressed))) source = source.replace('PREAMBLE_LEN', str(len(zlib.decompress(preamble_compressed)))) - encoded = zlib.compress(source, 9).encode('base64').replace('\n', '') + compressed = zlib.compress(source.encode(), 9) + encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) # We can't use bytes.decode() in 3.x since it was restricted to always # return unicode, so codecs.decode() is used instead. In 3.x # codecs.decode() requires a bytes object. Since we must be compatible @@ -815,7 +843,7 @@ class Stream(mitogen.core.Stream): return [ self.python_path, '-c', 'import codecs,os,sys;_=codecs.decode;' - 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded,) + 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) ] def get_econtext_config(self): @@ -840,11 +868,11 @@ class Stream(mitogen.core.Stream): source += '\nExternalContext(%r).main()\n' % ( self.get_econtext_config(), ) - return zlib.compress(source, 9) + return zlib.compress(source.encode('utf-8'), 9) create_child = staticmethod(create_child) create_child_args = {} - name_prefix = 'local' + name_prefix = u'local' def start_child(self): args = self.get_boot_command() @@ -858,7 +886,7 @@ class Stream(mitogen.core.Stream): def connect(self): LOG.debug('%r.connect()', self) self.pid, fd, extra_fd = self.start_child() - self.name = '%s.%s' % (self.name_prefix, self.pid) + self.name = u'%s.%s' % (self.name_prefix, self.pid) self.receive_side = mitogen.core.Side(self, fd) self.transmit_side = mitogen.core.Side(self, os.dup(fd)) LOG.debug('%r.connect(): child process stdin/stdout=%r', @@ -871,16 +899,18 @@ class Stream(mitogen.core.Stream): raise #: For ssh.py, this must be at least max(len('password'), len('debug1:')) - EC0_MARKER = 'MITO000\n' - EC1_MARKER = 'MITO001\n' + EC0_MARKER = mitogen.core.b('MITO000\n') + EC1_MARKER = mitogen.core.b('MITO001\n') def _ec0_received(self): LOG.debug('%r._ec0_received()', self) write_all(self.transmit_side.fd, self.get_preamble()) - discard_until(self.receive_side.fd, 'MITO001\n', self.connect_deadline) + discard_until(self.receive_side.fd, self.EC1_MARKER, + self.connect_deadline) def _connect_bootstrap(self, extra_fd): - discard_until(self.receive_side.fd, 'MITO000\n', self.connect_deadline) + discard_until(self.receive_side.fd, self.EC0_MARKER, + self.connect_deadline) self._ec0_received() @@ -976,7 +1006,7 @@ class RouteMonitor(object): self.parent.send( mitogen.core.Message( handle=handle, - data=data, + data=data.encode('utf-8'), ) ) @@ -986,7 +1016,8 @@ class RouteMonitor(object): stream, we're also responsible for broadcasting DEL_ROUTE upstream if/when that child disconnects. """ - self.propagate(mitogen.core.ADD_ROUTE, stream.remote_id, stream.name) + self.propagate(mitogen.core.ADD_ROUTE, stream.remote_id, + stream.name) mitogen.core.listen( obj=stream, name='disconnect', @@ -1011,7 +1042,8 @@ class RouteMonitor(object): if msg.is_dead: return - target_id_s, _, target_name = msg.data.partition(':') + target_id_s, _, target_name = msg.data.partition(b(':')) + target_name = target_name.decode() target_id = int(target_id_s) self.router.context_by_id(target_id).name = target_name stream = self.router.stream_by_id(msg.auth_id) @@ -1124,7 +1156,7 @@ class Router(mitogen.core.Router): self._context_by_id[context_id] = context return context - connection_timeout_msg = "Connection timed out." + connection_timeout_msg = u"Connection timed out." def _connect(self, klass, name=None, **kwargs): context_id = self.allocate_id() @@ -1145,11 +1177,11 @@ class Router(mitogen.core.Router): def connect(self, method_name, name=None, **kwargs): klass = stream_by_method_name(method_name) - kwargs.setdefault('debug', self.debug) - kwargs.setdefault('profiling', self.profiling) - kwargs.setdefault('unidirectional', self.unidirectional) + kwargs.setdefault(u'debug', self.debug) + kwargs.setdefault(u'profiling', self.profiling) + kwargs.setdefault(u'unidirectional', self.unidirectional) - via = kwargs.pop('via', None) + via = kwargs.pop(u'via', None) if via is not None: return self.proxy_connect(via, method_name, name=name, **kwargs) return self._connect(klass, name=name, **kwargs) @@ -1158,43 +1190,43 @@ class Router(mitogen.core.Router): resp = via_context.call(_proxy_connect, name=name, method_name=method_name, - kwargs=kwargs + kwargs=mitogen.core.Kwargs(kwargs), ) if resp['msg'] is not None: raise mitogen.core.StreamError(resp['msg']) - name = '%s.%s' % (via_context.name, resp['name']) + name = u'%s.%s' % (via_context.name, resp['name']) context = self.context_class(self, resp['id'], name=name) context.via = via_context self._context_by_id[context.context_id] = context return context def docker(self, **kwargs): - return self.connect('docker', **kwargs) + return self.connect(u'docker', **kwargs) def fork(self, **kwargs): - return self.connect('fork', **kwargs) + return self.connect(u'fork', **kwargs) def jail(self, **kwargs): - return self.connect('jail', **kwargs) + return self.connect(u'jail', **kwargs) def local(self, **kwargs): - return self.connect('local', **kwargs) + return self.connect(u'local', **kwargs) def lxc(self, **kwargs): - return self.connect('lxc', **kwargs) + return self.connect(u'lxc', **kwargs) def setns(self, **kwargs): - return self.connect('setns', **kwargs) + return self.connect(u'setns', **kwargs) def su(self, **kwargs): - return self.connect('su', **kwargs) + return self.connect(u'su', **kwargs) def sudo(self, **kwargs): - return self.connect('sudo', **kwargs) + return self.connect(u'sudo', **kwargs) def ssh(self, **kwargs): - return self.connect('ssh', **kwargs) + return self.connect(u'ssh', **kwargs) class ProcessMonitor(object): @@ -1251,7 +1283,8 @@ class ModuleForwarder(object): if msg.is_dead: return - context_id_s, _, fullname = msg.data.partition('\x00') + context_id_s, _, fullname = msg.data.partition(b('\x00')) + fullname = mitogen.core.to_text(fullname) context_id = int(context_id_s) stream = self.router.stream_by_id(context_id) if stream.remote_id == mitogen.parent_id: @@ -1279,9 +1312,17 @@ class ModuleForwarder(object): if msg.is_dead: return - fullname = msg.data - self.importer._request_module(fullname, - lambda: self._on_cache_callback(msg, fullname) + fullname = msg.data.decode('utf-8') + callback = lambda: self._on_cache_callback(msg, fullname) + self.importer._request_module(fullname, callback) + + def _send_one_module(self, msg, tup): + self.router._async_route( + mitogen.core.Message.pickled( + tup, + dst_id=msg.src_id, + handle=mitogen.core.LOAD_MODULE, + ) ) def _on_cache_callback(self, msg, fullname): diff --git a/mitogen/service.py b/mitogen/service.py index ffb45649..9d87839f 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -38,6 +38,7 @@ import time import mitogen.core import mitogen.select +from mitogen.core import b from mitogen.core import LOG @@ -48,6 +49,14 @@ _pool_pid = None _pool_lock = threading.Lock() +if mitogen.core.PY3: + def func_code(func): + return func.__code__ +else: + def func_code(func): + return func.func_code + + @mitogen.core.takes_router def get_or_create_pool(size=None, router=None): global _pool @@ -221,7 +230,7 @@ class Invoker(object): def _invoke(self, method_name, kwargs, msg): method = getattr(self.service, method_name) - if 'msg' in method.func_code.co_varnames: + if 'msg' in func_code(method).co_varnames: kwargs['msg'] = msg # TODO: hack no_reply = getattr(method, 'mitogen_service__no_reply', False) @@ -371,7 +380,7 @@ class Service(object): @classmethod def name(cls): - return '%s.%s' % (cls.__module__, cls.__name__) + return u'%s.%s' % (cls.__module__, cls.__name__) def __init__(self, router): self.router = router @@ -468,7 +477,7 @@ class Pool(object): def join(self): for th in self._threads: th.join() - for invoker in self._invoker_by_name.itervalues(): + for invoker in self._invoker_by_name.values(): invoker.service.on_shutdown() def get_invoker(self, name, msg): @@ -490,17 +499,21 @@ class Pool(object): def _validate(self, msg): tup = msg.unpickle(throw=False) + LOG.debug('_validate(): %r', tup) + LOG.debug('_validate(): %r', mitogen.core.PY3) + LOG.debug('_validate(): %r', list(map(type, tup))) + LOG.debug('_validate(): UnicodeType=%r', mitogen.core.UnicodeType) if not (isinstance(tup, tuple) and len(tup) == 3 and - isinstance(tup[0], basestring) and - isinstance(tup[1], basestring) and + isinstance(tup[0], mitogen.core.AnyTextType) and + isinstance(tup[1], mitogen.core.AnyTextType) and isinstance(tup[2], dict)): raise mitogen.core.CallError('Invalid message format.') def _on_service_call(self, recv, msg): - self._validate(msg) - service_name, method_name, kwargs = msg.unpickle() try: + self._validate(msg) + service_name, method_name, kwargs = msg.unpickle() invoker = self.get_invoker(service_name, msg) return invoker.invoke(method_name, kwargs, msg) except mitogen.core.CallError: @@ -628,7 +641,7 @@ class PushFileService(Service): @expose(policy=AllowParents()) @arg_spec({ 'context': mitogen.core.Context, - 'path': basestring, + 'path': mitogen.core.FsPathTypes, }) def propagate_to(self, context, path): LOG.debug('%r.propagate_to(%r, %r)', self, context, path) @@ -651,7 +664,7 @@ class PushFileService(Service): @expose(policy=AllowParents()) @no_reply() @arg_spec({ - 'path': basestring, + 'path': mitogen.core.FsPathTypes, 'data': mitogen.core.Blob, 'context': mitogen.core.Context, }) @@ -667,7 +680,7 @@ class PushFileService(Service): @expose(policy=AllowParents()) @no_reply() @arg_spec({ - 'path': basestring, + 'path': mitogen.core.FsPathTypes, 'context': mitogen.core.Context, }) def forward(self, path, context): @@ -752,7 +765,7 @@ class FileService(Service): @expose(policy=AllowParents()) @arg_spec({ - 'path': basestring, + 'path': mitogen.core.FsPathTypes, }) def register(self, path): """ @@ -801,7 +814,7 @@ class FileService(Service): IO_SIZE = mitogen.core.CHUNK_SIZE - (mitogen.core.Stream.HEADER_LEN + ( len( mitogen.core.Message.pickled( - mitogen.core.Blob(' ' * mitogen.core.CHUNK_SIZE) + mitogen.core.Blob(b(' ') * mitogen.core.CHUNK_SIZE) ).data ) - mitogen.core.CHUNK_SIZE )) @@ -831,7 +844,7 @@ class FileService(Service): @expose(policy=AllowAny()) @no_reply() @arg_spec({ - 'path': basestring, + 'path': mitogen.core.FsPathTypes, 'sender': mitogen.core.Sender, }) def fetch(self, path, sender, msg): diff --git a/mitogen/setns.py b/mitogen/setns.py index c95b3217..b07b4df7 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -214,4 +214,4 @@ class Stream(mitogen.parent.Stream): LOG.debug('Leader PID for %s container %r: %d', self.kind, self.container, self.leader_pid) super(Stream, self).connect() - self.name = 'setns.' + self.container + self.name = u'setns.' + self.container diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 161b5323..36efbffc 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -39,18 +39,19 @@ except ImportError: from pipes import quote as shlex_quote import mitogen.parent +from mitogen.core import b LOG = logging.getLogger('mitogen') # sshpass uses 'assword' because it doesn't lowercase the input. -PASSWORD_PROMPT = 'password' -PERMDENIED_PROMPT = 'permission denied' -HOSTKEY_REQ_PROMPT = 'are you sure you want to continue connecting (yes/no)?' -HOSTKEY_FAIL = 'host key verification failed.' +PASSWORD_PROMPT = b('password') +PERMDENIED_PROMPT = b('permission denied') +HOSTKEY_REQ_PROMPT = b('are you sure you want to continue connecting (yes/no)?') +HOSTKEY_FAIL = b('host key verification failed.') -DEBUG_PREFIXES = ('debug1:', 'debug2:', 'debug3:') +DEBUG_PREFIXES = (b('debug1:'), b('debug2:'), b('debug3:')) def filter_debug(stream, it): @@ -62,7 +63,7 @@ def filter_debug(stream, it): lines such as the password prompt. """ state = 'start_of_line' - buf = '' + buf = b('') for chunk in it: buf += chunk while buf: @@ -78,13 +79,13 @@ def filter_debug(stream, it): else: state = 'in_plain' elif state == 'in_debug': - if '\n' not in buf: + if b('\n') not in buf: break - line, _, buf = buf.partition('\n') + line, _, buf = buf.partition(b('\n')) LOG.debug('%r: %s', stream, line.rstrip()) state = 'start_of_line' elif state == 'in_plain': - line, nl, buf = buf.partition('\n') + line, nl, buf = buf.partition(b('\n')) yield line + nl if nl: state = 'start_of_line' @@ -189,9 +190,9 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'ssh.' + self.hostname + self.name = u'ssh.' + mitogen.core.to_text(self.hostname) if self.port: - self.name += ':%s' % (self.port,) + self.name += u':%s' % (self.port,) auth_incorrect_msg = 'SSH authentication is incorrect' password_incorrect_msg = 'SSH password is incorrect' @@ -249,7 +250,7 @@ class Stream(mitogen.parent.Stream): if self.password is None: raise PasswordError(self.password_required_msg) LOG.debug('%r: sending password', self) - self.tty_stream.transmit_side.write(self.password + '\n') + self.tty_stream.transmit_side.write((self.password + '\n').encode()) password_sent = True raise mitogen.core.StreamError('bootstrap failed') diff --git a/mitogen/su.py b/mitogen/su.py index dd340554..45229d6d 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -31,6 +31,7 @@ import os import mitogen.core import mitogen.parent +from mitogen.core import b LOG = logging.getLogger(__name__) @@ -54,11 +55,11 @@ class Stream(mitogen.parent.Stream): username = 'root' password = None su_path = 'su' - password_prompt = 'password:' + password_prompt = b('password:') incorrect_prompts = ( - 'su: sorry', # BSD - 'su: authentication failure', # Linux - 'su: incorrect password', # CentOS 6 + b('su: sorry'), # BSD + b('su: authentication failure'), # Linux + b('su: incorrect password'), # CentOS 6 ) def construct(self, username=None, password=None, su_path=None, @@ -77,7 +78,7 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'su.' + self.username + self.name = u'su.' + mitogen.core.to_text(self.username) def on_disconnect(self, broker): super(Stream, self).on_disconnect(broker) @@ -110,6 +111,8 @@ class Stream(mitogen.parent.Stream): if password_sent: raise PasswordError(self.password_incorrect_msg) LOG.debug('sending password') - self.transmit_side.write(self.password + '\n') + self.transmit_side.write( + mitogen.core.to_text(self.password + '\n').encode('utf-8') + ) password_sent = True raise mitogen.core.StreamError('bootstrap failed') diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 32287f9f..1ec5c2f6 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -33,10 +33,11 @@ import time import mitogen.core import mitogen.parent +from mitogen.core import b LOG = logging.getLogger(__name__) -PASSWORD_PROMPT = 'password' +PASSWORD_PROMPT = b('password') SUDO_OPTIONS = [ #(False, 'bool', '--askpass', '-A') #(False, 'str', '--auth-type', '-a') @@ -135,7 +136,7 @@ class Stream(mitogen.parent.Stream): def connect(self): super(Stream, self).connect() - self.name = 'sudo.' + self.username + self.name = u'sudo.' + mitogen.core.to_text(self.username) def on_disconnect(self, broker): self.tty_stream.on_disconnect(broker) @@ -176,7 +177,8 @@ class Stream(mitogen.parent.Stream): raise PasswordError(self.password_required_msg) if password_sent: raise PasswordError(self.password_incorrect_msg) - LOG.debug('sending password') - self.tty_stream.transmit_side.write(self.password + '\n') + self.tty_stream.transmit_side.write( + mitogen.core.to_text(self.password + '\n').encode('utf-8') + ) password_sent = True raise mitogen.core.StreamError('bootstrap failed') diff --git a/mitogen/unix.py b/mitogen/unix.py index 8eda7692..efcc59cc 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -73,7 +73,7 @@ class Listener(mitogen.core.BasicStream): os.unlink(self.path) self._sock.bind(self.path) - os.chmod(self.path, 0600) + os.chmod(self.path, int('0600', 8)) self._sock.listen(backlog) self.receive_side = mitogen.core.Side(self, self._sock.fileno()) router.broker.start_receive(self) @@ -87,7 +87,7 @@ class Listener(mitogen.core.BasicStream): context = mitogen.parent.Context(self._router, context_id) stream = mitogen.core.Stream(self._router, context_id) stream.accept(sock.fileno(), sock.fileno()) - stream.name = 'unix_client.%d' % (pid,) + stream.name = u'unix_client.%d' % (pid,) stream.auth_id = mitogen.context_id stream.is_privileged = True self._router.register(context, stream) @@ -111,7 +111,7 @@ def connect(path, broker=None): router = mitogen.master.Router(broker=broker) stream = mitogen.core.Stream(router, remote_id) stream.accept(sock.fileno(), sock.fileno()) - stream.name = 'unix_listener.%d' % (pid,) + stream.name = u'unix_listener.%d' % (pid,) context = mitogen.parent.Context(router, remote_id) router.register(context, stream) diff --git a/mitogen/utils.py b/mitogen/utils.py index d02c3f9c..e326bd6c 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -39,6 +39,11 @@ import mitogen.master LOG = logging.getLogger('mitogen') iteritems = getattr(dict, 'iteritems', dict.items) +if mitogen.core.PY3: + iteritems = dict.items +else: + iteritems = dict.iteritems + def disable_site_packages(): for entry in sys.path[:]: @@ -118,9 +123,9 @@ def cast(obj): return [cast(v) for v in obj] if isinstance(obj, PASSTHROUGH): return obj - if isinstance(obj, unicode): - return unicode(obj) - if isinstance(obj, str): - return str(obj) + if isinstance(obj, mitogen.core.UnicodeType): + return mitogen.core.UnicodeType(obj) + if isinstance(obj, mitogen.core.BytesType): + return mitogen.core.BytesType(obj) raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj)) diff --git a/preamble_size.py b/preamble_size.py index bca55ab1..66d5ccf3 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -19,11 +19,11 @@ router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) stream = mitogen.ssh.Stream(router, 0, max_message_size=0, hostname='foo') -print 'SSH command size: %s' % (len(' '.join(stream.get_boot_command())),) -print 'Preamble size: %s (%.2fKiB)' % ( +print('SSH command size: %s' % (len(' '.join(stream.get_boot_command())),)) +print('Preamble size: %s (%.2fKiB)' % ( len(stream.get_preamble()), len(stream.get_preamble()) / 1024.0, -) +)) print( ' ' diff --git a/setup.py b/setup.py index 00dd1cd6..fdda624a 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,6 @@ setup( license = 'New BSD', url = 'https://github.com/dw/mitogen/', packages = find_packages(exclude=['tests', 'examples']), - use_2to3=True, zip_safe = False, classifiers = [ 'Development Status :: 3 - Alpha', diff --git a/tests/ansible/compare_output_test.py b/tests/ansible/compare_output_test.py index 1ad2f01f..5a091f15 100755 --- a/tests/ansible/compare_output_test.py +++ b/tests/ansible/compare_output_test.py @@ -55,8 +55,8 @@ for suffix in suffixes: tofile='mitogen-output.txt', )) if diff: - print '++ differ! suffix: %r' % (suffix,) + print('++ differ! suffix: %r' % (suffix,)) for line in diff: - print line + print(line) print print diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml index 2a5a7b56..c6845cff 100644 --- a/tests/ansible/integration/action/transfer_data.yml +++ b/tests/ansible/integration/action/transfer_data.yml @@ -23,7 +23,7 @@ - assert: that: | - out.content.decode('base64') == '{"I am JSON": true}' + out.content|b64decode == '{"I am JSON": true}' # Ensure it handles strings. @@ -41,7 +41,7 @@ - assert: that: - out.content.decode('base64') == 'I am text.' + out.content|b64decode == 'I am text.' - file: path: /tmp/transfer-data diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index 1bf5e9a7..f0e9439b 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -4,9 +4,9 @@ import json from ansible.module_utils.basic import path def main(): - print json.dumps({ + print(json.dumps({ 'path': path() - }) + })) if __name__ == '__main__': main() diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index 7e722253..d86bff4a 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -21,4 +21,4 @@ - assert: that: - - out.argv_types == [""] + - out.argv_types_correct diff --git a/tests/ansible/lib/callback/nice_stdout.py b/tests/ansible/lib/callback/nice_stdout.py index 08a3757b..fa720fd2 100644 --- a/tests/ansible/lib/callback/nice_stdout.py +++ b/tests/ansible/lib/callback/nice_stdout.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import io +from ansible.module_utils import six + try: from ansible.plugins import callback_loader except ImportError: @@ -25,15 +27,17 @@ def printi(tio, obj, key=None, indent=0): write(']') elif isinstance(obj, dict): write('{') - for key2, obj2 in sorted(obj.iteritems()): + for key2, obj2 in sorted(six.iteritems(obj)): if not (key2.startswith('_ansible_') or key2.endswith('_lines')): printi(tio, obj2, key=key2, indent=indent+1) key = None write('}') - elif isinstance(obj, basestring): - if isinstance(obj, str): - obj = obj.decode('utf-8', 'replace') + elif isinstance(obj, six.text_type): + for line in obj.splitlines(): + write('%s', line.rstrip('\r\n')) + elif isinstance(obj, six.binary_type): + obj = obj.decode('utf-8', 'replace') for line in obj.splitlines(): write('%s', line.rstrip('\r\n')) else: @@ -47,7 +51,7 @@ class CallbackModule(DefaultModule): try: tio = io.StringIO() printi(tio, result) - return tio.getvalue().encode('ascii', 'replace') + return tio.getvalue() #.encode('ascii', 'replace') except: import traceback traceback.print_exc() diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index d4795422..8fe50bbc 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -3,6 +3,7 @@ # interpreter I run within. from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils import six import os import pwd @@ -16,7 +17,7 @@ def main(): python_version=sys.version[:3], argv=sys.argv, __file__=__file__, - argv_types=[str(type(s)) for s in sys.argv], + argv_types_correct=all(type(s) is str for s in sys.argv), env=dict(os.environ), cwd=os.getcwd(), python_path=sys.path, diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 45689584..008829f6 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -6,8 +6,8 @@ import sys json_arguments = """<>""" -print "{" -print " \"changed\": false," -print " \"msg\": \"Here is my input\"," -print " \"input\": [%s]" % (json_arguments,) -print "}" +print("{") +print(" \"changed\": false,") +print(" \"msg\": \"Here is my input\",") +print(" \"input\": [%s]" % (json_arguments,)) +print("}") diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 26022ff3..6b3f9ef7 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -13,8 +13,8 @@ def usage(): input_json = sys.stdin.read() -print "{" -print " \"changed\": false," -print " \"msg\": \"Here is my input\"," -print " \"input\": [%s]" % (input_json,) -print "}" +print("{") +print(" \"changed\": false,") +print(" \"msg\": \"Here is my input\",") +print(" \"input\": [%s]" % (input_json,)) +print("}") diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 69b4dae0..9641e88d 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -14,13 +14,13 @@ def usage(): input_json = sys.stdin.read() -print "{" -print " \"changed\": false," +print("{") +print(" \"changed\": false,") # v2.5.1. apt.py started depending on this. # https://github.com/dw/mitogen/issues/210 -print " \"__file__\": \"%s\"," % (__file__,) +print(" \"__file__\": \"%s\"," % (__file__,)) # Python sets this during a regular import. -print " \"__package__\": \"%s\"," % (__package__,) -print " \"msg\": \"Here is my input\"," -print " \"input\": [%s]" % (input_json,) -print "}" +print(" \"__package__\": \"%s\"," % (__package__,)) +print(" \"msg\": \"Here is my input\",") +print(" \"input\": [%s]" % (input_json,)) +print("}") diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index bd12704e..439aa522 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -25,9 +25,9 @@ try: except IOError: usage() -print "{" -print " \"changed\": false," -print " \"msg\": \"Here is my input\"," -print " \"source\": [%s]," % (json.dumps(me),) -print " \"input\": [%s]" % (input_json,) -print "}" +print("{") +print(" \"changed\": false,") +print(" \"msg\": \"Here is my input\",") +print(" \"source\": [%s]," % (json.dumps(me),)) +print(" \"input\": [%s]" % (input_json,)) +print("}") diff --git a/tests/bench/local.py b/tests/bench/local.py index 25a67b00..31f3c952 100644 --- a/tests/bench/local.py +++ b/tests/bench/local.py @@ -10,13 +10,13 @@ import time @mitogen.main() #(log_level='DEBUG') def main(router): - for x in xrange(1000): + for x in range(1000): t = time.time() f = router.local()# debug=True) tt = time.time() - print x, 1000 * (tt - t) + print(x, 1000 * (tt - t)) - print f - print 'EEK', f.call(socket.gethostname) - print 'MY PID', os.getpid() - print 'EEKERY', f.call(os.getpid) + print(f) + print('EEK', f.call(socket.gethostname)) + print('MY PID', os.getpid()) + print('EEKERY', f.call(os.getpid)) diff --git a/tests/bench/megatime.py b/tests/bench/megatime.py index 0964424d..6f5f3b5d 100755 --- a/tests/bench/megatime.py +++ b/tests/bench/megatime.py @@ -6,12 +6,12 @@ import time times = [] -for x in xrange(5): +for x in range(5): t0 = time.time() os.spawnvp(os.P_WAIT, sys.argv[1], sys.argv[1:]) t = time.time() - t0 times.append(t) - print '+++', t + print('+++', t) -print 'all:', times -print 'min %s max %s diff %s' % (min(times), max(times), (max(times) - min(times))) +print('all:', times) +print('min %s max %s diff %s' % (min(times), max(times), (max(times) - min(times)))) diff --git a/tests/call_error_test.py b/tests/call_error_test.py index ab9a2660..15de01f8 100644 --- a/tests/call_error_test.py +++ b/tests/call_error_test.py @@ -1,26 +1,31 @@ import os import pickle +import sys import unittest2 import mitogen.core +class MyError(Exception): + pass + + class ConstructorTest(unittest2.TestCase): klass = mitogen.core.CallError def test_string_noargs(self): e = self.klass('%s%s') - self.assertEquals(e[0], '%s%s') + self.assertEquals(e.args[0], '%s%s') def test_string_args(self): e = self.klass('%s%s', 1, 1) - self.assertEquals(e[0], '11') + self.assertEquals(e.args[0], '11') def test_from_exc(self): - ve = ValueError('eek') + ve = MyError('eek') e = self.klass(ve) - self.assertEquals(e[0], 'exceptions.ValueError: eek') + self.assertEquals(e.args[0], '__main__.MyError: eek') def test_form_base_exc(self): ve = SystemExit('eek') @@ -29,12 +34,13 @@ class ConstructorTest(unittest2.TestCase): def test_from_exc_tb(self): try: - raise ValueError('eek') - except ValueError, ve: + raise MyError('eek') + except MyError: + ve = sys.exc_info()[1] e = self.klass(ve) - self.assertTrue(e[0].startswith('exceptions.ValueError: eek')) - self.assertTrue('test_from_exc_tb' in e[0]) + self.assertTrue(e.args[0].startswith('__main__.MyError: eek')) + self.assertTrue('test_from_exc_tb' in e.args[0]) class PickleTest(unittest2.TestCase): @@ -43,28 +49,29 @@ class PickleTest(unittest2.TestCase): def test_string_noargs(self): e = self.klass('%s%s') e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2[0], '%s%s') + self.assertEquals(e2.args[0], '%s%s') def test_string_args(self): e = self.klass('%s%s', 1, 1) e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2[0], '11') + self.assertEquals(e2.args[0], '11') def test_from_exc(self): - ve = ValueError('eek') + ve = MyError('eek') e = self.klass(ve) e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2[0], 'exceptions.ValueError: eek') + self.assertEquals(e2.args[0], '__main__.MyError: eek') def test_from_exc_tb(self): try: - raise ValueError('eek') - except ValueError, ve: + raise MyError('eek') + except MyError: + ve = sys.exc_info()[1] e = self.klass(ve) e2 = pickle.loads(pickle.dumps(e)) - self.assertTrue(e2[0].startswith('exceptions.ValueError: eek')) - self.assertTrue('test_from_exc_tb' in e2[0]) + self.assertTrue(e2.args[0].startswith('__main__.MyError: eek')) + self.assertTrue('test_from_exc_tb' in e2.args[0]) if __name__ == '__main__': diff --git a/tests/call_function_test.py b/tests/call_function_test.py index 199aaa77..b9951e20 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -9,6 +9,10 @@ import mitogen.master import testlib +class MyError(Exception): + pass + + class CrazyType(object): pass @@ -18,7 +22,7 @@ def function_that_adds_numbers(x, y): def function_that_fails(): - raise ValueError('exception text') + raise MyError('exception text') def func_with_bad_return_value(): @@ -49,7 +53,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): s = str(exc) etype, _, s = s.partition(': ') - self.assertEqual(etype, 'exceptions.ValueError') + self.assertEqual(etype, '__main__.MyError') msg, _, s = s.partition('\n') self.assertEqual(msg, 'exception text') @@ -61,7 +65,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): exc = self.assertRaises(mitogen.core.StreamError, lambda: self.local.call(func_with_bad_return_value)) self.assertEquals( - exc[0], + exc.args[0], "cannot unpickle '%s'/'CrazyType'" % (__name__,), ) @@ -72,7 +76,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): self.broker.defer(stream.on_disconnect, self.broker) exc = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(exc[0], mitogen.core.ChannelError.local_msg) + self.assertEquals(exc.args[0], mitogen.core.ChannelError.local_msg) def test_aborted_on_local_broker_shutdown(self): stream = self.router._stream_by_id[self.local.context_id] @@ -81,7 +85,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): self.broker.shutdown() exc = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(exc[0], mitogen.core.ChannelError.local_msg) + self.assertEquals(exc.args[0], mitogen.core.ChannelError.local_msg) def test_accepts_returns_context(self): context = self.local.call(func_accepts_returns_context, self.local) diff --git a/tests/data/module_finder_testmod/regular_mod.py b/tests/data/module_finder_testmod/regular_mod.py index 865a115e..a7c0403b 100644 --- a/tests/data/module_finder_testmod/regular_mod.py +++ b/tests/data/module_finder_testmod/regular_mod.py @@ -3,4 +3,4 @@ import sys def say_hi(): - print 'hi' + print('hi') diff --git a/tests/data/self_contained_program.py b/tests/data/self_contained_program.py index 9ef18acb..13c20f03 100644 --- a/tests/data/self_contained_program.py +++ b/tests/data/self_contained_program.py @@ -12,4 +12,4 @@ def repr_stuff(): @mitogen.main() def main(router): context = router.local() - print context.call(repr_stuff) + print(context.call(repr_stuff)) diff --git a/tests/data/six_brokenpkg/__init__.py b/tests/data/six_brokenpkg/__init__.py index 8859928a..42e633c7 100644 --- a/tests/data/six_brokenpkg/__init__.py +++ b/tests/data/six_brokenpkg/__init__.py @@ -30,8 +30,8 @@ import os.path try: import six as _system_six print('six_brokenpkg: using system six:', _system_six) -except ImportError, e: - print('six_brokenpkg: no system six available', e) +except ImportError: + print('six_brokenpkg: no system six available') _system_six = None if _system_six: diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 1698a435..feaa34ce 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -4,6 +4,8 @@ import subprocess import unittest2 import mitogen.parent +from mitogen.core import b + import testlib @@ -36,7 +38,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): stdout, stderr = proc.communicate() self.assertEquals(0, proc.returncode) self.assertEquals(mitogen.parent.Stream.EC0_MARKER, stdout) - self.assertIn("Error -5 while decompressing data: incomplete or truncated stream", stderr) + self.assertIn(b("Error -5 while decompressing data: incomplete or truncated stream"), stderr) if __name__ == '__main__': diff --git a/tests/fork_test.py b/tests/fork_test.py index 47160855..8b396bbf 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -19,8 +19,12 @@ PLATFORM_TO_PATH = { ('darwin', True): '/usr/lib/libssl.dylib', ('linux2', False): '/usr/lib/libssl.so', ('linux2', True): '/usr/lib/x86_64-linux-gnu/libssl.so', + # Python 2.6 ('linux3', False): '/usr/lib/libssl.so', ('linux3', True): '/usr/lib/x86_64-linux-gnu/libssl.so', + # Python 3 + ('linux', False): '/usr/lib/libssl.so', + ('linux', True): '/usr/lib/x86_64-linux-gnu/libssl.so', } c_ssl = ctypes.CDLL(PLATFORM_TO_PATH[sys.platform, IS_64BIT]) diff --git a/tests/id_allocation_test.py b/tests/id_allocation_test.py index 1e8d8d1e..e290c8a4 100644 --- a/tests/id_allocation_test.py +++ b/tests/id_allocation_test.py @@ -15,7 +15,7 @@ def allocate_an_id(econtext): class SlaveTest(testlib.RouterMixin, testlib.TestCase): def test_slave_allocates_id(self): - context = self.router.local() + context = self.router.local(python_path='/users/dmw/src/cpython/build/bin/python2') # Master's allocator named the context 1. self.assertEquals(1, context.context_id) diff --git a/tests/importer_test.py b/tests/importer_test.py index 1a654a2d..b3dc49b3 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -11,6 +11,7 @@ import unittest2 import mitogen.core import mitogen.utils +from mitogen.core import b import testlib @@ -45,7 +46,7 @@ class ImporterMixin(testlib.RouterMixin): class LoadModuleTest(ImporterMixin, testlib.TestCase): - data = zlib.compress("data = 1\n\n") + data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' modname = 'fake_module' @@ -83,7 +84,7 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): - data = zlib.compress("data = 1\n\n") + data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' modname = 'mypkg.fake_module' # 0:fullname 1:pkg_present 2:path 3:compressed 4:related @@ -96,7 +97,7 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): class LoadModulePackageTest(ImporterMixin, testlib.TestCase): - data = zlib.compress("func = lambda: 1\n\n") + data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' modname = 'fake_pkg' # 0:fullname 1:pkg_present 2:path 3:compressed 4:related diff --git a/tests/latch_test.py b/tests/latch_test.py index 4ea2bc50..5be12d73 100644 --- a/tests/latch_test.py +++ b/tests/latch_test.py @@ -1,4 +1,5 @@ +import sys import threading import unittest2 @@ -66,7 +67,8 @@ class ThreadedGetTest(testlib.TestCase): def _worker(self, func): try: self.results.append(func()) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.results.append(None) self.excs.append(e) @@ -89,12 +91,12 @@ class ThreadedGetTest(testlib.TestCase): def test_five_threads(self): latch = self.klass() - for x in xrange(5): + for x in range(5): self.start_one(lambda: latch.get(timeout=3.0)) - for x in xrange(5): + for x in range(5): latch.put(x) self.join() - self.assertEquals(sorted(self.results), range(5)) + self.assertEquals(sorted(self.results), list(range(5))) self.assertEquals(self.excs, []) @@ -171,7 +173,8 @@ class ThreadedCloseTest(testlib.TestCase): def _worker(self, func): try: self.results.append(func()) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.results.append(None) self.excs.append(e) @@ -195,7 +198,7 @@ class ThreadedCloseTest(testlib.TestCase): def test_five_threads(self): latch = self.klass() - for x in xrange(5): + for x in range(5): self.start_one(lambda: latch.get(timeout=3.0)) latch.close() self.join() diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index d4db641f..971cd9ed 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -188,7 +188,6 @@ class FindRelatedTest(testlib.TestCase): SIMPLE_EXPECT = set([ 'mitogen', 'mitogen.compat', - 'mitogen.compat.collections', 'mitogen.compat.functools', 'mitogen.core', 'mitogen.master', diff --git a/tests/nested_test.py b/tests/nested_test.py index c11c2e4e..668c21e3 100644 --- a/tests/nested_test.py +++ b/tests/nested_test.py @@ -9,7 +9,6 @@ class NestedTest(testlib.RouterMixin, testlib.TestCase): def test_nested(self): context = None for x in range(1, 11): - #print 'Connect local%d via %s' % (x, context) context = self.router.local(via=context, name='local%d' % x) pid = context.call(os.getpid) diff --git a/tests/receiver_test.py b/tests/receiver_test.py index 3d67cf80..5dd6273a 100644 --- a/tests/receiver_test.py +++ b/tests/receiver_test.py @@ -6,7 +6,7 @@ import testlib def yield_stuff_then_die(sender): - for x in xrange(5): + for x in range(5): sender.send(x) sender.close() return 10 diff --git a/tests/router_test.py b/tests/router_test.py index 01f64d87..f02746ce 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -1,4 +1,3 @@ -import Queue import StringIO import logging import subprocess @@ -11,6 +10,11 @@ import mitogen.master import mitogen.parent import mitogen.utils +try: + import Queue +except ImportError: + import queue as Queue + def ping(): return True diff --git a/tests/show_docker_hostname.py b/tests/show_docker_hostname.py index d4326f92..995c744b 100644 --- a/tests/show_docker_hostname.py +++ b/tests/show_docker_hostname.py @@ -6,4 +6,4 @@ daemon from the environment. """ import testlib -print testlib.get_docker_host() +print(testlib.get_docker_host()) diff --git a/tests/soak/latch.py b/tests/soak/latch.py index ce00cee0..7483c6e3 100644 --- a/tests/soak/latch.py +++ b/tests/soak/latch.py @@ -26,10 +26,10 @@ def cons(): try: while 1: g = l.get() - print 'got=%s consumed=%s produced=%s crash=%s' % (g, consumed, produced, crash) + print('got=%s consumed=%s produced=%s crash=%s' % (g, consumed, produced, crash)) consumed += 1 time.sleep(g) - for x in xrange(int(g * 1000)): + for x in range(int(g * 1000)): pass except: crash += 1 diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 114c72b9..70a3f2d3 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -1,3 +1,5 @@ +import sys + import mitogen import mitogen.ssh import mitogen.utils @@ -53,8 +55,8 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): username='mitogen__has_sudo', ) assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError, e: - pass + except mitogen.ssh.PasswordError: + e = sys.exc_info()[1] self.assertEqual(e[0], self.stream_class.password_required_msg) @@ -65,8 +67,8 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): password='badpw', ) assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError, e: - pass + except mitogen.ssh.PasswordError: + e = sys.exc_info()[1] self.assertEqual(e[0], self.stream_class.password_incorrect_msg) @@ -87,8 +89,8 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): username='mitogen__has_sudo_pubkey', ) assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError, e: - pass + except mitogen.ssh.PasswordError: + e = sys.exc_info()[1] self.assertEqual(e[0], self.stream_class.password_required_msg) diff --git a/tests/testlib.py b/tests/testlib.py index 46868f6d..4fd2fa42 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,5 +1,4 @@ -import StringIO import logging import os import random @@ -8,7 +7,6 @@ import socket import subprocess import sys import time -import urlparse import unittest2 @@ -16,6 +14,16 @@ import mitogen.core import mitogen.master import mitogen.utils +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') sys.path.append(DATA_DIR) @@ -28,7 +36,7 @@ def data_path(suffix): path = os.path.join(DATA_DIR, suffix) if path.endswith('.key'): # SSH is funny about private key permissions. - os.chmod(path, 0600) + os.chmod(path, int('0600', 8)) return path @@ -82,7 +90,7 @@ def wait_for_port( return sock.settimeout(receive_timeout) - data = '' + data = mitogen.core.b('') found = False while time.time() < end: try: @@ -97,21 +105,25 @@ def wait_for_port( break data += resp - if re.search(pattern, data): + if re.search(mitogen.core.b(pattern), data): found = True break try: sock.shutdown(socket.SHUT_RDWR) - except socket.error, e: - # On Mac OS X - a BSD variant - the above code only succeeds if the operating system thinks that the - # socket is still open when shutdown() is invoked. If Python is too slow and the FIN packet arrives - # before that statement can be reached, then OS X kills the sock.shutdown() statement with: + except socket.error: + e = sys.exc_info()[1] + # On Mac OS X - a BSD variant - the above code only succeeds if the + # operating system thinks that the socket is still open when + # shutdown() is invoked. If Python is too slow and the FIN packet + # arrives before that statement can be reached, then OS X kills the + # sock.shutdown() statement with: # # socket.error: [Errno 57] Socket is not connected # - # Protect shutdown() with a try...except that catches the socket.error, test to make sure Errno is - # right, and ignore it if Errno matches. + # Protect shutdown() with a try...except that catches the + # socket.error, test to make sure Errno is right, and ignore it if + # Errno matches. if e.errno == 57: pass else: @@ -147,7 +159,7 @@ def sync_with_broker(broker, timeout=10.0): class LogCapturer(object): def __init__(self, name=None): - self.sio = StringIO.StringIO() + self.sio = StringIO() self.logger = logging.getLogger(name) self.handler = logging.StreamHandler(self.sio) self.old_propagate = self.logger.propagate @@ -169,9 +181,11 @@ class TestCase(unittest2.TestCase): raised. Can't use context manager because tests must run on Python2.4""" try: func(*args, **kwargs) - except exc, e: + except exc: + e = sys.exc_info()[1] return e - except BaseException, e: + except BaseException: + e = sys.exc_info()[1] assert 0, '%r raised %r, not %r' % (func, e, exc) assert 0, '%r did not raise %r' % (func, exc) diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py new file mode 100644 index 00000000..928807e1 --- /dev/null +++ b/tests/two_three_compat_test.py @@ -0,0 +1,33 @@ + +import logging +import time + +import unittest2 + +import mitogen.core +import mitogen.master + +import testlib + + +def roundtrip(*args): + return args + +class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): + if mitogen.core.PY3: + python_path = 'python2' + else: + python_path = 'python3' + + def test_succeeds(self): + self.router.enable_debug() + spare = self.router.fork() + target = self.router.local(python_path=self.python_path) + + spare2, = target.call(roundtrip, spare) + self.assertEquals(spare.context_id, spare2.context_id) + self.assertEquals(spare.name, spare2.name) + + +if __name__ == '__main__': + unittest2.main() From ec176bf1f8d83c1d1c01f6d0f4952a800dbe3230 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 02:17:50 +0100 Subject: [PATCH 03/50] tests: unittest2 import behaviour appears to differ between 2/3. In 2 the module is loaded as __main__ In 3 it has its actual name. --- tests/call_error_test.py | 25 ++++++++++++------------- tests/call_function_test.py | 9 +++------ tests/data/plain_old_module.py | 4 ++++ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/call_error_test.py b/tests/call_error_test.py index 15de01f8..b78d9f48 100644 --- a/tests/call_error_test.py +++ b/tests/call_error_test.py @@ -6,9 +6,8 @@ import unittest2 import mitogen.core - -class MyError(Exception): - pass +import testlib +import plain_old_module class ConstructorTest(unittest2.TestCase): @@ -23,9 +22,9 @@ class ConstructorTest(unittest2.TestCase): self.assertEquals(e.args[0], '11') def test_from_exc(self): - ve = MyError('eek') + ve = plain_old_module.MyError('eek') e = self.klass(ve) - self.assertEquals(e.args[0], '__main__.MyError: eek') + self.assertEquals(e.args[0], 'plain_old_module.MyError: eek') def test_form_base_exc(self): ve = SystemExit('eek') @@ -34,12 +33,12 @@ class ConstructorTest(unittest2.TestCase): def test_from_exc_tb(self): try: - raise MyError('eek') - except MyError: + raise plain_old_module.MyError('eek') + except plain_old_module.MyError: ve = sys.exc_info()[1] e = self.klass(ve) - self.assertTrue(e.args[0].startswith('__main__.MyError: eek')) + self.assertTrue(e.args[0].startswith('plain_old_module.MyError: eek')) self.assertTrue('test_from_exc_tb' in e.args[0]) @@ -57,20 +56,20 @@ class PickleTest(unittest2.TestCase): self.assertEquals(e2.args[0], '11') def test_from_exc(self): - ve = MyError('eek') + ve = plain_old_module.MyError('eek') e = self.klass(ve) e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2.args[0], '__main__.MyError: eek') + self.assertEquals(e2.args[0], 'plain_old_module.MyError: eek') def test_from_exc_tb(self): try: - raise MyError('eek') - except MyError: + raise plain_old_module.MyError('eek') + except plain_old_module.MyError: ve = sys.exc_info()[1] e = self.klass(ve) e2 = pickle.loads(pickle.dumps(e)) - self.assertTrue(e2.args[0].startswith('__main__.MyError: eek')) + self.assertTrue(e2.args[0].startswith('plain_old_module.MyError: eek')) self.assertTrue('test_from_exc_tb' in e2.args[0]) diff --git a/tests/call_function_test.py b/tests/call_function_test.py index b9951e20..f0074258 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -7,10 +7,7 @@ import mitogen.core import mitogen.master import testlib - - -class MyError(Exception): - pass +import plain_old_module class CrazyType(object): @@ -22,7 +19,7 @@ def function_that_adds_numbers(x, y): def function_that_fails(): - raise MyError('exception text') + raise plain_old_module.MyError('exception text') def func_with_bad_return_value(): @@ -53,7 +50,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): s = str(exc) etype, _, s = s.partition(': ') - self.assertEqual(etype, '__main__.MyError') + self.assertEqual(etype, 'plain_old_module.MyError') msg, _, s = s.partition('\n') self.assertEqual(msg, 'exception text') diff --git a/tests/data/plain_old_module.py b/tests/data/plain_old_module.py index 342b25a8..67991973 100755 --- a/tests/data/plain_old_module.py +++ b/tests/data/plain_old_module.py @@ -6,6 +6,10 @@ fiddlery. import math +class MyError(Exception): + pass + + def get_sentinel_value(): # Some proof we're even talking to the mitogen-test Docker image return open('/etc/sentinel').read() From cbcd8267d144e821999fe524dd5e273d47081972 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 03:19:59 +0100 Subject: [PATCH 04/50] tests: must decode() zlib result for 3.x. --- tests/importer_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/importer_test.py b/tests/importer_test.py index b3dc49b3..5c970438 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -118,7 +118,8 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) source = mod.__loader__.get_source(self.modname) - self.assertEquals(source, zlib.decompress(self.data)) + self.assertEquals(source, + mitogen.core.to_text(zlib.decompress(self.data))) def test_module_loader_set(self): self.set_get_module_response(self.response) From 826c477061b1713f0934879d29f3bd7348de102f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 03:20:25 +0100 Subject: [PATCH 05/50] tests: built-in exceptions live in a different module in 3.x --- tests/call_error_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/call_error_test.py b/tests/call_error_test.py index b78d9f48..447a80a9 100644 --- a/tests/call_error_test.py +++ b/tests/call_error_test.py @@ -29,7 +29,9 @@ class ConstructorTest(unittest2.TestCase): def test_form_base_exc(self): ve = SystemExit('eek') e = self.klass(ve) - self.assertEquals(e[0], 'exceptions.SystemExit: eek') + self.assertEquals(e.args[0], + # varies across 2/3. + '%s.%s: eek' % (type(ve).__module__, type(ve).__name__)) def test_from_exc_tb(self): try: From 9fb2371d64d63dafca0a0f663e42c3e8318375cd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 03:22:44 +0100 Subject: [PATCH 06/50] importer: reorder/tweak find_module() tests to cope with six.moves The old hack on the master side we had is broken for some reason on 3.x. Instead tweak the client to be more selective: if a request is for a module within a package, the package must be loaded (in sys.modules), and its __loader__ must be us. Previously if the module didn't exist in sys.modules, we'd still try to fetch from the master, which doesn't appear to ever make sense. --- mitogen/core.py | 28 +++++++++++++--------------- mitogen/master.py | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index db3d4be6..af7bf28e 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -682,27 +682,25 @@ class Importer(object): _tls.running = True try: - pkgname, dot, _ = fullname.rpartition('.') - _vv and IOLOG.debug('%r.find_module(%r)', self, fullname) - suffix = fullname[len(pkgname+dot):] - if suffix not in self._present.get(pkgname, (suffix,)): - _v and LOG.debug('%r: master doesn\'t know %r', self, fullname) - return None - - pkg = sys.modules.get(pkgname) - if pkg and getattr(pkg, '__loader__', None) is not self: - _vv and IOLOG.debug( - '%r: %r is submodule of a package we did not load', - self, fullname - ) - return None - + _v and LOG.debug('%r.find_module(%r)', self, fullname) # #114: explicitly whitelisted prefixes override any # system-installed package. if self.whitelist != ['']: if any(fullname.startswith(s) for s in self.whitelist): return self + pkgname, dot, _ = fullname.rpartition('.') + pkg = sys.modules.get(pkgname) + if pkgname and getattr(pkg, '__loader__', None) is not self: + LOG.debug('%r: %r is submodule of a package we did not load', + self, fullname) + return None + + suffix = fullname[len(pkgname+dot):] + if pkgname and suffix not in self._present.get(pkgname, ()): + _v and LOG.debug('%r: master doesn\'t know %r', self, fullname) + return None + try: self.builtin_find_module(fullname) _vv and IOLOG.debug('%r: %r is available locally', diff --git a/mitogen/master.py b/mitogen/master.py index 25bf8e83..46b1e84b 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -471,7 +471,7 @@ class ModuleFinder(object): for name in maybe_names if sys.modules.get(name) is not None and not is_stdlib_name(name) - and 'six.moves' not in name # TODO: crap + and u'six.moves' not in name # TODO: crap ) )) From 9903692811655a2d47636a1cd9917eda7f778cf4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 03:56:45 +0100 Subject: [PATCH 07/50] master: update scan_code_imports to cope with wordcode Constant-sized opcodes were introduced as an optimization in Python 3.6. See https://bugs.python.org/issue26647 --- mitogen/master.py | 41 ++++++++++++++++++++++++++++------------- tests/master_test.py | 19 +++++++++++++------ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 46b1e84b..159bf427 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -105,6 +105,33 @@ LOAD_CONST = dis.opname.index('LOAD_CONST') IMPORT_NAME = dis.opname.index('IMPORT_NAME') +if sys.version_info < (3, 0): + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = imap(ord, co.co_code) + nextb = ordit.next + return ((c, (None + if c < dis.HAVE_ARGUMENT else + (nextb() | (nextb() << 8)))) + for c in ordit) +elif sys.version_info < (3, 6): + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = iter(co.co_code) + nextb = ordit.__next__ + return ((c, (None + if c < dis.HAVE_ARGUMENT else + (nextb() | (nextb() << 8)))) + for c in ordit) +else: + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = iter(co.co_code) + nextb = ordit.__next__ + # https://github.com/abarnert/cpython/blob/c095a32f/Python/wordcode.md + return ((c, nextb()) for c in ordit) + + def scan_code_imports(co): """Given a code object `co`, scan its bytecode yielding any ``IMPORT_NAME`` and associated prior ``LOAD_CONST`` instructions @@ -120,19 +147,7 @@ def scan_code_imports(co): * `namelist`: for `ImportFrom`, the list of names to be imported from `modname`. """ - # Yield `(op, oparg)` tuples from the code object `co`. - if mitogen.core.PY3: - ordit = iter(co.co_code) - nextb = ordit.__next__ - else: - ordit = imap(ord, co.co_code) - nextb = ordit.next - - opit = ((c, (None - if c < dis.HAVE_ARGUMENT else - (nextb() | (nextb() << 8)))) - for c in ordit) - + opit = iter_opcodes(co) opit, opit2, opit3 = itertools.tee(opit, 3) try: next(opit2) diff --git a/tests/master_test.py b/tests/master_test.py index cf16d6c5..19a9b414 100644 --- a/tests/master_test.py +++ b/tests/master_test.py @@ -9,15 +9,22 @@ import mitogen.master class ScanCodeImportsTest(unittest2.TestCase): func = staticmethod(mitogen.master.scan_code_imports) + if mitogen.core.PY3: + level = 0 + else: + level = -1 + + SIMPLE_EXPECT = [ + (level, 'inspect', ()), + (level, 'unittest2', ()), + (level, 'testlib', ()), + (level, 'mitogen.master', ()), + ] + def test_simple(self): source_path = inspect.getsourcefile(ScanCodeImportsTest) co = compile(open(source_path).read(), source_path, 'exec') - self.assertEquals(list(self.func(co)), [ - (-1, 'inspect', ()), - (-1, 'unittest2', ()), - (-1, 'testlib', ()), - (-1, 'mitogen.master', ()), - ]) + self.assertEquals(list(self.func(co)), self.SIMPLE_EXPECT) if __name__ == '__main__': From b0404bef401ae294aa9a5c1072cb3b051c931215 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 04:11:10 +0100 Subject: [PATCH 08/50] tests: fix get_module_via_* encoding issues --- mitogen/master.py | 5 +++-- tests/module_finder_test.py | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 159bf427..f3009999 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -308,9 +308,10 @@ class ModuleFinder(object): resembles a Python script. For now we simply verify the file contains ASCII text. """ - fp = open(path, 'r') + fp = open(path, 'rb') try: - return not set(fp.read(512)).difference(string.printable) + sample = fp.read(512).decode('latin-1') + return not set(sample).difference(string.printable) finally: fp.close() diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 971cd9ed..f4bbd712 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -60,14 +60,14 @@ class GetModuleViaPkgutilTest(testlib.TestCase): path, src, is_pkg = self.call('module_finder_testmod') self.assertEquals(path, testlib.data_path('module_finder_testmod/__init__.py')) - self.assertEquals('', src) + self.assertEquals(mitogen.core.b(''), src) self.assertTrue(is_pkg) def test_empty_source_module(self): path, src, is_pkg = self.call('module_finder_testmod.empty_mod') self.assertEquals(path, testlib.data_path('module_finder_testmod/empty_mod.py')) - self.assertEquals('', src) + self.assertEquals(mitogen.core.b(''), src) self.assertFalse(is_pkg) def test_regular_mod(self): @@ -75,7 +75,8 @@ class GetModuleViaPkgutilTest(testlib.TestCase): path, src, is_pkg = self.call('module_finder_testmod.regular_mod') self.assertEquals(path, testlib.data_path('module_finder_testmod/regular_mod.py')) - self.assertEquals(src, inspect.getsource(regular_mod)) + self.assertEquals(mitogen.core.to_text(src), + inspect.getsource(regular_mod)) self.assertFalse(is_pkg) @@ -89,7 +90,7 @@ class GetModuleViaSysModulesTest(testlib.TestCase): import __main__ path, src, is_pkg = self.call('__main__') self.assertEquals(path, __main__.__file__) - self.assertEquals(src, open(path).read()) + self.assertEquals(src, open(path, 'rb').read()) self.assertFalse(is_pkg) def test_dylib_fails(self): From c141dd10ec70b2d306c60c064f122f28528ae156 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 04:11:35 +0100 Subject: [PATCH 09/50] master: fix resolve_relpath() looks like this was just as broken on 2.x, and suddenly we're finding a bunch more legit Django deps. It seems anywhere absolute_import appeared in 2.x, we skipped some imports. --- mitogen/master.py | 4 ++-- tests/module_finder_test.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index f3009999..04464a5d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -436,7 +436,7 @@ class ModuleFinder(object): # This would be an ImportError in real code. return '' - return '.'.join(bits[:-level]) + return '.'.join(bits[:-level]) + '.' def generate_parent_names(self, fullname): while '.' in fullname: @@ -471,7 +471,7 @@ class ModuleFinder(object): modnames = [modname, '%s.%s' % (fullname, modname)] else: modnames = [ - '%s.%s' % (self.resolve_relpath(fullname, level), modname) + '%s%s' % (self.resolve_relpath(fullname, level), modname) ] maybe_names.extend(modnames) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index f4bbd712..2e9447de 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -120,7 +120,7 @@ class ResolveRelPathTest(testlib.TestCase): self.assertEquals('', self.call('email.utils', 0)) def test_rel1(self): - self.assertEquals('email', self.call('email.utils', 1)) + self.assertEquals('email.', self.call('email.utils', 1)) def test_rel2(self): self.assertEquals('', self.call('email.utils', 2)) @@ -188,14 +188,15 @@ class FindRelatedTest(testlib.TestCase): SIMPLE_EXPECT = set([ 'mitogen', - 'mitogen.compat', - 'mitogen.compat.functools', 'mitogen.core', 'mitogen.master', 'mitogen.minify', 'mitogen.parent', ]) + if sys.version_info < (3, 2): + SIMPLE_EXPECT.add('mitogen.compat') + SIMPLE_EXPECT.add('mitogen.compat.functools') if sys.version_info < (2, 7): SIMPLE_EXPECT.add('mitogen.compat.tokenize') @@ -289,25 +290,45 @@ class DjangoFindRelatedTest(testlib.TestCase): 'django.db.models.related', 'django.db.models.signals', 'django.db.models.sql', + 'django.db.models.sql.aggregates', + 'django.db.models.sql.constants', + 'django.db.models.sql.datastructures', + 'django.db.models.sql.expressions', + 'django.db.models.sql.query', + 'django.db.models.sql.subqueries', + 'django.db.models.sql.where', 'django.db.transaction', 'django.db.utils', 'django.dispatch', 'django.dispatch.dispatcher', 'django.dispatch.saferef', 'django.forms', + 'django.forms.fields', + 'django.forms.forms', + 'django.forms.formsets', + 'django.forms.models', + 'django.forms.util', + 'django.forms.widgets', 'django.utils', 'django.utils._os', 'django.utils.crypto', 'django.utils.datastructures', + 'django.utils.dateformat', 'django.utils.dateparse', + 'django.utils.dates', + 'django.utils.datetime_safe', 'django.utils.decorators', 'django.utils.deprecation', 'django.utils.encoding', + 'django.utils.formats', 'django.utils.functional', + 'django.utils.html', + 'django.utils.html_parser', 'django.utils.importlib', 'django.utils.ipv6', 'django.utils.itercompat', 'django.utils.module_loading', + 'django.utils.numberformat', 'django.utils.safestring', 'django.utils.six', 'django.utils.text', @@ -316,6 +337,10 @@ class DjangoFindRelatedTest(testlib.TestCase): 'django.utils.tree', 'django.utils.tzinfo', 'pkg_resources', + 'pkg_resources.extern', + 'pkg_resources.extern.appdirs', + 'pkg_resources.extern.packaging', + 'pkg_resources.extern.six', 'pytz', 'pytz.exceptions', 'pytz.tzfile', From 76978610116c16fbeae8422366628ee217bc376b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 04:46:07 +0100 Subject: [PATCH 10/50] tests: skip one test that triggers bug in ancient vendered six.py --- tests/module_finder_test.py | 51 ++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 2e9447de..9c85e26c 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -129,7 +129,35 @@ class ResolveRelPathTest(testlib.TestCase): self.assertEquals('', self.call('email.utils', 3)) -class FindRelatedImportsTest(testlib.TestCase): +class DjangoMixin(object): + WEBPROJECT_PATH = testlib.data_path('webproject') + + # TODO: rip out Django and replace with a static tree of weird imports that + # don't depend on .. Django! The hack below is because the version of + # Django we need to test against 2.6 doesn't actually run on 3.6. But we + # don't care, we just need to be able to import it. + # + # File "django/utils/html_parser.py", line 12, in + # AttributeError: module 'html.parser' has no attribute 'HTMLParseError' + # + import pkg_resources._vendor.six + from django.utils.six.moves import html_parser as _html_parser + _html_parser.HTMLParseError = Exception + + @classmethod + def setUpClass(cls): + super(DjangoMixin, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoMixin, cls).tearDownClass() + + +class FindRelatedImportsTest(DjangoMixin, testlib.TestCase): klass = mitogen.master.ModuleFinder def call(self, fullname): @@ -180,7 +208,7 @@ class FindRelatedImportsTest(testlib.TestCase): ]) -class FindRelatedTest(testlib.TestCase): +class FindRelatedTest(DjangoMixin, testlib.TestCase): klass = mitogen.master.ModuleFinder def call(self, fullname): @@ -206,27 +234,13 @@ class FindRelatedTest(testlib.TestCase): self.assertEquals(set(related), self.SIMPLE_EXPECT) -class DjangoFindRelatedTest(testlib.TestCase): +class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): klass = mitogen.master.ModuleFinder maxDiff = None def call(self, fullname): return self.klass().find_related(fullname) - WEBPROJECT_PATH = testlib.data_path('webproject') - - @classmethod - def setUpClass(cls): - super(DjangoFindRelatedTest, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoFindRelatedTest, cls).tearDownClass() - def test_django_db(self): import django.db related = self.call('django.db') @@ -251,6 +265,9 @@ class DjangoFindRelatedTest(testlib.TestCase): ]) def test_django_db_models(self): + if sys.version_info >= (3, 0): + raise unittest2.SkipTest('broken due to ancient vendored six.py') + import django.db.models related = self.call('django.db.models') self.assertEquals(related, [ From 0eb77b5f7c0661c3e87311667fdcd29e71d3fa15 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 06:24:46 +0100 Subject: [PATCH 11/50] utils: always enable microsecond logging. It's too useful, and the logs are fairly out of control already, may as well just capture everything in the first pass. --- ansible_mitogen/logging.py | 2 +- docs/api.rst | 6 +----- docs/getting_started.rst | 4 ---- mitogen/utils.py | 12 ++++-------- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 3d14377b..d0fa13c6 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -41,7 +41,7 @@ class Handler(logging.Handler): """ def __init__(self, display, normal_method): logging.Handler.__init__(self) - self.formatter = mitogen.utils.log_get_formatter(usec=True) + self.formatter = mitogen.utils.log_get_formatter() self.display = display self.normal_method = normal_method diff --git a/docs/api.rst b/docs/api.rst index febaeb91..b4e2e40b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1323,7 +1323,7 @@ A random assortment of utility functions useful on masters and children. OS X bundles some ancient version of the :py:mod:`six` module. .. currentmodule:: mitogen.utils -.. function:: log_to_file (path=None, io=False, usec=False, level='INFO') +.. function:: log_to_file (path=None, io=False, level='INFO') Install a new :py:class:`logging.Handler` writing applications logs to the filesystem. Useful when debugging slave IO problems. @@ -1339,10 +1339,6 @@ A random assortment of utility functions useful on masters and children. If :data:`True`, include extremely verbose IO logs in the output. Useful for debugging hangs, less useful for debugging application code. - :parm bool usec: - If :data:`True`, include microsecond timestamps. This greatly helps - when debugging races and similar determinism issues. - :param str level: Name of the :py:mod:`logging` package constant that is the minimum level to log at. Useful levels are ``DEBUG``, ``INFO``, ``WARNING``, diff --git a/docs/getting_started.rst b/docs/getting_started.rst index c3fac279..77defd1c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -155,10 +155,6 @@ Logging Environment Variables produces verbose records of any IO interaction, which is useful for debugging hangs and deadlocks. -``MITOGEN_LOG_USEC`` - If present, forces microsecond-level timestamps for any call to - :py:func:`mitogen.utils.log_to_file`. - Logging Records diff --git a/mitogen/utils.py b/mitogen/utils.py index e326bd6c..4dc4a23d 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -56,19 +56,15 @@ def _formatTime(record, datefmt=None): return dt.strftime(datefmt) -def log_get_formatter(usec=False): - usec = ('MITOGEN_LOG_USEC' in os.environ) or usec - datefmt = '%H:%M:%S' - if usec: - datefmt += '.%f' - +def log_get_formatter(): + datefmt = '%H:%M:%S.%f' fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s' formatter = logging.Formatter(fmt, datefmt) formatter.formatTime = _formatTime return formatter -def log_to_file(path=None, io=False, usec=False, level='INFO'): +def log_to_file(path=None, io=False, level='INFO'): log = logging.getLogger('') if path: fp = open(path, 'w', 1) @@ -86,7 +82,7 @@ def log_to_file(path=None, io=False, usec=False, level='INFO'): log.setLevel(level) handler = logging.StreamHandler(fp) - handler.formatter = log_get_formatter(usec=usec) + handler.formatter = log_get_formatter() log.handlers.insert(0, handler) From db529e8228e0d5e31c9ed00f7fe6bb45b54a9930 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 06:56:46 +0100 Subject: [PATCH 12/50] core: fix Receiver.__iter__ regression on EOF --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index af7bf28e..a897af2f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -575,7 +575,7 @@ class Receiver(object): def __iter__(self): while True: - msg = self.get() + msg = self.get(throw_dead=False) if msg.is_dead: return yield msg From 0422a8c263ae52272e78eaaabee6bf8827a703d3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 07:01:10 +0100 Subject: [PATCH 13/50] parent: python_path setting depends on local or remote For local, we want to default to the same Python version as the current process. For remote, we want whatever is on offer. --- docs/api.rst | 2 +- mitogen/parent.py | 2 +- mitogen/ssh.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b4e2e40b..0bb42ec2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -485,7 +485,7 @@ Router Class :param str python_path: Path to the Python interpreter to use for bootstrap. Defaults to - ``python2.7``. In future this may default to ``sys.executable``. + :data:`sys.executable`. For SSH, defaults to ``python``. :param bool debug: If :data:`True`, arrange for debug logging (:py:meth:`enable_debug`) to diff --git a/mitogen/parent.py b/mitogen/parent.py index d1175838..9bc069bc 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -672,7 +672,7 @@ class Stream(mitogen.core.Stream): Base for streams capable of starting new slaves. """ #: The path to the remote Python interpreter. - python_path = 'python' + python_path = sys.executable #: Maximum time to wait for a connection attempt. connect_timeout = 30.0 diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 36efbffc..eae70a36 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -103,6 +103,10 @@ class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) child_is_immediate_subprocess = False + #: Default to whatever is available as 'python' on the remote machine, + #: overriding sys.executable use. + python_path = 'python' + #: Number of -v invocations to pass on command line. ssh_debug_level = 0 From 6cbf34d5fe234dd61c686b3151dafc6009d059bf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 08:25:40 +0100 Subject: [PATCH 14/50] tests: another smattering of 3.x type/API fixes. --- mitogen/utils.py | 5 ++++- tests/data/plain_old_module.py | 2 +- tests/responder_test.py | 6 +++--- tests/router_test.py | 3 +-- tests/service_test.py | 2 +- tests/ssh_test.py | 6 +++--- tests/testlib.py | 4 ++-- tests/types_test.py | 22 ++++++++++++++-------- 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/mitogen/utils.py b/mitogen/utils.py index 4dc4a23d..e0acf7d0 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -99,7 +99,10 @@ def run_with_router(func, *args, **kwargs): def with_router(func): def wrapper(*args, **kwargs): return run_with_router(func, *args, **kwargs) - wrapper.func_name = func.func_name + if mitogen.core.PY3: + wrapper.func_name = func.__name__ + else: + wrapper.func_name = func.func_name return wrapper diff --git a/tests/data/plain_old_module.py b/tests/data/plain_old_module.py index 67991973..49294464 100755 --- a/tests/data/plain_old_module.py +++ b/tests/data/plain_old_module.py @@ -12,7 +12,7 @@ class MyError(Exception): def get_sentinel_value(): # Some proof we're even talking to the mitogen-test Docker image - return open('/etc/sentinel').read() + return open('/etc/sentinel').read().decode() def add(x, y): diff --git a/tests/responder_test.py b/tests/responder_test.py index 86e6dd7e..dfdd67fa 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -30,7 +30,7 @@ class GoodModulesTest(testlib.RouterMixin, unittest2.TestCase): # Ensure a program composed of a single script can be imported # successfully. args = [sys.executable, testlib.data_path('self_contained_program.py')] - output = testlib.subprocess__check_output(args) + output = testlib.subprocess__check_output(args).decode() self.assertEquals(output, "['__main__', 50]\n") @@ -45,7 +45,7 @@ class BrokenModulesTest(unittest2.TestCase): router.stream_by_id = lambda n: stream msg = mitogen.core.Message( - data='non_existent_module', + data=mitogen.core.b('non_existent_module'), reply_to=50, ) msg.router = router @@ -74,7 +74,7 @@ class BrokenModulesTest(unittest2.TestCase): router.stream_by_id = lambda n: stream msg = mitogen.core.Message( - data='six_brokenpkg._six', + data=mitogen.core.b('six_brokenpkg._six'), reply_to=50, ) msg.router = router diff --git a/tests/router_test.py b/tests/router_test.py index f02746ce..68474e00 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -1,4 +1,3 @@ -import StringIO import logging import subprocess import time @@ -146,7 +145,7 @@ class PolicyTest(testlib.RouterMixin, testlib.TestCase): # Verify CallError received by reply_to target. e = self.assertRaises(mitogen.core.CallError, lambda: reply_target.get().unpickle()) - self.assertEquals(e[0], self.router.refused_msg) + self.assertEquals(e.args[0], self.router.refused_msg) class CrashTest(testlib.BrokerMixin, unittest2.TestCase): diff --git a/tests/service_test.py b/tests/service_test.py index 8cb18258..8e2cdac3 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -83,7 +83,7 @@ class PermissionTest(testlib.RouterMixin, testlib.TestCase): exc = self.assertRaises(mitogen.core.CallError, lambda: l2.call(call_service_in, l1, MyService.name(), 'privileged_op')) msg = mitogen.service.Invoker.unauthorized_msg % ( - 'privileged_op', + u'privileged_op', MyService.name(), ) self.assertTrue(msg in exc.args[0]) diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 70a3f2d3..35f88cc8 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -58,7 +58,7 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): except mitogen.ssh.PasswordError: e = sys.exc_info()[1] - self.assertEqual(e[0], self.stream_class.password_required_msg) + self.assertEqual(e.args[0], self.stream_class.password_required_msg) def test_password_incorrect(self): try: @@ -70,7 +70,7 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): except mitogen.ssh.PasswordError: e = sys.exc_info()[1] - self.assertEqual(e[0], self.stream_class.password_incorrect_msg) + self.assertEqual(e.args[0], self.stream_class.password_incorrect_msg) def test_password_specified(self): context = self.docker_ssh( @@ -92,7 +92,7 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase): except mitogen.ssh.PasswordError: e = sys.exc_info()[1] - self.assertEqual(e[0], self.stream_class.password_required_msg) + self.assertEqual(e.args[0], self.stream_class.password_required_msg) def test_pubkey_specified(self): context = self.docker_ssh( diff --git a/tests/testlib.py b/tests/testlib.py index 4fd2fa42..008a3dc0 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -214,7 +214,7 @@ class DockerizedSshDaemon(object): def _get_container_port(self): s = subprocess__check_output(['docker', 'port', self.container_name]) - for line in s.splitlines(): + for line in s.decode().splitlines(): dport, proto, baddr, bport = self.PORT_RE.match(line).groups() if dport == '22' and proto == 'tcp': self.port = int(bport) @@ -291,7 +291,7 @@ class DockerMixin(RouterMixin): kwargs.setdefault('hostname', self.dockerized_ssh.host) kwargs.setdefault('port', self.dockerized_ssh.port) kwargs.setdefault('check_host_keys', 'ignore') - kwargs.setdefault('ssh_debug_level', '3') + kwargs.setdefault('ssh_debug_level', 3) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): diff --git a/tests/types_test.py b/tests/types_test.py index 44296656..4f80e076 100644 --- a/tests/types_test.py +++ b/tests/types_test.py @@ -1,16 +1,22 @@ -import cStringIO +try: + from io import StringIO + from io import BytesIO +except ImportError: + from StringIO import StringIO as StringIO + from StringIO import StringIO as BytesIO import unittest2 import mitogen.core +from mitogen.core import b class BlobTest(unittest2.TestCase): klass = mitogen.core.Blob def make(self): - return self.klass('x' * 128) + return self.klass(b('x') * 128) def test_repr(self): blob = self.make() @@ -18,14 +24,14 @@ class BlobTest(unittest2.TestCase): def test_decays_on_constructor(self): blob = self.make() - self.assertEquals('x'*128, mitogen.core.BytesType(blob)) + self.assertEquals(b('x')*128, mitogen.core.BytesType(blob)) def test_decays_on_write(self): blob = self.make() - io = cStringIO.StringIO() + io = BytesIO() io.write(blob) self.assertEquals(128, io.tell()) - self.assertEquals('x'*128, io.getvalue()) + self.assertEquals(b('x')*128, io.getvalue()) def test_message_roundtrip(self): blob = self.make() @@ -53,7 +59,7 @@ class SecretTest(unittest2.TestCase): def test_decays_on_write(self): secret = self.make() - io = cStringIO.StringIO() + io = StringIO() io.write(secret) self.assertEquals(8, io.tell()) self.assertEquals('password', io.getvalue()) @@ -64,8 +70,8 @@ class SecretTest(unittest2.TestCase): secret2 = msg.unpickle() self.assertEquals(type(secret), type(secret2)) self.assertEquals(repr(secret), repr(secret2)) - self.assertEquals(mitogen.core.BytesType(secret), - mitogen.core.BytesType(secret2)) + self.assertEquals(mitogen.core.b(secret), + mitogen.core.b(secret2)) if __name__ == '__main__': From 6b4e0470177bfa178826e2f39d408d43e498f139 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 15:41:48 +0100 Subject: [PATCH 15/50] tests: 3.x parent_test fixes. --- mitogen/parent.py | 2 +- tests/parent_test.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 9bc069bc..e2c22110 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -340,7 +340,7 @@ def iter_read(fds, deadline=None): if not poller.readers: raise mitogen.core.StreamError( 'EOF on stream; last 300 bytes received: %r' % - (b('').join(bits)[-300:],) + (b('').join(bits)[-300:].decode('latin1'),) ) raise mitogen.core.TimeoutError('read timed out') diff --git a/tests/parent_test.py b/tests/parent_test.py index c4dafb62..31c168e9 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -78,7 +78,7 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): connect_timeout=3, ) ) - prefix = 'Child start failed: [Errno 2] No such file or directory.' + prefix = 'Child start failed: [Errno 2] No such file or directory' self.assertTrue(e.args[0].startswith(prefix)) def test_via_enoent(self): @@ -90,7 +90,7 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): connect_timeout=3, ) ) - s = 'Child start failed: [Errno 2] No such file or directory.' + s = 'Child start failed: [Errno 2] No such file or directory' self.assertTrue(s in e.args[0]) @@ -123,12 +123,12 @@ class TtyCreateChildTest(unittest2.TestCase): ]) deadline = time.time() + 5.0 for line in mitogen.parent.iter_read([fd], deadline): - self.assertEquals('hi\n', line) + self.assertEquals(mitogen.core.b('hi\n'), line) break waited_pid, status = os.waitpid(pid, 0) self.assertEquals(pid, waited_pid) self.assertEquals(0, status) - self.assertEquals('', tf.read()) + self.assertEquals(mitogen.core.b(''), tf.read()) finally: tf.close() @@ -194,7 +194,7 @@ class WriteAllTest(unittest2.TestCase): mitogen.core.set_nonblock(proc.stdin.fileno()) return proc - ten_ms_chunk = ('x' * 65535) + ten_ms_chunk = (mitogen.core.b('x') * 65535) def test_no_deadline(self): proc = self.make_proc() From 9e3e02fb651d93a98868968b9992a527454fe9d8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 16:06:44 +0100 Subject: [PATCH 16/50] tokenize must parse under 3.x even if it won't run there To support 3.x parent 2.x child --- mitogen/compat/tokenize.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mitogen/compat/tokenize.py b/mitogen/compat/tokenize.py index d426cd2d..dd12af83 100644 --- a/mitogen/compat/tokenize.py +++ b/mitogen/compat/tokenize.py @@ -33,7 +33,6 @@ from token import * import token __all__ = [x for x in dir(token) if not x.startswith("_")] __all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] -del x del token COMMENT = N_TOKENS @@ -150,8 +149,8 @@ class StopTokenizing(Exception): pass def printtoken(type, token, srow_scol, erow_ecol, line): # for testing srow, scol = srow_scol erow, ecol = erow_ecol - print "%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token)) + print("%d,%d-%d,%d:\t%s\t%s" % \ + (srow, scol, erow, ecol, tok_name[type], repr(token))) def tokenize(readline, tokeneater=printtoken): """ @@ -316,7 +315,7 @@ def generate_tokens(readline): if contstr: # continued string if not line: - raise TokenError, ("EOF in multi-line string", strstart) + raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) @@ -377,7 +376,7 @@ def generate_tokens(readline): else: # continued statement if not line: - raise TokenError, ("EOF in multi-line statement", (lnum, 0)) + raise TokenError("EOF in multi-line statement", (lnum, 0)) continued = 0 while pos < max: From 90f2f823db2c43120a76863708d68b3bacc4bd98 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 16:18:19 +0100 Subject: [PATCH 17/50] tests: remove stray hard-coded interpreter. --- tests/id_allocation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/id_allocation_test.py b/tests/id_allocation_test.py index e290c8a4..1e8d8d1e 100644 --- a/tests/id_allocation_test.py +++ b/tests/id_allocation_test.py @@ -15,7 +15,7 @@ def allocate_an_id(econtext): class SlaveTest(testlib.RouterMixin, testlib.TestCase): def test_slave_allocates_id(self): - context = self.router.local(python_path='/users/dmw/src/cpython/build/bin/python2') + context = self.router.local() # Master's allocator named the context 1. self.assertEquals(1, context.context_id) From 24ecfb3b134f299b4ea6b40c487e5ea199436b75 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 16:28:57 +0100 Subject: [PATCH 18/50] Fix setns.py syntax for 3.x --- mitogen/setns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitogen/setns.py b/mitogen/setns.py index b07b4df7..1779ca77 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -151,7 +151,8 @@ class Stream(mitogen.parent.Stream): os.readlink(nspath + name) != os.readlink(selfpath + name) ) ] - except Exception, e: + except Exception: + e = sys.exc_info()[1] raise Error(str(e)) os.chdir('/proc/%s/root' % (self.leader_pid,)) From a52064a24fb1a007c75c67bdcabc362cb2a04845 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 16:35:48 +0100 Subject: [PATCH 19/50] core: reordered find_module() test was broken (again) e81b3bd0652b5eb125eb224ceca281b9d540dd5e The whitelist check must happen /after/ the other checks, otherwise we unconditionally retunr self for crap like 'ansible.module_utils.json'. --- mitogen/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index a897af2f..6bed7132 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -683,12 +683,6 @@ class Importer(object): _tls.running = True try: _v and LOG.debug('%r.find_module(%r)', self, fullname) - # #114: explicitly whitelisted prefixes override any - # system-installed package. - if self.whitelist != ['']: - if any(fullname.startswith(s) for s in self.whitelist): - return self - pkgname, dot, _ = fullname.rpartition('.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: @@ -701,6 +695,12 @@ class Importer(object): _v and LOG.debug('%r: master doesn\'t know %r', self, fullname) return None + # #114: explicitly whitelisted prefixes override any + # system-installed package. + if self.whitelist != ['']: + if any(fullname.startswith(s) for s in self.whitelist): + return self + try: self.builtin_find_module(fullname) _vv and IOLOG.debug('%r: %r is available locally', From 4ff47d6a9352a50ebfb90e6fb22886a424df70c2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 26 Jun 2018 16:43:09 +0100 Subject: [PATCH 20/50] parent: more 2/3x format fixes --- mitogen/parent.py | 2 +- tests/parent_test.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index e2c22110..d8485e15 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -339,7 +339,7 @@ def iter_read(fds, deadline=None): if not poller.readers: raise mitogen.core.StreamError( - 'EOF on stream; last 300 bytes received: %r' % + u'EOF on stream; last 300 bytes received: %r' % (b('').join(bits)[-300:].decode('latin1'),) ) raise mitogen.core.TimeoutError('read timed out') diff --git a/tests/parent_test.py b/tests/parent_test.py index 31c168e9..06eac97e 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -57,7 +57,8 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): connect_timeout=3, ) ) - self.assertEquals(e.args[0], "EOF on stream; last 300 bytes received: ''") + prefix = "EOF on stream; last 300 bytes received: " + self.assertTrue(e.args[0].startswith(prefix)) def test_via_eof(self): # Verify FD leakage does not keep failed process open. @@ -69,7 +70,8 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): connect_timeout=3, ) ) - self.assertTrue("EOF on stream; last 300 bytes received: ''" in e.args[0]) + s = "EOF on stream; last 300 bytes received: " + self.assertTrue(s in e.args[0]) def test_direct_enoent(self): e = self.assertRaises(mitogen.core.StreamError, From 1f6c47554ab0bce68220cbfd9e90e8b01e3e5816 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 27 Jun 2018 15:37:08 +0100 Subject: [PATCH 21/50] examples: update mitogen-fuse.py for 3.x. --- examples/mitogen-fuse.py | 48 +++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index 4f6bcf81..b086398d 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -1,9 +1,13 @@ #!/usr/bin/env python +# +# pip install fusepy +# # This implementation could improve a /lot/, but the core library is missing # some functionality (#213) to make that easy. Additionally it needs a set of # Python bindings for FUSE that stupidly require use of a thread pool. from __future__ import absolute_import, division +from __future__ import unicode_literals import errno import logging @@ -23,13 +27,25 @@ import os LOG = logging.getLogger(__name__) +def to_text(p): + """ + On 3.x, fusepy returns paths as bytes. + """ + if isinstance(p, bytes): + return p.decode('utf-8') + return p + + def errno_wrap(modname, func, *args): try: return getattr(globals()[modname], func)(*args), None except (IOError, OSError): - LOG.exception('While running %r(**%r)', func, args) e = sys.exc_info()[1] - return None, errno.errorcode[e.args[0]] + if e.args[0] == errno.ENOENT: + LOG.error('%r(**%r): %s', func, args, e) + else: + LOG.exception('While running %r(**%r)', func, args) + return None, to_text(errno.errorcode[e.args[0]]) def errno_call(context, func, *args): @@ -128,14 +144,17 @@ class Operations(fuse.Operations): # fuse.LoggingMixIn, return self._context def chmod(self, path, mode): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.chmod, path, mode) def chown(self, path, uid, gid): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.chown, path, uid, gid) def create(self, path, mode): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, _create, path, mode) or 0 @@ -156,10 +175,12 @@ class Operations(fuse.Operations): # fuse.LoggingMixIn, return errno_call(self._context, _stat, path) def mkdir(self, path, mode): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.mkdir, path, mode) def read(self, path, size, offset, fh): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, _read, path, size, offset) @@ -176,41 +197,52 @@ class Operations(fuse.Operations): # fuse.LoggingMixIn, return errno_call(self._context, os.readlink, path) def rename(self, old, new): + old = old.decode(self.encoding) + new = new.decode(self.encoding) return errno_call(self._context, os.rename, old, new) # TODO return self.sftp.rename(old, self.root + new) def rmdir(self, path): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.rmdir, path) def symlink(self, target, source): + target = target.decode(self.encoding) + source = source.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.symlink, source, target) def truncate(self, path, length, fh=None): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, _truncate, path, length) def unlink(self, path): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.unlink, path) def utimens(self, path, times=None): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, os.utime, path, times) def write(self, path, data, offset, fh): + path = path.decode(self.encoding) _evil_name(path) return errno_call(self._context, _write, path, data, offset) -if __name__ == '__main__': +@mitogen.main(log_level='DEBUG') +def main(router): if len(sys.argv) != 3: print('usage: %s ' % sys.argv[0]) sys.exit(1) - ops = Operations(sys.argv[1]) - mount_point = sys.argv[2] - - mitogen.utils.log_to_file(level='DEBUG') - blerp = fuse.FUSE(ops, mount_point, foreground=True) + blerp = fuse.FUSE( + operations=Operations(sys.argv[1]), + mountpoint=sys.argv[2], + foreground=True, + volname='%s (Mitogen)' % (sys.argv[1],), + ) From 8791d400818ac6cbebbe78543ac93ca4fabac881 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 27 Jun 2018 15:38:49 +0100 Subject: [PATCH 22/50] parent: tidy up call_async() logging. --- mitogen/parent.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index d8485e15..486c2850 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -451,6 +451,9 @@ def _proxy_connect(name, method_name, kwargs, econtext): class Argv(object): + """ + Wrapper to defer argv formatting when debug logging is disabled. + """ def __init__(self, argv): self.argv = argv @@ -467,6 +470,38 @@ class Argv(object): return ' '.join(map(self.escape, self.argv)) +class CallSpec(object): + """ + Wrapper to defer call argument formatting when debug logging is disabled. + """ + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def _get_name(self): + return u'%s.%s' % (self.func.__module__, + self.func.__name__) + + def _get_args(self): + return u', '.join(repr(a) for a in self.args) + + def _get_kwargs(self): + s = u'' + if self.kwargs: + s = u', '.join('%s=%r' % (k, v) for k, v in self.kwargs.items()) + if self.args: + s = u', ' + s + return s + + def __repr__(self): + return '%s(%s%s)' % ( + self._get_name(), + self._get_args(), + self._get_kwargs(), + ) + + class KqueuePoller(mitogen.core.Poller): _repr = 'KqueuePoller()' @@ -949,8 +984,7 @@ class Context(mitogen.core.Context): return hash((self.router, self.context_id)) def call_async(self, fn, *args, **kwargs): - LOG.debug('%r.call_async(%r, *%r, **%r)', - self, fn, args, kwargs) + LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs)) return self.send_async(make_call_msg(fn, *args, **kwargs)) def call(self, fn, *args, **kwargs): From caa648b1ea98d2907df32f8182afc29b7cbc1237 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 3 Jul 2018 17:37:35 +0100 Subject: [PATCH 23/50] utils: handle duplicate log_to_file() calls. --- mitogen/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mitogen/utils.py b/mitogen/utils.py index e0acf7d0..4fd80aa1 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -81,7 +81,14 @@ def log_to_file(path=None, io=False, level='INFO'): level = getattr(logging, level, logging.INFO) log.setLevel(level) + # Prevent accidental duplicate log_to_file() calls from generating + # duplicate output. + for handler_ in reversed(log.handlers): + if getattr(handler_, 'is_mitogen', None): + log.handlers.remove(handler_) + handler = logging.StreamHandler(fp) + handler.is_mitogen = True handler.formatter = log_get_formatter() log.handlers.insert(0, handler) From 970f54de8575154cd64dadd9c32a8ae48e66acd8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 3 Jul 2018 17:39:38 +0100 Subject: [PATCH 24/50] service: remove stray debug --- mitogen/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mitogen/service.py b/mitogen/service.py index 9d87839f..217ed6ef 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -499,10 +499,6 @@ class Pool(object): def _validate(self, msg): tup = msg.unpickle(throw=False) - LOG.debug('_validate(): %r', tup) - LOG.debug('_validate(): %r', mitogen.core.PY3) - LOG.debug('_validate(): %r', list(map(type, tup))) - LOG.debug('_validate(): UnicodeType=%r', mitogen.core.UnicodeType) if not (isinstance(tup, tuple) and len(tup) == 3 and isinstance(tup[0], mitogen.core.AnyTextType) and From 42276f158b931b6ce6a29c39bfb722817aac24f9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 15:46:04 +0100 Subject: [PATCH 25/50] core: log the data received on the latch file handle. --- mitogen/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 6bed7132..cccf2dfe 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1258,8 +1258,10 @@ class Latch(object): if i >= self._waking: raise e or TimeoutError() self._waking -= 1 - if rsock.recv(2) != b('\x7f'): - raise LatchError('internal error: received >1 wakeups') + byte = rsock.recv(10) + if byte != b('\x7f'): + raise LatchError('internal error: received >1 wakeups: %r' + % (byte,)) if e: raise e if self.closed: From 7ff9d573b675116ec6258ab5af5cc2d82229eeff Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 15:48:50 +0100 Subject: [PATCH 26/50] tests: remove stray enable_debug() --- tests/two_three_compat_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index 928807e1..babbbe39 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -20,7 +20,6 @@ class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): python_path = 'python3' def test_succeeds(self): - self.router.enable_debug() spare = self.router.fork() target = self.router.local(python_path=self.python_path) From e24eddb1ce32b8585deeb2cf72fbd23f50eeb25d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 16:16:43 +0100 Subject: [PATCH 27/50] core: move Latch docs back inline. --- docs/internals.rst | 63 +------------------------------------- mitogen/core.py | 76 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 66 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 3aa1fac4..63030215 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -17,69 +17,8 @@ Latch Class .. currentmodule:: mitogen.core -.. class:: Latch () +.. autoclass:: Latch () - A latch is a :py:class:`Queue.Queue`-like object that supports mutation and - waiting from multiple threads, however unlike :py:class:`Queue.Queue`, - waiting threads always remain interruptible, so CTRL+C always succeeds, and - waits where a timeout is set experience no wake up latency. These - properties are not possible in combination using the built-in threading - primitives available in Python 2.x. - - Latches implement queues using the UNIX self-pipe trick, and a per-thread - :py:func:`socket.socketpair` that is lazily created the first time any - latch attempts to sleep on a thread, and dynamically associated with the - waiting Latch only for duration of the wait. - - See :ref:`waking-sleeping-threads` for further discussion. - - .. method:: empty () - - Return :py:data:`True` if calling :py:meth:`get` would block. - - As with :py:class:`Queue.Queue`, :py:data:`True` may be returned even - though a subsequent call to :py:meth:`get` will succeed, since a - message may be posted at any moment between :py:meth:`empty` and - :py:meth:`get`. - - As with :py:class:`Queue.Queue`, :py:data:`False` may be returned even - though a subsequent call to :py:meth:`get` will block, since another - waiting thread may be woken at any moment between :py:meth:`empty` and - :py:meth:`get`. - - .. method:: get (timeout=None, block=True) - - Return the next object enqueued on this latch, or sleep waiting for - one. - - :param float timeout: - If not :py:data:`None`, specifies a timeout in seconds. - - :param bool block: - If :py:data:`False`, immediately raise - :py:class:`mitogen.core.TimeoutError` if the latch is empty. - - :raises mitogen.core.LatchError: - :py:meth:`close` has been called, and the object is no longer valid. - - :raises mitogen.core.TimeoutError: - Timeout was reached. - - :returns: - The de-queued object. - - .. method:: put (obj) - - Enqueue an object on this latch, waking the first thread that is asleep - waiting for a result, if one exists. - - :raises mitogen.core.LatchError: - :py:meth:`close` has been called, and the object is no longer valid. - - .. method:: close () - - Mark the latch as closed, and cause every sleeping thread to be woken, - with :py:class:`mitogen.core.LatchError` raised in each thread. Side Class diff --git a/mitogen/core.py b/mitogen/core.py index cccf2dfe..351e56ee 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1175,6 +1175,21 @@ class Poller(object): class Latch(object): + """ + A latch is a :py:class:`Queue.Queue`-like object that supports mutation and + waiting from multiple threads, however unlike :py:class:`Queue.Queue`, + waiting threads always remain interruptible, so CTRL+C always succeeds, and + waits where a timeout is set experience no wake up latency. These + properties are not possible in combination using the built-in threading + primitives available in Python 2.x. + + Latches implement queues using the UNIX self-pipe trick, and a per-thread + :py:func:`socket.socketpair` that is lazily created the first time any + latch attempts to sleep on a thread, and dynamically associated with the + waiting Latch only for duration of the wait. + + See :ref:`waking-sleeping-threads` for further discussion. + """ poller_class = Poller closed = False _waking = 0 @@ -1188,11 +1203,18 @@ class Latch(object): @classmethod def _on_fork(cls): + """ + Clean up any files belonging to the parent process after a fork. + """ cls._sockets = [] while cls._allsockets: cls._allsockets.pop().close() def close(self): + """ + Mark the latch as closed, and cause every sleeping thread to be woken, + with :py:class:`mitogen.core.LatchError` raised in each thread. + """ self._lock.acquire() try: self.closed = True @@ -1203,9 +1225,25 @@ class Latch(object): self._lock.release() def empty(self): + """ + Return :py:data:`True` if calling :py:meth:`get` would block. + + As with :py:class:`Queue.Queue`, :py:data:`True` may be returned even + though a subsequent call to :py:meth:`get` will succeed, since a + message may be posted at any moment between :py:meth:`empty` and + :py:meth:`get`. + + As with :py:class:`Queue.Queue`, :py:data:`False` may be returned even + though a subsequent call to :py:meth:`get` will block, since another + waiting thread may be woken at any moment between :py:meth:`empty` and + :py:meth:`get`. + """ return len(self._queue) == 0 - def _tls_init(self): + def _get_socketpair(self): + """ + Return an unused socketpair, creating one if none exist. + """ # pop() must be atomic, which is true for GIL-equipped interpreters. try: return self._sockets.pop() @@ -1217,6 +1255,25 @@ class Latch(object): return rsock, wsock def get(self, timeout=None, block=True): + """ + Return the next enqueued object, or sleep waiting for one. + + :param float timeout: + If not :py:data:`None`, specifies a timeout in seconds. + + :param bool block: + If :py:data:`False`, immediately raise + :py:class:`mitogen.core.TimeoutError` if the latch is empty. + + :raises mitogen.core.LatchError: + :py:meth:`close` has been called, and the object is no longer valid. + + :raises mitogen.core.TimeoutError: + Timeout was reached. + + :returns: + The de-queued object. + """ _vv and IOLOG.debug('%r.get(timeout=%r, block=%r)', self, timeout, block) self._lock.acquire() @@ -1229,7 +1286,7 @@ class Latch(object): return self._queue.pop(i) if not block: raise TimeoutError() - rsock, wsock = self._tls_init() + rsock, wsock = self._get_socketpair() self._sleeping.append(wsock) finally: self._lock.release() @@ -1242,11 +1299,15 @@ class Latch(object): poller.close() def _get_sleep(self, poller, timeout, block, rsock, wsock): + """ + When a result is not immediately available, sleep waiting for + :meth:`put` to write a byte to our socket pair. + """ _vv and IOLOG.debug('%r._get_sleep(timeout=%r, block=%r)', self, timeout, block) e = None try: - list(poller.poll(timeout)) + l = list(poller.poll(timeout)) except Exception: e = sys.exc_info()[1] @@ -1256,7 +1317,7 @@ class Latch(object): del self._sleeping[i] self._sockets.append((rsock, wsock)) if i >= self._waking: - raise e or TimeoutError() + raise e or TimeoutError(repr(l)) self._waking -= 1 byte = rsock.recv(10) if byte != b('\x7f'): @@ -1272,6 +1333,13 @@ class Latch(object): self._lock.release() def put(self, obj): + """ + Enqueue an object, waking the first thread waiting for a result, if one + exists. + + :raises mitogen.core.LatchError: + :py:meth:`close` has been called, and the object is no longer valid. + """ _vv and IOLOG.debug('%r.put(%r)', self, obj) self._lock.acquire() try: From 3e48f954396d3e973ef2bbc3871ddbfba4327a0a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 15:22:18 +0000 Subject: [PATCH 28/50] tests: import cpu_load.py. --- tests/soak/cpu_load.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/soak/cpu_load.py diff --git a/tests/soak/cpu_load.py b/tests/soak/cpu_load.py new file mode 100644 index 00000000..ed473b15 --- /dev/null +++ b/tests/soak/cpu_load.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +""" +Put the machine's CPUs under pressure to increase the likelihood of scheduling +weirdness. Useful for exposing otherwise difficult to hit races in the library. +""" + +import multiprocessing + +def burn(): + while 1: pass + +mul = 2 +count = int(mul * multiprocessing.cpu_count()) +print count + +procs = [multiprocessing.Process(target=burn) + for _ in range(count)] + +for i, proc in enumerate(procs): + print([i]) + proc.start() From ff2f44b046109a1fe515003ba86a385d830e3677 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 16:59:56 +0000 Subject: [PATCH 29/50] core: reduce chance of Latch.read()/write()/close() race. Previously it was possible for a thread to call Waker.defer() after Broker has torns its Waker down, and the underlying file descriptor reallocated by the OS to some other component. This manifested as latches of a subsequent test invocation receiving the waker byte (' ') rather than their expected byte '\x7f'. This doesn't fix the problem, it just significantly reduces the chance of it occurring. In future Side.write()/read()/close() must be synchronized with a lock. Previously the problem could be reliably triggered with: while :; do python tests/call_function_test.py -vf CallFunctionTest.{test_aborted_on_local_broker_shutdown,test_aborted_on_local_context_disconnect} done --- mitogen/core.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 351e56ee..593b9d21 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -869,17 +869,23 @@ class Side(object): def close(self): if not self.closed: _vv and IOLOG.debug('%r.close()', self) - os.close(self.fd) self.closed = True + os.close(self.fd) def read(self, n=CHUNK_SIZE): + if self.closed: + # Refuse to touch the handle after closed, it may have been reused + # by another thread. TODO: synchronize read()/write()/close(). + return b('') s, disconnected = io_op(os.read, self.fd, n) if disconnected: - return '' + return b('') return s def write(self, s): - if self.fd is None: + if self.closed or self.fd is None: + # Refuse to touch the handle after closed, it may have been reused + # by another thread. return None written, disconnected = io_op(os.write, self.fd, s) @@ -1303,11 +1309,13 @@ class Latch(object): When a result is not immediately available, sleep waiting for :meth:`put` to write a byte to our socket pair. """ - _vv and IOLOG.debug('%r._get_sleep(timeout=%r, block=%r)', - self, timeout, block) + _vv and IOLOG.debug('%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)', + self, timeout, block, rsock.fileno(), + wsock.fileno()) e = None + woken = None try: - l = list(poller.poll(timeout)) + woken = list(poller.poll(timeout)) except Exception: e = sys.exc_info()[1] @@ -1317,7 +1325,9 @@ class Latch(object): del self._sleeping[i] self._sockets.append((rsock, wsock)) if i >= self._waking: - raise e or TimeoutError(repr(l)) + recv = rsock.recv(10) if woken else None + s = '%r: woken=%r, recv=%r' % (self, woken, recv) + raise e or TimeoutError(s) self._waking -= 1 byte = rsock.recv(10) if byte != b('\x7f'): From 46ec0ffcaf3896f4af85d08ec67f5ba936421c27 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 17:38:18 +0000 Subject: [PATCH 30/50] tests: log original traceback in assertRaises(). --- tests/testlib.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testlib.py b/tests/testlib.py index 008a3dc0..d812609c 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -25,6 +25,7 @@ except ImportError: from io import StringIO +LOG = logging.getLogger(__name__) DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') sys.path.append(DATA_DIR) @@ -185,6 +186,7 @@ class TestCase(unittest2.TestCase): e = sys.exc_info()[1] return e except BaseException: + LOG.exception('Original exception') e = sys.exc_info()[1] assert 0, '%r raised %r, not %r' % (func, e, exc) assert 0, '%r did not raise %r' % (func, exc) From b4f4b6667913c1727f0312f8515861e3535bf1cd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 17:38:34 +0000 Subject: [PATCH 31/50] tests: make cpu_load.py more random. --- tests/soak/cpu_load.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/soak/cpu_load.py b/tests/soak/cpu_load.py index ed473b15..8890272c 100644 --- a/tests/soak/cpu_load.py +++ b/tests/soak/cpu_load.py @@ -4,12 +4,26 @@ Put the machine's CPUs under pressure to increase the likelihood of scheduling weirdness. Useful for exposing otherwise difficult to hit races in the library. """ +import ctypes import multiprocessing +import os +import time + +LIBC = ctypes.CDLL('libc.so.6') +sched_yield = LIBC.sched_yield + def burn(): - while 1: pass + while 1: + a, b, c = os.urandom(3) + n = int(((ord(a) << 16) | + (ord(b) << 8) | + (ord(c) << 0)) / 1.6) + print(n) + for x in xrange(n): pass + sched_yield() -mul = 2 +mul = 1.5 count = int(mul * multiprocessing.cpu_count()) print count From dd62009f70c60483503a396495984e58dab8aa79 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 17:38:34 +0000 Subject: [PATCH 32/50] Record ulimit -a in run_test output. --- run_tests | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_tests b/run_tests index 71ce18aa..122cd79e 100755 --- a/run_tests +++ b/run_tests @@ -1,4 +1,10 @@ -#/bin/sh +#/usr/bin/env bash + +echo '----- ulimits -----' +ulimit -a +echo '-------------------' +echo + set -o errexit set -o nounset set -o pipefail From 04e138e060915883e16fe49d02182defa67f3abc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 20:55:14 +0100 Subject: [PATCH 33/50] core: fix serialization of empty bytes() on 3.x. --- mitogen/core.py | 2 ++ tests/serialization_test.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/serialization_test.py diff --git a/mitogen/core.py b/mitogen/core.py index 593b9d21..51350351 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -425,6 +425,8 @@ class Message(object): return Kwargs elif module == '_codecs' and func == 'encode': return self._unpickle_bytes + elif module == '__builtin__' and func == 'bytes': + return BytesType raise StreamError('cannot unpickle %r/%r', module, func) @property diff --git a/tests/serialization_test.py b/tests/serialization_test.py new file mode 100644 index 00000000..3b56e10a --- /dev/null +++ b/tests/serialization_test.py @@ -0,0 +1,37 @@ + +try: + from io import StringIO + from io import BytesIO +except ImportError: + from StringIO import StringIO as StringIO + from StringIO import StringIO as BytesIO + +import unittest2 + +import mitogen.core +from mitogen.core import b + + +def roundtrip(v): + msg = mitogen.core.Message.pickled(v) + return mitogen.core.Message(data=msg.data).unpickle() + + +class BlobTest(unittest2.TestCase): + klass = mitogen.core.Blob + + # Python 3 pickle protocol 2 does weird stuff depending on whether an empty + # or nonempty bytes is being serialized. For non-empty, it yields a + # _codecs.encode() call. For empty, it yields a bytes() call. + + def test_nonempty_bytes(self): + v = mitogen.core.Blob(b('dave')) + self.assertEquals(b('dave'), roundtrip(v)) + + def test_empty_bytes(self): + v = mitogen.core.Blob(b('')) + self.assertEquals(b(''), roundtrip(v)) + + +if __name__ == '__main__': + unittest2.main() From 34751c38b0a2f6dfb83daed334d9fdfa3d22cc52 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Jul 2018 23:28:25 +0000 Subject: [PATCH 34/50] ansible: cStringIO interprets unicode as a buffer obj on 2.x --- ansible_mitogen/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 5cd1f032..37f3b788 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -52,7 +52,8 @@ import mitogen.core import ansible_mitogen.target # TODO: circular import try: - from cStringIO import StringIO + # Cannot use cStringIO as it does not support Unicode. + from StringIO import StringIO except ImportError: from io import StringIO From b21d51be474ac0809cee4c979ca28a3a6f6f9d86 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 00:41:58 +0100 Subject: [PATCH 35/50] ansible: fix test for 3.x. --- tests/ansible/lib/modules/mitogen_test_gethostbyname.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 23dff9bd..f6070b03 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -9,7 +9,7 @@ def main(): module = AnsibleModule(argument_spec={'name': {'type': 'str'}}) try: module.exit_json(addr=socket.gethostbyname(module.params['name'])) - except socket.error, e: + except socket.error as e: module.fail_json(msg=str(e)) if __name__ == '__main__': From 3e73e6ae1153021b5958cd1950d669b60377f732 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 00:10:00 +0000 Subject: [PATCH 36/50] Reenable all Travis Jobs. --- .travis.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2696c437..85bbc1e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,52 +29,52 @@ matrix: - python: "2.7" env: MODE=mitogen DISTRO=debian # 2.7 -> 2.6 - #- python: "2.7" - #env: MODE=mitogen DISTRO=centos6 + - python: "2.7" + env: MODE=mitogen DISTRO=centos6 # 2.6 -> 2.7 - #- python: "2.6" - #env: MODE=mitogen DISTRO=centos7 + - python: "2.6" + env: MODE=mitogen DISTRO=centos7 # 2.6 -> 2.6 - #- python: "2.6" - #env: MODE=mitogen DISTRO=centos6 + - python: "2.6" + env: MODE=mitogen DISTRO=centos6 # 3.6 -> 2.7 - python: "3.6" env: MODE=mitogen DISTRO=debian # Debops tests. # 2.4.3.0; 2.7 -> 2.7 - #- python: "2.7" - #env: MODE=debops_common VER=2.4.3.0 + - python: "2.7" + env: MODE=debops_common VER=2.4.3.0 # 2.5.5; 2.7 -> 2.7 - #- python: "2.7" - #env: MODE=debops_common VER=2.5.5 + - python: "2.7" + env: MODE=debops_common VER=2.5.5 # 2.5.5; 3.6 -> 2.7 - #- python: "3.6" - #env: MODE=debops_common VER=2.5.5 + - python: "3.6" + env: MODE=debops_common VER=2.5.5 # ansible_mitogen tests. # 2.4.3.0; Debian; 2.7 -> 2.7 - #- python: "2.7" - #env: MODE=ansible VER=2.4.3.0 DISTRO=debian + - python: "2.7" + env: MODE=ansible VER=2.4.3.0 DISTRO=debian # 2.5.5; Debian; 2.7 -> 2.7 - python: "2.7" env: MODE=ansible VER=2.5.5 DISTRO=debian # 2.5.5; CentOS; 2.7 -> 2.7 - #- python: "2.7" - #env: MODE=ansible VER=2.5.5 DISTRO=centos7 + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=centos7 # 2.5.5; CentOS; 2.7 -> 2.6 - #- python: "2.7" - #env: MODE=ansible VER=2.5.5 DISTRO=centos6 + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=centos6 # 2.5.5; CentOS; 2.6 -> 2.7 - #- python: "2.6" - #env: MODE=ansible VER=2.5.5 DISTRO=centos7 + - python: "2.6" + env: MODE=ansible VER=2.5.5 DISTRO=centos7 # 2.5.5; CentOS; 2.6 -> 2.6 - #- python: "2.6" - #env: MODE=ansible VER=2.5.5 DISTRO=centos6 + - python: "2.6" + env: MODE=ansible VER=2.5.5 DISTRO=centos6 # 2.5.5; Debian; 3.6 -> 2.7 - python: "3.6" env: MODE=ansible VER=2.5.5 DISTRO=centos6 # Sanity check our tests against vanilla Ansible, they should pass. - #- python: "2.7" - #env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear + - python: "2.7" + env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear From 29f15c236cf1180ba82ac8e578fdeb8d09606ab4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 18:12:38 +0100 Subject: [PATCH 37/50] core: remove needless size prefix from core_src_fd. I think this is brainwrong held over from an early attempt to write the duplicate copy of core_src on stdin. --- mitogen/core.py | 3 +-- mitogen/parent.py | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 51350351..4d8ece60 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1910,8 +1910,7 @@ class ExternalContext(object): if core_src_fd: fp = os.fdopen(core_src_fd, 'rb', 1) try: - core_size = int(fp.readline()) - core_src = fp.read(core_size) + core_src = fp.read() # Strip "ExternalContext.main()" call from last line. core_src = b('\n').join(core_src.splitlines()[:-1]) finally: diff --git a/mitogen/parent.py b/mitogen/parent.py index 486c2850..fb63612a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -825,13 +825,20 @@ class Stream(mitogen.core.Stream): # file descriptor 0 as 100, creates a pipe, then execs a new interpreter # with a custom argv. # * Optimized for minimum byte count after minification & compression. - # * 'CONTEXT_NAME', 'PREAMBLE_COMPRESSED_LEN', and 'PREAMBLE_LEN' are - # substituted with their respective values. + # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with + # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in # order to allow virtualenvs to detect their install prefix. # * For Darwin, OS X installs a craptacular argv0-introspecting Python # version switcher as /usr/bin/python. Override attempts to call it # with an explicit call to python2.7 + # + # Locals: + # R: read side of interpreter stdin. + # W: write side of interpreter stdin. + # r: read side of core_src FD. + # w: write side of core_src FD. + # C: the decompressed core source. @staticmethod def _first_stage(): R,W=os.pipe() @@ -850,12 +857,12 @@ class Stream(mitogen.core.Stream): os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') - wfp=os.fdopen(W,'wb',0) - wfp.write(C) - Wfp=os.fdopen(w,'wb',0) - Wfp.write('PREAMBLE_LEN\n'.encode()+C) - wfp.close() - Wfp.close() + fp=os.fdopen(W,'wb',0) + fp.write(C) + fp.close() + fp=os.fdopen(w,'wb',0) + fp.write(C) + fp.close() os.write(1,'MITO001\n'.encode()) def get_boot_command(self): @@ -866,8 +873,6 @@ class Stream(mitogen.core.Stream): preamble_compressed = self.get_preamble() source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) - source = source.replace('PREAMBLE_LEN', - str(len(zlib.decompress(preamble_compressed)))) compressed = zlib.compress(source.encode(), 9) encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) # We can't use bytes.decode() in 3.x since it was restricted to always From b92545e61a60efc5fda778df54fecbea9ab1bbff Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 19:02:59 +0100 Subject: [PATCH 38/50] minify: avoid cStringIO use. On 2.7 it was "accidentally fine" because the buffer object the StringIO was initialized from happened to look like ASCII, but in 2.6 either UCS-2 or UCS-4 is used for that buffer, and so the result was junk. Just use the io module everywhere if we can, falling back to pure-Python StringIO for Python<2.6. --- mitogen/minify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/minify.py b/mitogen/minify.py index 13b0d364..26ecf62f 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -29,9 +29,9 @@ import sys try: - from cStringIO import StringIO -except ImportError: from io import StringIO +except ImportError: + from StringIO import StringIO import mitogen.core From bd65e50b2f8a1c406a38d6da372548aa10228946 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 19:49:59 +0000 Subject: [PATCH 39/50] ansible: reduce() does not exist in 3.x. --- ansible_mitogen/target.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 85cd67c0..c9be4bfb 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -35,6 +35,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import errno +import functools import grp import json import logging @@ -573,7 +574,7 @@ def apply_mode_spec(spec, mode): mask = CHMOD_MASKS[ch] bits = CHMOD_BITS[ch] cur_perm_bits = mode & mask - new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0) + new_perm_bits = functools.reduce(operator.or_, (bits[p] for p in perms), 0) mode &= ~mask if op == '=': mode |= new_perm_bits From ea5a4ed38526ce90bf80fd47043aa3e996de8365 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 21:29:40 +0100 Subject: [PATCH 40/50] tests: use Ansible 2.5.5 by default --- .travis/ansible_tests.sh | 2 +- .travis/debops_common_tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index ffe775fe..5c236521 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -3,7 +3,7 @@ TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" TMPDIR="/tmp/ansible-tests-$$" -ANSIBLE_VERSION="${VER:-2.4.3.0}" +ANSIBLE_VERSION="${VER:-2.5.5}" export ANSIBLE_STRATEGY="${STRATEGY:-mitogen_linear}" DISTRO="${DISTRO:-debian}" diff --git a/.travis/debops_common_tests.sh b/.travis/debops_common_tests.sh index bdfeb146..f1909c10 100755 --- a/.travis/debops_common_tests.sh +++ b/.travis/debops_common_tests.sh @@ -4,7 +4,7 @@ TMPDIR="/tmp/debops-$$" TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" TARGET_COUNT="${TARGET_COUNT:-2}" -ANSIBLE_VERSION="${VER:-2.4.3.0}" +ANSIBLE_VERSION="${VER:-2.5.5}" DISTRO=debian # Naturally DebOps only supports Debian. export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}" From 18c9cd23658d28a335fc2b74a6a7ad1b4362610a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 23:46:48 +0100 Subject: [PATCH 41/50] docs: update supported Python version list. --- docs/ansible.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index b7502586..3a33594a 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -53,8 +53,8 @@ Installation ------------ 1. Thoroughly review the documented behavioural differences. -2. Verify Ansible 2.3/2.4/2.5 and Python 2.7 are listed in ``ansible --version`` - output. +2. Verify Ansible 2.3-2.5 and Python 2.6, 2.7 or 3.6 are listed in ``ansible + --version`` output. 3. Download and extract https://github.com/dw/mitogen/archive/master.zip 4. Modify ``ansible.cfg``: From a74903d59a6ebbae237190547f3ed55748e8ec74 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 23:58:01 +0100 Subject: [PATCH 42/50] docs: update compatibility notice on main page. --- docs/index.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3e5271bf..b7ec66cf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -406,14 +406,12 @@ a large fleet of machines, or to alert the parent of unexpected state changes. Compatibility ############# -The package is written using syntax compatible all the way back to **Python -2.4** released November 2004, making it suitable for managing a fleet of -potentially ancient corporate hardware. For example Mitogen can be used out of -the box against Red Hat Enterprise Linux 5, released in 2007. - -Support for Python 3 is included using 2to3 triggered during setup.py, however -such a Mitogen install does not support communicating with older 2.x systems. A -future revision will support full cross 2/3 compatibility. +Mitogen is syntax-compatible with **Python 2.4** released November 2004, making +it suitable for managing a fleet of potentially ancient corporate hardware, +such as Red Hat Enterprise Linux 5, released in 2007. + +Every combination of Python 3.x/2.x parent and child should be possible, +however at present only Python 2.6, 2.7 and 3.6 are tested automatically. Zero Dependencies From a53f2e7137c561251895d7e7b5ac18e6bfe919ba Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Jul 2018 23:58:20 +0100 Subject: [PATCH 43/50] Update trove classifiers. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fdda624a..2547e6ed 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 2 :: Only', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', From 484d4fdb74a89c9153bd71ea03194477c14383ec Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 7 Jul 2018 07:31:07 +0000 Subject: [PATCH 44/50] core: fix Latch socket sharing race. If thread A is about to wake as thread B is about to sleep, and A loses the GIL at an inopportune moment, it was possible for two latches to share the same socketpair, causing wakeups routed to the wrong latch. The pair was returned to the 'idle sockets' list before .recv() had been called. This manifested as TimeoutError() thrown rarely with many active threads and the host is heavily loaded (such as Travis CI). Add more documentation and stop writing single wake bytes. Instead the recipient's identity is written instead, making it simpler to detect future bugs. --- mitogen/core.py | 99 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4d8ece60..b6936a40 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1199,24 +1199,41 @@ class Latch(object): See :ref:`waking-sleeping-threads` for further discussion. """ poller_class = Poller - closed = False - _waking = 0 - _sockets = [] - _allsockets = [] + + # The _cls_ prefixes here are to make it crystal clear in the code which + # state mutation isn't covered by :attr:`_lock`. + + #: List of reusable :func:`socket.socketpair` tuples. The list is from + #: multiple threads, the only safe operations are `append()` and `pop()`. + _cls_idle_socketpairs = [] + + #: List of every socket object that must be closed by :meth:`_on_fork`. + #: Inherited descriptors cannot be reused, as the duplicated handles + #: reference the same underlying kernel-side sockets still in use by + #: the parent process. + _cls_all_sockets = [] def __init__(self): + self.closed = False self._lock = threading.Lock() + #: List of unconsumed enqueued items. self._queue = [] + #: List of `(wsock, cookie)` awaiting an element, where `wsock` is the + #: socketpair's write side, and `cookie` is the string to write. self._sleeping = [] + #: Number of elements of :attr:`_sleeping` that have already been + #: woken, and have a corresponding element index from :attr:`_queue` + #: assigned to them. + self._waking = 0 @classmethod def _on_fork(cls): """ Clean up any files belonging to the parent process after a fork. """ - cls._sockets = [] - while cls._allsockets: - cls._allsockets.pop().close() + cls._cls_idle_socketpairs = [] + while cls._cls_all_sockets: + cls._cls_all_sockets.pop().close() def close(self): """ @@ -1227,7 +1244,8 @@ class Latch(object): try: self.closed = True while self._waking < len(self._sleeping): - self._wake(self._sleeping[self._waking]) + wsock, cookie = self._sleeping[self._waking] + self._wake(wsock, cookie) self._waking += 1 finally: self._lock.release() @@ -1252,16 +1270,26 @@ class Latch(object): """ Return an unused socketpair, creating one if none exist. """ - # pop() must be atomic, which is true for GIL-equipped interpreters. try: - return self._sockets.pop() + return self._cls_idle_socketpairs.pop() # pop() must be atomic except IndexError: rsock, wsock = socket.socketpair() set_cloexec(rsock.fileno()) set_cloexec(wsock.fileno()) - self._allsockets.extend((rsock, wsock)) + self._cls_all_sockets.extend((rsock, wsock)) return rsock, wsock + COOKIE_SIZE = 33 + + def _make_cookie(self): + """ + Return a 33-byte string encoding the ID of the instance and the current + thread. This disambiguates legitimate wake-ups, accidental writes to + the FD, and buggy internal FD sharing. + """ + ident = threading.currentThread().ident + return b(u'%016x-%016x' % (int(id(self)), ident)) + def get(self, timeout=None, block=True): """ Return the next enqueued object, or sleep waiting for one. @@ -1295,25 +1323,28 @@ class Latch(object): if not block: raise TimeoutError() rsock, wsock = self._get_socketpair() - self._sleeping.append(wsock) + cookie = self._make_cookie() + self._sleeping.append((wsock, cookie)) finally: self._lock.release() poller = self.poller_class() poller.start_receive(rsock.fileno()) try: - return self._get_sleep(poller, timeout, block, rsock, wsock) + return self._get_sleep(poller, timeout, block, rsock, wsock, cookie) finally: poller.close() - def _get_sleep(self, poller, timeout, block, rsock, wsock): + def _get_sleep(self, poller, timeout, block, rsock, wsock, cookie): """ When a result is not immediately available, sleep waiting for :meth:`put` to write a byte to our socket pair. """ - _vv and IOLOG.debug('%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)', - self, timeout, block, rsock.fileno(), - wsock.fileno()) + _vv and IOLOG.debug( + '%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)', + self, timeout, block, rsock.fileno(), wsock.fileno() + ) + e = None woken = None try: @@ -1323,20 +1354,22 @@ class Latch(object): self._lock.acquire() try: - i = self._sleeping.index(wsock) + i = self._sleeping.index((wsock, cookie)) del self._sleeping[i] - self._sockets.append((rsock, wsock)) - if i >= self._waking: - recv = rsock.recv(10) if woken else None - s = '%r: woken=%r, recv=%r' % (self, woken, recv) - raise e or TimeoutError(s) + if not woken: + raise e or TimeoutError() + + got_cookie = rsock.recv(self.COOKIE_SIZE) + self._cls_idle_socketpairs.append((rsock, wsock)) + + assert cookie == got_cookie, ( + "Cookie incorrect; got %r, expected %r" \ + % (got_cookie, cookie) + ) + assert i < self._waking, ( + "Cookie correct, but no queue element assigned." + ) self._waking -= 1 - byte = rsock.recv(10) - if byte != b('\x7f'): - raise LatchError('internal error: received >1 wakeups: %r' - % (byte,)) - if e: - raise e if self.closed: raise LatchError() _vv and IOLOG.debug('%r.get() wake -> %r', self, self._queue[i]) @@ -1360,17 +1393,17 @@ class Latch(object): self._queue.append(obj) if self._waking < len(self._sleeping): - sock = self._sleeping[self._waking] + wsock, cookie = self._sleeping[self._waking] self._waking += 1 _vv and IOLOG.debug('%r.put() -> waking wfd=%r', self, sock.fileno()) - self._wake(sock) + self._wake(wsock, cookie) finally: self._lock.release() - def _wake(self, sock): + def _wake(self, wsock, cookie): try: - os.write(sock.fileno(), b('\x7f')) + os.write(wsock.fileno(), cookie) except OSError: e = sys.exc_info()[1] if e.args[0] != errno.EBADF: From 5da371c5dd602cf8190505ae67af6e2033549429 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 7 Jul 2018 14:49:30 +0100 Subject: [PATCH 45/50] service: fix UnboundLocalError. --- mitogen/service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitogen/service.py b/mitogen/service.py index 217ed6ef..38f7ad9b 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -507,6 +507,8 @@ class Pool(object): raise mitogen.core.CallError('Invalid message format.') def _on_service_call(self, recv, msg): + service_name = None + method_name = None try: self._validate(msg) service_name, method_name, kwargs = msg.unpickle() From 3a8ea930d7df66bdac3193df7acc9b6700f1dff0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 7 Jul 2018 18:02:52 +0100 Subject: [PATCH 46/50] core: fix NameError in Latch.put(), FileService exception --- mitogen/core.py | 4 ++-- mitogen/service.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index b6936a40..53f32bb0 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -694,7 +694,7 @@ class Importer(object): suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): - _v and LOG.debug('%r: master doesn\'t know %r', self, fullname) + LOG.debug('%r: master doesn\'t know %r', self, fullname) return None # #114: explicitly whitelisted prefixes override any @@ -1396,7 +1396,7 @@ class Latch(object): wsock, cookie = self._sleeping[self._waking] self._waking += 1 _vv and IOLOG.debug('%r.put() -> waking wfd=%r', - self, sock.fileno()) + self, wsock.fileno()) self._wake(wsock, cookie) finally: self._lock.release() diff --git a/mitogen/service.py b/mitogen/service.py index 38f7ad9b..8af02d0e 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -778,7 +778,7 @@ class FileService(Service): st = os.stat(path) if not stat.S_ISREG(st.st_mode): - raise IOError('%r is not a regular file.' % (in_path,)) + raise IOError('%r is not a regular file.' % (path,)) LOG.debug('%r: registering %r', self, path) self._metadata_by_path[path] = { @@ -920,7 +920,7 @@ class FileService(Service): :param mitogen.core.Context context: Reference to the context hosting the FileService that will be used to fetch the file. - :param bytes in_path: + :param bytes path: FileService registered name of the input file. :param bytes out_path: Name of the output path on the local disk. From 3b7a1d4f36ff84b385f2dd1fff6769fb092c8eb4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 8 Jul 2018 16:06:08 +0100 Subject: [PATCH 47/50] ansible: fix 3.x dependency scanning on OS X On OS X with case-insensitive filenames, resolving 'ansible.module_utils.facts.base.Hardware' finds 'ansible.module_utils.facts.hardware/__init__.py', because module_finder's procedure is completely wrong for resolving child modules. Patch over it for now since it otherwise works for Ansible. --- ansible_mitogen/module_finder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index b9b62b73..56e8b82e 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -91,18 +91,22 @@ def find(name, path=(), parent=None): return parent fp, modpath, (suffix, mode, kind) = tup + if fp: + fp.close() + if parent and modpath == parent.path: # 'from timeout import timeout', where 'timeout' is a function but also # the name of the module being imported. return None - if fp: - fp.close() - if kind == imp.PKG_DIRECTORY: modpath = os.path.join(modpath, '__init__.py') + module = Module(head, modpath, kind, parent) - if tail: + # TODO: this code is entirely wrong on Python 3.x, but works well enough + # for Ansible. We need a new find_child() that only looks in the package + # directory, never falling back to the parent search path. + if tail and kind == imp.PKG_DIRECTORY: return find_relative(module, tail, path) return module From 038f8d5decc842fbd37ea6dda8d6b9fcb0a70944 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 8 Jul 2018 21:45:50 +0100 Subject: [PATCH 48/50] master: fix another case where built-in module loader throws ImportError requests/packages.py just imports urllib3 normally, then makes up new names for it. pkgutil can't cope with that, and returns the loader (builtin) for the requests package. The built-in loader obviously can't find_module() for "requests/packages/urllib3/contrib/pyopenssl" because it doesn't exist on disk. --- mitogen/master.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mitogen/master.py b/mitogen/master.py index 04464a5d..d057f7f1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -350,7 +350,17 @@ class ModuleFinder(object): path = self._py_filename(loader.get_filename(fullname)) source = loader.get_source(fullname) is_pkg = loader.is_package(fullname) - except AttributeError: + except (AttributeError, ImportError): + # - Per PEP-302, get_source() and is_package() are optional, + # calling them may throw AttributeError. + # - get_filename() may throw ImportError if pkgutil.find_loader() + # picks a "parent" package's loader for some crap that's been + # stuffed in sys.modules, for example in the case of urllib3: + # "loader for urllib3.contrib.pyopenssl cannot handle + # requests.packages.urllib3.contrib.pyopenssl" + e = sys.exc_info()[1] + LOG.debug('%r: loading %r using %r failed: %s', + self, fullname, loader) return if path is None or source is None: From 692275064b871ca759ee0539e2309a7549d8721e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 8 Jul 2018 21:56:13 +0100 Subject: [PATCH 49/50] parent: fix TtyLogger str/unicode crash on 3.x. --- mitogen/parent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index fb63612a..36d107f5 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -694,7 +694,7 @@ class TtyLogStream(mitogen.core.BasicStream): if not buf: return self.on_disconnect(broker) - self.buf += buf + self.buf += buf.decode('utf-8', 'replace') while '\n' in self.buf: lines = self.buf.split('\n') self.buf = lines[-1] From 0461738ca69e30ef561191517e67ed788b4015c3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 8 Jul 2018 22:00:00 +0100 Subject: [PATCH 50/50] docs: small reference fixes. --- docs/ansible.rst | 4 ++-- docs/internals.rst | 2 +- docs/services.rst | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 3a33594a..26592755 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -5,7 +5,6 @@ Mitogen for Ansible =================== - An extension to `Ansible`_ is included that implements connections over Mitogen, replacing embedded shell invocations with pure-Python equivalents invoked via highly efficient remote procedure calls to persistent interpreters @@ -19,6 +18,7 @@ will ensure soundness. .. _Bug reports: https://goo.gl/yLKZiJ + Overview -------- @@ -71,7 +71,7 @@ Installation 5. If targets have a restrictive ``sudoers`` file, add a rule like: - .. code-block:: plain +:: deploy = (ALL) NOPASSWD:/usr/bin/python -c* diff --git a/docs/internals.rst b/docs/internals.rst index 63030215..3d4d4130 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -407,7 +407,7 @@ Helper Functions List of submodule name suffixes. -.. currentmodule:: mitogen.parent +.. currentmodule:: mitogen.minify .. autofunction:: minimize_source (source) diff --git a/docs/services.rst b/docs/services.rst index 49108e80..44c49d96 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -119,10 +119,11 @@ Reference .. autofunction:: mitogen.service.Service -.. autoclass:: mitogen.service.Service - :members: +.. autoclass:: mitogen.service.Invoker +.. autoclass:: mitogen.service.SerializedInvoker +.. autoclass:: mitogen.service.DeduplicatingInvoker -.. autoclass:: mitogen.service.DeduplicatingService +.. autoclass:: mitogen.service.Service :members: .. autoclass:: mitogen.service.Pool