diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 98e45ab8..4df2dc70 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -3,6 +3,7 @@ import glob import os +import signal import sys import ci_lib @@ -13,11 +14,23 @@ TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') +def pause_if_interactive(): + if os.path.exists('/tmp/interactive'): + while True: + signal.pause() + + +interesting = ci_lib.get_interesting_procs() + + with ci_lib.Fold('unit_tests'): os.environ['SKIP_MITOGEN'] = '1' ci_lib.run('./run_tests -v') +ci_lib.check_stray_processes(interesting) + + with ci_lib.Fold('docker_setup'): containers = ci_lib.make_containers() ci_lib.start_containers(containers) @@ -56,8 +69,19 @@ with ci_lib.Fold('job_setup'): run("sudo apt-get update") run("sudo apt-get install -y sshpass") + run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'") + run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'") with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') - run('./run_ansible_playbook.py %s -i "%s" %s', - playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) + try: + run('./run_ansible_playbook.py %s -i "%s" %s', + playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) + except: + pause_if_interactive() + raise + + +ci_lib.check_stray_processes(interesting, containers) + +pause_if_interactive() diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index a377d795..e880eded 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -5,16 +5,27 @@ parameters: sign: false steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' +- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py" + displayName: "Run prep_azure.py" -- script: .ci/prep_azure.py - displayName: "Install requirements." +# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage, +# broken symlinks, incorrect permissions and missing codecs. So we use the +# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our +# stuff into. The virtualenv can probably be removed again, but this was a +# hard-fought battle and for now I am tired of this crap. +- script: | + sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python + /usr/bin/python -m pip install -U virtualenv setuptools wheel + /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version) + echo "##vso[task.prependpath]/tmp/venv/bin" + + displayName: activate venv + +- script: .ci/spawn_reverse_shell.py + displayName: "Spawn reverse shell" - script: .ci/$(MODE)_install.py - displayName: "Install requirements." + displayName: "Run $(MODE)_install.py" - script: .ci/$(MODE)_tests.py - displayName: Run tests. + displayName: "Run $(MODE)_tests.py" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index dc5f7162..920e82a1 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -15,6 +15,9 @@ jobs: Mito27_27: python.version: '2.7' MODE: mitogen + Ans280_27: + python.version: '2.7' + MODE: localhost_ansible - job: Linux @@ -87,3 +90,13 @@ jobs: #VER: 2.6.2 #DISTROS: debian #STRATEGY: linear + + Ansible_280_27: + python.version: '2.7' + MODE: ansible + VER: 2.8.0 + + Ansible_280_35: + python.version: '3.5' + MODE: ansible + VER: 2.8.0 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index e1cb84d5..dc7a02a8 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -57,8 +57,10 @@ def have_docker(): # ----------------- -# Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. +# Force line buffering on stdout. +sys.stdout = os.fdopen(1, 'w', 1) +# Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. if 'TRAVIS_HOME' in os.environ: proc = subprocess.Popen( args=['stdbuf', '-oL', 'cat'], @@ -86,8 +88,13 @@ def _argv(s, *args): def run(s, *args, **kwargs): argv = ['/usr/bin/time', '--'] + _argv(s, *args) print('Running: %s' % (argv,)) - ret = subprocess.check_call(argv, **kwargs) - print('Finished running: %s' % (argv,)) + try: + ret = subprocess.check_call(argv, **kwargs) + print('Finished running: %s' % (argv,)) + except Exception: + print('Exception occurred while running: %s' % (argv,)) + raise + return ret @@ -208,6 +215,46 @@ def make_containers(name_prefix='', port_offset=0): return lst +# ssh removed from here because 'linear' strategy relies on processes that hang +# around after the Ansible run completes +INTERESTING_COMMS = ('python', 'sudo', 'su', 'doas') + + +def proc_is_docker(pid): + try: + fp = open('/proc/%s/cgroup' % (pid,), 'r') + except IOError: + return False + + try: + return 'docker' in fp.read() + finally: + fp.close() + + +def get_interesting_procs(container_name=None): + args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand='] + if container_name is not None: + args = ['docker', 'exec', container_name] + args + + out = [] + for line in subprocess__check_output(args).decode().splitlines(): + ppid, pid, comm, rest = line.split(None, 3) + if ( + ( + any(comm.startswith(s) for s in INTERESTING_COMMS) or + 'mitogen:' in rest + ) and + ( + container_name is not None or + (not proc_is_docker(pid)) + ) + ): + out.append((int(pid), line)) + + return sorted(out) + + def start_containers(containers): if os.environ.get('KEEP'): return @@ -217,6 +264,7 @@ def start_containers(containers): "docker rm -f %(name)s || true" % container, "docker run " "--rm " + "--cpuset-cpus 0,1 " "--detach " "--privileged " "--cap-add=SYS_PTRACE " @@ -228,9 +276,44 @@ def start_containers(containers): ] for container in containers ]) + + for container in containers: + container['interesting'] = get_interesting_procs(container['name']) + return containers +def verify_procs(hostname, old, new): + oldpids = set(pid for pid, _ in old) + if any(pid not in oldpids for pid, _ in new): + print('%r had stray processes running:' % (hostname,)) + for pid, line in new: + if pid not in oldpids: + print('New process:', line) + + print() + return False + + return True + + +def check_stray_processes(old, containers=None): + ok = True + + new = get_interesting_procs() + if old is not None: + ok &= verify_procs('test host machine', old, new) + + for container in containers or (): + ok &= verify_procs( + container['name'], + container['interesting'], + get_interesting_procs(container['name']) + ) + + assert ok, 'stray processes were found' + + def dump_file(path): print() print('--- %s ---' % (path,)) diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index b0e2e4e8..e8f2907b 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -3,6 +3,7 @@ from __future__ import print_function import os import shutil +import sys import ci_lib @@ -67,9 +68,15 @@ with ci_lib.Fold('job_setup'): os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' +interesting = ci_lib.get_interesting_procs() + with ci_lib.Fold('first_run'): - ci_lib.run('debops common') + ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) + +ci_lib.check_stray_processes(interesting, containers) with ci_lib.Fold('second_run'): - ci_lib.run('debops common') + ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) + +ci_lib.check_stray_processes(interesting, containers) diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py new file mode 100755 index 00000000..0cb47374 --- /dev/null +++ b/.ci/localhost_ansible_install.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import ci_lib + +batches = [ + [ + # Must be installed separately, as PyNACL indirect requirement causes + # newer version to be installed if done in a single pip run. + 'pip install "pycparser<2.19" "idna<2.7"', + 'pip install ' + '-r tests/requirements.txt ' + '-r tests/ansible/requirements.txt', + ] +] + +ci_lib.run_batches(batches) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py new file mode 100755 index 00000000..f7e1ecbd --- /dev/null +++ b/.ci/localhost_ansible_tests.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen + +import glob +import os +import shutil +import sys + +import ci_lib +from ci_lib import run + + +TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') +IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') +HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') +KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key') + + +with ci_lib.Fold('unit_tests'): + os.environ['SKIP_MITOGEN'] = '1' + ci_lib.run('./run_tests -v') + + +with ci_lib.Fold('job_setup'): + # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. + run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION) + + os.chmod(KEY_PATH, int('0600', 8)) + if not ci_lib.exists_in_path('sshpass'): + run("brew install http://git.io/sshpass.rb") + + +with ci_lib.Fold('machine_prep'): + ssh_dir = os.path.expanduser('~/.ssh') + if not os.path.exists(ssh_dir): + os.makedirs(ssh_dir, int('0700', 8)) + + key_path = os.path.expanduser('~/.ssh/id_rsa') + shutil.copy(KEY_PATH, key_path) + + auth_path = os.path.expanduser('~/.ssh/authorized_keys') + os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path)) + os.chmod(auth_path, int('0600', 8)) + + if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': + os.chdir(IMAGE_PREP_DIR) + run("ansible-playbook -c local -i localhost, _user_accounts.yml") + + +with ci_lib.Fold('ansible'): + os.chdir(TESTS_DIR) + playbook = os.environ.get('PLAYBOOK', 'all.yml') + run('./run_ansible_playbook.py %s -l target %s', + playbook, ' '.join(sys.argv[1:])) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 72bc75e3..b8862f89 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -14,4 +14,5 @@ if ci_lib.have_docker(): 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ]) + ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 36928ac9..4de94b4c 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -14,4 +14,6 @@ os.environ.update({ if not ci_lib.have_docker(): os.environ['SKIP_DOCKER_TESTS'] = '1' +interesting = ci_lib.get_interesting_procs() ci_lib.run('./run_tests -v') +ci_lib.check_stray_processes(interesting) diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 5199a87e..344564e8 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -7,19 +7,43 @@ import ci_lib batches = [] +if 0 and os.uname()[0] == 'Linux': + batches += [ + [ + "sudo chown `whoami`: ~", + "chmod u=rwx,g=rx,o= ~", + + "sudo mkdir /var/run/sshd", + "sudo /etc/init.d/ssh start", + + "mkdir -p ~/.ssh", + "chmod u=rwx,go= ~/.ssh", + + "ssh-keyscan -H localhost >> ~/.ssh/known_hosts", + "chmod u=rw,go= ~/.ssh/known_hosts", + + "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa", + "chmod u=rw,go= ~/.ssh/id_rsa", + + "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys", + "chmod u=rw,go=r ~/.ssh/authorized_keys", + ] + ] + if ci_lib.have_apt(): batches.append([ 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', 'sudo add-apt-repository ppa:deadsnakes/ppa', 'sudo apt-get update', - 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', + 'sudo apt-get -y install ' + 'python{pv} ' + 'python{pv}-dev ' + 'libsasl2-dev ' + 'libldap2-dev ' + .format(pv=os.environ['PYTHONVERSION']) ]) -#batches.append([ - #'pip install -r dev_requirements.txt', -#]) - if ci_lib.have_docker(): batches.extend( ['docker pull %s' % (ci_lib.image_for_distro(distro),)] diff --git a/.ci/spawn_reverse_shell.py b/.ci/spawn_reverse_shell.py new file mode 100755 index 00000000..8a6b9500 --- /dev/null +++ b/.ci/spawn_reverse_shell.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +""" +Allow poking around Azure while the job is running. +""" + +import os +import pty +import socket +import subprocess +import sys +import time + + +if os.fork(): + sys.exit(0) + + +def try_once(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(("k3.botanicus.net", 9494)) + open('/tmp/interactive', 'w').close() + + os.dup2(s.fileno(), 0) + os.dup2(s.fileno(), 1) + os.dup2(s.fileno(), 2) + p = pty.spawn("/bin/sh") + + +while True: + try: + try_once() + except: + time.sleep(5) + continue + diff --git a/.gitignore b/.gitignore index e244ca12..aa75f691 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,11 @@ venvs/** MANIFEST build/ dist/ +extra/ +tests/ansible/.*.pid docs/_build/ htmlcov/ *.egg-info __pycache__/ +extra +**/.*.pid diff --git a/.lgtm.yml b/.lgtm.yml index 3e45b21e..a8e91c02 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -1,3 +1,10 @@ path_classifiers: - thirdparty: - - "mitogen/compat/*.py" + library: + - "mitogen/compat" + - "ansible_mitogen/compat" +queries: + # Mitogen 2.4 compatibility trips this query everywhere, so just disable it + - exclude: py/unreachable-statement + - exclude: py/should-use-with + # mitogen.core.b() trips this query everywhere, so just disable it + - exclude: py/import-and-import-from diff --git a/.travis.yml b/.travis.yml index 921ad12b..580ced0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: required +dist: trusty notifications: email: false @@ -6,15 +7,21 @@ notifications: language: python +branches: + except: + - docs-master + cache: - pip - directories: - /home/travis/virtualenv install: +- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v - .ci/${MODE}_install.py script: +- .ci/spawn_reverse_shell.py - .ci/${MODE}_tests.py @@ -22,49 +29,56 @@ script: # newest->oldest in various configuartions. matrix: - include: - # Mitogen tests. - # 2.4 -> 2.4 + allow_failures: + # Python 2.4 tests are still unreliable - language: c env: MODE=mitogen_py24 DISTRO=centos5 - # 2.7 -> 2.7 -- moved to Azure - # 2.7 -> 2.6 - #- python: "2.7" - #env: MODE=mitogen DISTRO=centos6 - # 2.6 -> 2.7 - - python: "2.6" - env: MODE=mitogen DISTRO=centos7 - # 2.6 -> 3.5 - - python: "2.6" - env: MODE=mitogen DISTRO=debian-py3 - # 3.6 -> 2.6 -- moved to Azure + include: # Debops tests. + # 2.8.3; 3.6 -> 2.7 + - python: "3.6" + env: MODE=debops_common VER=2.8.3 # 2.4.6.0; 2.7 -> 2.7 - python: "2.7" env: MODE=debops_common VER=2.4.6.0 - # 2.5.7; 3.6 -> 2.7 - - python: "3.6" - env: MODE=debops_common VER=2.6.2 + + # Sanity check against vanilla Ansible. One job suffices. + - python: "2.7" + env: MODE=ansible VER=2.8.3 DISTROS=debian STRATEGY=linear # ansible_mitogen tests. + # 2.8.3 -> {debian, centos6, centos7} + - python: "3.6" + env: MODE=ansible VER=2.8.3 + # 2.8.3 -> {debian, centos6, centos7} + - python: "2.7" + env: MODE=ansible VER=2.8.3 + + # 2.4.6.0 -> {debian, centos6, centos7} + - python: "3.6" + env: MODE=ansible VER=2.4.6.0 + # 2.4.6.0 -> {debian, centos6, centos7} + - python: "2.6" + env: MODE=ansible VER=2.4.6.0 + # 2.3 -> {centos5} - python: "2.6" env: MODE=ansible VER=2.3.3.0 DISTROS=centos5 - # 2.6 -> {debian, centos6, centos7} + # Mitogen tests. + # 2.4 -> 2.4 + - language: c + env: MODE=mitogen_py24 DISTRO=centos5 + # 2.7 -> 2.7 -- moved to Azure + # 2.7 -> 2.6 + #- python: "2.7" + #env: MODE=mitogen DISTRO=centos6 + # 2.6 -> 2.7 - python: "2.6" - env: MODE=ansible VER=2.4.6.0 + env: MODE=mitogen DISTRO=centos7 + # 2.6 -> 3.5 - python: "2.6" - env: MODE=ansible VER=2.6.2 - - # 3.6 -> {debian, centos6, centos7} - - python: "3.6" - env: MODE=ansible VER=2.4.6.0 - - python: "3.6" - env: MODE=ansible VER=2.6.2 - - # Sanity check against vanilla Ansible. One job suffices. - - python: "2.7" - env: MODE=ansible VER=2.6.2 DISTROS=debian STRATEGY=linear + env: MODE=mitogen DISTRO=debian-py3 + # 3.6 -> 2.6 -- moved to Azure diff --git a/README.md b/README.md index 5ef2447f..da93a80b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Mitogen -Please see the documentation. +Please see the documentation. ![](https://i.imgur.com/eBM6LhJ.gif) diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index 09a6acee..67e16d8a 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -73,7 +73,9 @@ necessarily involves preventing the scheduler from making load balancing decisions. """ +from __future__ import absolute_import import ctypes +import logging import mmap import multiprocessing import os @@ -83,41 +85,44 @@ import mitogen.core import mitogen.parent +LOG = logging.getLogger(__name__) + + try: _libc = ctypes.CDLL(None, use_errno=True) _strerror = _libc.strerror _strerror.restype = ctypes.c_char_p - _pthread_mutex_init = _libc.pthread_mutex_init - _pthread_mutex_lock = _libc.pthread_mutex_lock - _pthread_mutex_unlock = _libc.pthread_mutex_unlock + _sem_init = _libc.sem_init + _sem_wait = _libc.sem_wait + _sem_post = _libc.sem_post _sched_setaffinity = _libc.sched_setaffinity except (OSError, AttributeError): _libc = None _strerror = None - _pthread_mutex_init = None - _pthread_mutex_lock = None - _pthread_mutex_unlock = None + _sem_init = None + _sem_wait = None + _sem_post = None _sched_setaffinity = None -class pthread_mutex_t(ctypes.Structure): +class sem_t(ctypes.Structure): """ - Wrap pthread_mutex_t to allow storing a lock in shared memory. + Wrap sem_t to allow storing a lock in shared memory. """ _fields_ = [ - ('data', ctypes.c_uint8 * 512), + ('data', ctypes.c_uint8 * 128), ] def init(self): - if _pthread_mutex_init(self.data, 0): + if _sem_init(self.data, 1, 1): raise Exception(_strerror(ctypes.get_errno())) def acquire(self): - if _pthread_mutex_lock(self.data): + if _sem_wait(self.data): raise Exception(_strerror(ctypes.get_errno())) def release(self): - if _pthread_mutex_unlock(self.data): + if _sem_post(self.data): raise Exception(_strerror(ctypes.get_errno())) @@ -128,7 +133,7 @@ class State(ctypes.Structure): the context of the new child process. """ _fields_ = [ - ('lock', pthread_mutex_t), + ('lock', sem_t), ('counter', ctypes.c_uint8), ] @@ -142,7 +147,7 @@ class Policy(object): Assign the Ansible top-level policy to this process. """ - def assign_muxprocess(self): + def assign_muxprocess(self, index): """ Assign the MuxProcess policy to this process. """ @@ -177,9 +182,9 @@ class FixedPolicy(Policy): cores, before reusing the second hyperthread of an existing core. A hook is installed that causes :meth:`reset` to run in the child of any - process created with :func:`mitogen.parent.detach_popen`, ensuring - CPU-intensive children like SSH are not forced to share the same core as - the (otherwise potentially very busy) parent. + process created with :func:`mitogen.parent.popen`, ensuring CPU-intensive + children like SSH are not forced to share the same core as the (otherwise + potentially very busy) parent. """ def __init__(self, cpu_count=None): #: For tests. @@ -207,11 +212,13 @@ class FixedPolicy(Policy): self._reserve_mask = 3 self._reserve_shift = 2 - def _set_affinity(self, mask): + def _set_affinity(self, descr, mask): + if descr: + LOG.debug('CPU mask for %s: %#08x', descr, mask) mitogen.parent._preexec_hook = self._clear self._set_cpu_mask(mask) - def _balance(self): + def _balance(self, descr): self.state.lock.acquire() try: n = self.state.counter @@ -219,28 +226,28 @@ class FixedPolicy(Policy): finally: self.state.lock.release() - self._set_cpu(self._reserve_shift + ( + self._set_cpu(descr, self._reserve_shift + ( (n % (self.cpu_count - self._reserve_shift)) )) - def _set_cpu(self, cpu): - self._set_affinity(1 << cpu) + def _set_cpu(self, descr, cpu): + self._set_affinity(descr, 1 << (cpu % self.cpu_count)) def _clear(self): all_cpus = (1 << self.cpu_count) - 1 - self._set_affinity(all_cpus & ~self._reserve_mask) + self._set_affinity(None, all_cpus & ~self._reserve_mask) def assign_controller(self): if self._reserve_controller: - self._set_cpu(1) + self._set_cpu('Ansible top-level process', 1) else: - self._balance() + self._balance('Ansible top-level process') - def assign_muxprocess(self): - self._set_cpu(0) + def assign_muxprocess(self, index): + self._set_cpu('MuxProcess %d' % (index,), index) def assign_worker(self): - self._balance() + self._balance('WorkerProcess') def assign_subprocess(self): self._clear() diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index b5f28d34..2dd3bfa9 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -37,7 +37,6 @@ import stat import sys import time -import jinja2.runtime import ansible.constants as C import ansible.errors import ansible.plugins.connection @@ -45,9 +44,9 @@ import ansible.utils.shlex import mitogen.core import mitogen.fork -import mitogen.unix import mitogen.utils +import ansible_mitogen.mixins import ansible_mitogen.parsing import ansible_mitogen.process import ansible_mitogen.services @@ -145,9 +144,29 @@ def _connect_ssh(spec): 'ssh_args': spec.ssh_args(), 'ssh_debug_level': spec.mitogen_ssh_debug_level(), 'remote_name': get_remote_name(spec), + 'keepalive_count': ( + spec.mitogen_ssh_keepalive_count() or 10 + ), + 'keepalive_interval': ( + spec.mitogen_ssh_keepalive_interval() or 30 + ), } } +def _connect_buildah(spec): + """ + Return ContextService arguments for a Buildah connection. + """ + return { + 'method': 'buildah', + 'kwargs': { + 'username': spec.remote_user(), + 'container': spec.remote_addr(), + 'python_path': spec.python_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } def _connect_docker(spec): """ @@ -356,7 +375,7 @@ def _connect_mitogen_doas(spec): 'username': spec.remote_user(), 'password': spec.password(), 'python_path': spec.python_path(), - 'doas_path': spec.become_exe(), + 'doas_path': spec.ansible_doas_exe(), 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } @@ -367,6 +386,7 @@ def _connect_mitogen_doas(spec): #: generating ContextService keyword arguments matching a connection #: specification. CONNECTION_METHOD = { + 'buildah': _connect_buildah, 'docker': _connect_docker, 'kubectl': _connect_kubectl, 'jail': _connect_jail, @@ -386,15 +406,6 @@ CONNECTION_METHOD = { } -class Broker(mitogen.master.Broker): - """ - WorkerProcess maintains at most 2 file descriptors, therefore does not need - the exuberant syscall expense of EpollPoller, so override it and restore - the poll() poller. - """ - poller_class = mitogen.core.Poller - - class CallChain(mitogen.parent.CallChain): """ Extend :class:`mitogen.parent.CallChain` to additionally cause the @@ -438,15 +449,10 @@ class CallChain(mitogen.parent.CallChain): class Connection(ansible.plugins.connection.ConnectionBase): - #: mitogen.master.Broker for this worker. - broker = None - - #: mitogen.master.Router for this worker. - router = None - - #: mitogen.parent.Context representing the parent Context, which is - #: presently always the connection multiplexer process. - parent = None + #: The :class:`ansible_mitogen.process.Binding` representing the connection + #: multiplexer this connection's target is assigned to. :data:`None` when + #: disconnected. + binding = None #: mitogen.parent.Context for the target account on the target, possibly #: reached via become. @@ -497,13 +503,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: matching vanilla Ansible behaviour. loader_basedir = None - def __init__(self, play_context, new_stdin, **kwargs): - assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( - 'Mitogen connection types may only be instantiated ' - 'while the "mitogen" strategy is active.' - ) - super(Connection, self).__init__(play_context, new_stdin) - def __del__(self): """ Ansible cannot be trusted to always call close() e.g. the synchronize @@ -535,6 +534,47 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.loader_basedir = loader_basedir self._mitogen_reset(mode='put') + def _get_task_vars(self): + """ + More information is needed than normally provided to an Ansible + connection. For proxied connections, intermediary configuration must + be inferred, and for any connection the configured Python interpreter + must be known. + + There is no clean way to access this information that would not deviate + from the running Ansible version. The least invasive method known is to + reuse the running task's task_vars dict. + + This method walks the stack to find task_vars of the Action plugin's + run(), or if no Action is present, from Strategy's _execute_meta(), as + in the case of 'meta: reset_connection'. The stack is walked in + addition to subclassing Action.run()/on_action_run(), as it is possible + for new connections to be constructed in addition to the preconstructed + connection passed into any running action. + """ + f = sys._getframe() + + while f: + if f.f_code.co_name == 'run': + f_locals = f.f_locals + f_self = f_locals.get('self') + if isinstance(f_self, ansible_mitogen.mixins.ActionModuleMixin): + task_vars = f_locals.get('task_vars') + if task_vars: + LOG.debug('recovered task_vars from Action') + return task_vars + elif f.f_code.co_name == '_execute_meta': + f_all_vars = f.f_locals.get('all_vars') + if isinstance(f_all_vars, dict): + LOG.debug('recovered task_vars from meta:') + return f_all_vars + + f = f.f_back + + LOG.warning('could not recover task_vars. This means some connection ' + 'settings may erroneously be reset to their defaults. ' + 'Please report a bug if you encounter this message.') + def get_task_var(self, key, default=None): """ Fetch the value of a task variable related to connection configuration, @@ -546,12 +586,13 @@ class Connection(ansible.plugins.connection.ConnectionBase): does not make sense to extract connection-related configuration for the delegated-to machine from them. """ - if self._task_vars: + task_vars = self._task_vars or self._get_task_vars() + if task_vars is not None: if self.delegate_to_hostname is None: - if key in self._task_vars: - return self._task_vars[key] + if key in task_vars: + return task_vars[key] else: - delegated_vars = self._task_vars['ansible_delegated_vars'] + delegated_vars = task_vars['ansible_delegated_vars'] if self.delegate_to_hostname in delegated_vars: task_vars = delegated_vars[self.delegate_to_hostname] if key in task_vars: @@ -564,6 +605,15 @@ class Connection(ansible.plugins.connection.ConnectionBase): self._connect() return self.init_child_result['home_dir'] + def get_binding(self): + """ + Return the :class:`ansible_mitogen.process.Binding` representing the + process that hosts the physical connection and services (context + establishment, file transfer, ..) for our desired target. + """ + assert self.binding is not None + return self.binding + @property def connected(self): return self.context is not None @@ -651,18 +701,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): return stack - def _connect_broker(self): - """ - Establish a reference to the Broker, Router and parent context used for - connections. - """ - if not self.broker: - self.broker = mitogen.master.Broker() - self.router, self.parent = mitogen.unix.connect( - path=ansible_mitogen.process.MuxProcess.unix_listener_path, - broker=self.broker, - ) - def _build_stack(self): """ Construct a list of dictionaries representing the connection @@ -670,14 +708,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): additionally used by the integration tests "mitogen_get_stack" action to fetch the would-be connection configuration. """ - return self._stack_from_spec( - ansible_mitogen.transport_config.PlayContextSpec( - connection=self, - play_context=self._play_context, - transport=self.transport, - inventory_name=self.inventory_hostname, - ) + spec = ansible_mitogen.transport_config.PlayContextSpec( + connection=self, + play_context=self._play_context, + transport=self.transport, + inventory_name=self.inventory_hostname, ) + stack = self._stack_from_spec(spec) + return spec.inventory_name(), stack def _connect_stack(self, stack): """ @@ -690,7 +728,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): description of the returned dictionary. """ try: - dct = self.parent.call_service( + dct = mitogen.service.call( + call_context=self.binding.get_service_context(), service_name='ansible_mitogen.services.ContextService', method_name='get', stack=mitogen.utils.cast(list(stack)), @@ -737,8 +776,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): if self.connected: return - self._connect_broker() - stack = self._build_stack() + inventory_name, stack = self._build_stack() + worker_model = ansible_mitogen.process.get_worker_model() + self.binding = worker_model.get_binding(inventory_name) self._connect_stack(stack) def _mitogen_reset(self, mode): @@ -755,7 +795,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): return self.chain.reset() - self.parent.call_service( + mitogen.service.call( + call_context=self.binding.get_service_context(), service_name='ansible_mitogen.services.ContextService', method_name=mode, context=self.context @@ -766,27 +807,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.init_child_result = None self.chain = None - def _shutdown_broker(self): - """ - Shutdown the broker thread during :meth:`close` or :meth:`reset`. - """ - if self.broker: - self.broker.shutdown() - self.broker.join() - self.broker = None - self.router = None - - # #420: Ansible executes "meta" actions in the top-level process, - # meaning "reset_connection" will cause :class:`mitogen.core.Latch` - # FDs to be cached and erroneously shared by children on subsequent - # WorkerProcess forks. To handle that, call on_fork() to ensure any - # shared state is discarded. - # #490: only attempt to clean up when it's known that some - # resources exist to cleanup, otherwise later __del__ double-call - # to close() due to GC at random moment may obliterate an unrelated - # Connection's resources. - mitogen.fork.on_fork() - def close(self): """ Arrange for the mitogen.master.Router running in the worker to @@ -794,7 +814,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): multiple times. """ self._mitogen_reset(mode='put') - self._shutdown_broker() + if self.binding: + self.binding.close() + self.binding = None def _reset_find_task_vars(self): """ @@ -832,7 +854,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): self._connect() self._mitogen_reset(mode='reset') - self._shutdown_broker() + self.binding.close() + self.binding = None # Compatibility with Ansible 2.4 wait_for_connection plug-in. _reset = reset @@ -927,11 +950,13 @@ class Connection(ansible.plugins.connection.ConnectionBase): :param str out_path: Local filesystem path to write. """ - output = self.get_chain().call( - ansible_mitogen.target.read_path, - mitogen.utils.cast(in_path), + self._connect() + ansible_mitogen.target.transfer_file( + context=self.context, + # in_path may be AnsibleUnicode + in_path=mitogen.utils.cast(in_path), + out_path=out_path ) - ansible_mitogen.target.write_path(out_path, output) def put_data(self, out_path, data, mode=None, utimes=None): """ @@ -1003,7 +1028,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): utimes=(st.st_atime, st.st_mtime)) self._connect() - self.parent.call_service( + mitogen.service.call( + call_context=self.binding.get_service_context(), service_name='mitogen.service.FileService', method_name='register', path=mitogen.utils.cast(in_path) @@ -1015,7 +1041,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): # file alive, but that requires more work. self.get_chain().call( ansible_mitogen.target.transfer_file, - context=self.parent, + context=self.binding.get_child_service_context(), in_path=in_path, out_path=out_path ) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index ff06c0c5..99294c1f 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -32,6 +32,15 @@ Stable names for PluginLoader instances across Ansible versions. from __future__ import absolute_import +__all__ = [ + 'action_loader', + 'connection_loader', + 'module_loader', + 'module_utils_loader', + 'shell_loader', + 'strategy_loader', +] + try: from ansible.plugins.loader import action_loader from ansible.plugins.loader import connection_loader diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 890467fd..eee1ecd7 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -182,14 +182,6 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): ) ) - def _generate_tmp_path(self): - return os.path.join( - self._connection.get_good_temp_dir(), - 'ansible_mitogen_action_%016x' % ( - random.getrandbits(8*8), - ) - ) - def _make_tmp_path(self, remote_user=None): """ Create a temporary subdirectory as a child of the temporary directory @@ -368,11 +360,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): ) ) - if ansible.__version__ < '2.5' and delete_remote_tmp and \ - getattr(self._connection._shell, 'tmpdir', None) is not None: + if tmp and ansible.__version__ < '2.5' and delete_remote_tmp: # Built-in actions expected tmpdir to be cleaned up automatically # on _execute_module(). - self._remove_tmp_path(self._connection._shell.tmpdir) + self._remove_tmp_path(tmp) return result diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 633e3cad..89aa2beb 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -57,7 +57,7 @@ def get_code(module): """ Compile and return a Module's code object. """ - fp = open(module.path) + fp = open(module.path, 'rb') try: return compile(fp.read(), str(module.name), 'exec') finally: diff --git a/ansible_mitogen/parsing.py b/ansible_mitogen/parsing.py index 525e60cf..27fca7cd 100644 --- a/ansible_mitogen/parsing.py +++ b/ansible_mitogen/parsing.py @@ -26,14 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -""" -Classes to detect each case from [0] and prepare arguments necessary for the -corresponding Runner class within the target, including preloading requisite -files/modules known missing. - -[0] "Ansible Module Architecture", developing_program_flow_modules.html -""" - from __future__ import absolute_import from __future__ import unicode_literals diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 2eebd36d..96b06995 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -148,6 +148,8 @@ class Planner(object): # named by `runner_name`. } """ + binding = self._inv.connection.get_binding() + new = dict((mitogen.core.UnicodeType(k), kwargs[k]) for k in kwargs) new.setdefault('good_temp_dir', @@ -155,7 +157,7 @@ class Planner(object): new.setdefault('cwd', self._inv.connection.get_default_cwd()) new.setdefault('extra_env', self._inv.connection.get_default_env()) new.setdefault('emulate_tty', True) - new.setdefault('service_context', self._inv.connection.parent) + new.setdefault('service_context', binding.get_child_service_context()) return new def __repr__(self): @@ -328,7 +330,9 @@ class NewStylePlanner(ScriptPlanner): def get_module_map(self): if self._module_map is None: - self._module_map = self._inv.connection.parent.call_service( + binding = self._inv.connection.get_binding() + self._module_map = mitogen.service.call( + call_context=binding.get_service_context(), service_name='ansible_mitogen.services.ModuleDepService', method_name='scan', @@ -405,9 +409,12 @@ def get_module_data(name): def _propagate_deps(invocation, planner, context): - invocation.connection.parent.call_service( + binding = invocation.connection.get_binding() + mitogen.service.call( + call_context=binding.get_service_context(), service_name='mitogen.service.PushFileService', method_name='propagate_paths_and_modules', + context=context, paths=planner.get_push_files(), modules=planner.get_module_deps(), diff --git a/ansible_mitogen/plugins/action/mitogen_fetch.py b/ansible_mitogen/plugins/action/mitogen_fetch.py new file mode 100644 index 00000000..1844efd8 --- /dev/null +++ b/ansible_mitogen/plugins/action/mitogen_fetch.py @@ -0,0 +1,162 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from ansible.module_utils._text import to_bytes +from ansible.module_utils.six import string_types +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.action import ActionBase +from ansible.utils.hashing import checksum, md5, secure_hash +from ansible.utils.path import makedirs_safe + + +REMOTE_CHECKSUM_ERRORS = { + '0': "unable to calculate the checksum of the remote file", + '1': "the remote file does not exist", + '2': "no read permission on remote file", + '3': "remote file is a directory, fetch cannot work on directories", + '4': "python isn't present on the system. Unable to compute checksum", + '5': "stdlib json was not found on the remote machine. Only the raw module can work without those installed", +} + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + ''' handler for fetch operations ''' + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + try: + if self._play_context.check_mode: + result['skipped'] = True + result['msg'] = 'check mode not (yet) supported for this module' + return result + + flat = boolean(self._task.args.get('flat'), strict=False) + fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False) + validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False) + + # validate source and dest are strings FIXME: use basic.py and module specs + source = self._task.args.get('src') + if not isinstance(source, string_types): + result['msg'] = "Invalid type supplied for source option, it must be a string" + + dest = self._task.args.get('dest') + if not isinstance(dest, string_types): + result['msg'] = "Invalid type supplied for dest option, it must be a string" + + if result.get('msg'): + result['failed'] = True + return result + + source = self._connection._shell.join_path(source) + source = self._remote_expand_user(source) + + # calculate checksum for the remote file, don't bother if using + # become as slurp will be used Force remote_checksum to follow + # symlinks because fetch always follows symlinks + remote_checksum = self._remote_checksum(source, all_vars=task_vars, follow=True) + + # calculate the destination name + if os.path.sep not in self._connection._shell.join_path('a', ''): + source = self._connection._shell._unquote(source) + source_local = source.replace('\\', '/') + else: + source_local = source + + dest = os.path.expanduser(dest) + if flat: + if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep): + result['msg'] = "dest is an existing directory, use a trailing slash if you want to fetch src into that directory" + result['file'] = dest + result['failed'] = True + return result + if dest.endswith(os.sep): + # if the path ends with "/", we'll use the source filename as the + # destination filename + base = os.path.basename(source_local) + dest = os.path.join(dest, base) + if not dest.startswith("/"): + # if dest does not start with "/", we'll assume a relative path + dest = self._loader.path_dwim(dest) + else: + # files are saved in dest dir, with a subdir for each host, then the filename + if 'inventory_hostname' in task_vars: + target_name = task_vars['inventory_hostname'] + else: + target_name = self._play_context.remote_addr + dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local) + + dest = dest.replace("//", "/") + + if remote_checksum in REMOTE_CHECKSUM_ERRORS: + result['changed'] = False + result['file'] = source + result['msg'] = REMOTE_CHECKSUM_ERRORS[remote_checksum] + # Historically, these don't fail because you may want to transfer + # a log file that possibly MAY exist but keep going to fetch other + # log files. Today, this is better achieved by adding + # ignore_errors or failed_when to the task. Control the behaviour + # via fail_when_missing + if fail_on_missing: + result['failed'] = True + del result['changed'] + else: + result['msg'] += ", not transferring, ignored" + return result + + # calculate checksum for the local file + local_checksum = checksum(dest) + + if remote_checksum != local_checksum: + # create the containing directories, if needed + makedirs_safe(os.path.dirname(dest)) + + # fetch the file and check for changes + self._connection.fetch_file(source, dest) + new_checksum = secure_hash(dest) + # For backwards compatibility. We'll return None on FIPS enabled systems + try: + new_md5 = md5(dest) + except ValueError: + new_md5 = None + + if validate_checksum and new_checksum != remote_checksum: + result.update(dict(failed=True, md5sum=new_md5, + msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, + checksum=new_checksum, remote_checksum=remote_checksum)) + else: + result.update({'changed': True, 'md5sum': new_md5, 'dest': dest, + 'remote_md5sum': None, 'checksum': new_checksum, + 'remote_checksum': remote_checksum}) + else: + # For backwards compatibility. We'll return None on FIPS enabled systems + try: + local_md5 = md5(dest) + except ValueError: + local_md5 = None + result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)) + + finally: + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result diff --git a/ansible_mitogen/plugins/action/mitogen_get_stack.py b/ansible_mitogen/plugins/action/mitogen_get_stack.py index 12afbfba..171f84ea 100644 --- a/ansible_mitogen/plugins/action/mitogen_get_stack.py +++ b/ansible_mitogen/plugins/action/mitogen_get_stack.py @@ -47,8 +47,9 @@ class ActionModule(ActionBase): 'skipped': True, } + _, stack = self._connection._build_stack() return { 'changed': True, - 'result': self._connection._build_stack(), + 'result': stack, '_ansible_verbose_always': True, } diff --git a/ansible_mitogen/plugins/connection/mitogen_buildah.py b/ansible_mitogen/plugins/connection/mitogen_buildah.py new file mode 100644 index 00000000..017214b2 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_buildah.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'buildah' diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index 24b84a03..a98c834c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -81,6 +81,6 @@ class Connection(ansible_mitogen.connection.Connection): from WorkerProcess, we must emulate that. """ return dict_diff( - old=ansible_mitogen.process.MuxProcess.original_env, + old=ansible_mitogen.process.MuxProcess.cls_original_env, new=os.environ, ) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index e4e61e8b..1fc7bf80 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -28,22 +28,28 @@ from __future__ import absolute_import import atexit -import errno import logging +import multiprocessing import os -import signal +import resource import socket +import signal import sys -import time try: import faulthandler except ImportError: faulthandler = None +try: + import setproctitle +except ImportError: + setproctitle = None + import mitogen import mitogen.core import mitogen.debug +import mitogen.fork import mitogen.master import mitogen.parent import mitogen.service @@ -52,6 +58,7 @@ import mitogen.utils import ansible import ansible.constants as C +import ansible.errors import ansible_mitogen.logging import ansible_mitogen.services @@ -66,21 +73,68 @@ ANSIBLE_PKG_OVERRIDE = ( u"__author__ = %r\n" ) +MAX_MESSAGE_SIZE = 4096 * 1048576 + +worker_model_msg = ( + 'Mitogen connection types may only be instantiated when one of the ' + '"mitogen_*" or "operon_*" strategies are active.' +) + +shutting_down_msg = ( + 'The task worker cannot connect. Ansible may be shutting down, or ' + 'the maximum open files limit may have been exceeded. If this occurs ' + 'midway through a run, please retry after increasing the open file ' + 'limit (ulimit -n). Original error: %s' +) + + +#: The worker model as configured by the currently running strategy. This is +#: managed via :func:`get_worker_model` / :func:`set_worker_model` functions by +#: :class:`StrategyMixin`. +_worker_model = None + + +#: A copy of the sole :class:`ClassicWorkerModel` that ever exists during a +#: classic run, as return by :func:`get_classic_worker_model`. +_classic_worker_model = None + + +def set_worker_model(model): + """ + To remove process model-wiring from + :class:`ansible_mitogen.connection.Connection`, it is necessary to track + some idea of the configured execution environment outside the connection + plug-in. + + That is what :func:`set_worker_model` and :func:`get_worker_model` are for. + """ + global _worker_model + assert model is None or _worker_model is None + _worker_model = model + + +def get_worker_model(): + """ + Return the :class:`WorkerModel` currently configured by the running + strategy. + """ + if _worker_model is None: + raise ansible.errors.AnsibleConnectionFailure(worker_model_msg) + return _worker_model + -def clean_shutdown(sock): +def get_classic_worker_model(**kwargs): """ - Shut the write end of `sock`, causing `recv` in the worker process to wake - up with a 0-byte read and initiate mux process exit, then wait for a 0-byte - read from the read end, which will occur after the the child closes the - descriptor on exit. - - This is done using :mod:`atexit` since Ansible lacks any more sensible hook - to run code during exit, and unless some synchronization exists with - MuxProcess, debug logs may appear on the user's terminal *after* the prompt - has been printed. + Return the single :class:`ClassicWorkerModel` instance, constructing it if + necessary. """ - sock.shutdown(socket.SHUT_WR) - sock.recv(1) + global _classic_worker_model + assert _classic_worker_model is None or (not kwargs), \ + "ClassicWorkerModel kwargs supplied but model already constructed" + + if _classic_worker_model is None: + _classic_worker_model = ClassicWorkerModel(**kwargs) + return _classic_worker_model def getenv_int(key, default=0): @@ -112,13 +166,445 @@ def save_pid(name): fp.write(str(os.getpid())) +def setup_pool(pool): + """ + Configure a connection multiplexer's :class:`mitogen.service.Pool` with + services accessed by clients and WorkerProcesses. + """ + pool.add(mitogen.service.FileService(router=pool.router)) + pool.add(mitogen.service.PushFileService(router=pool.router)) + pool.add(ansible_mitogen.services.ContextService(router=pool.router)) + pool.add(ansible_mitogen.services.ModuleDepService(pool.router)) + LOG.debug('Service pool configured: size=%d', pool.size) + + +def _setup_simplejson(responder): + """ + We support serving simplejson for Python 2.4 targets on Ansible 2.3, at + least so the package's own CI Docker scripts can run without external + help, however newer versions of simplejson no longer support Python + 2.4. Therefore override any installed/loaded version with a + 2.4-compatible version we ship in the compat/ directory. + """ + responder.whitelist_prefix('simplejson') + + # issue #536: must be at end of sys.path, in case existing newer + # version is already loaded. + compat_path = os.path.join(os.path.dirname(__file__), 'compat') + sys.path.append(compat_path) + + for fullname, is_pkg, suffix in ( + (u'simplejson', True, '__init__.py'), + (u'simplejson.decoder', False, 'decoder.py'), + (u'simplejson.encoder', False, 'encoder.py'), + (u'simplejson.scanner', False, 'scanner.py'), + ): + path = os.path.join(compat_path, 'simplejson', suffix) + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + responder.add_source_override( + fullname=fullname, + path=path, + source=source, + is_pkg=is_pkg, + ) + + +def _setup_responder(responder): + """ + Configure :class:`mitogen.master.ModuleResponder` to only permit + certain packages, and to generate custom responses for certain modules. + """ + responder.whitelist_prefix('ansible') + responder.whitelist_prefix('ansible_mitogen') + _setup_simplejson(responder) + + # Ansible 2.3 is compatible with Python 2.4 targets, however + # ansible/__init__.py is not. Instead, executor/module_common.py writes + # out a 2.4-compatible namespace package for unknown reasons. So we + # copy it here. + responder.add_source_override( + fullname='ansible', + path=ansible.__file__, + source=(ANSIBLE_PKG_OVERRIDE % ( + ansible.__version__, + ansible.__author__, + )).encode(), + is_pkg=True, + ) + + +def increase_open_file_limit(): + """ + #549: in order to reduce the possibility of hitting an open files limit, + increase :data:`resource.RLIMIT_NOFILE` from its soft limit to its hard + limit, if they differ. + + It is common that a low soft limit is configured by default, where the hard + limit is much higher. + """ + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if hard == resource.RLIM_INFINITY: + hard_s = '(infinity)' + # cap in case of O(RLIMIT_NOFILE) algorithm in some subprocess. + hard = 524288 + else: + hard_s = str(hard) + + LOG.debug('inherited open file limits: soft=%d hard=%s', soft, hard_s) + if soft >= hard: + LOG.debug('max open files already set to hard limit: %d', hard) + return + + # OS X is limited by kern.maxfilesperproc sysctl, rather than the + # advertised unlimited hard RLIMIT_NOFILE. Just hard-wire known defaults + # for that sysctl, to avoid the mess of querying it. + for value in (hard, 10240): + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (value, hard)) + LOG.debug('raised soft open file limit from %d to %d', soft, value) + break + except ValueError as e: + LOG.debug('could not raise soft open file limit from %d to %d: %s', + soft, value, e) + + +def common_setup(enable_affinity=True, _init_logging=True): + save_pid('controller') + ansible_mitogen.logging.set_process_name('top') + + if _init_logging: + ansible_mitogen.logging.setup() + + if enable_affinity: + ansible_mitogen.affinity.policy.assign_controller() + + mitogen.utils.setup_gil() + if faulthandler is not None: + faulthandler.enable() + + MuxProcess.profiling = getenv_int('MITOGEN_PROFILING') > 0 + if MuxProcess.profiling: + mitogen.core.enable_profiling() + + MuxProcess.cls_original_env = dict(os.environ) + increase_open_file_limit() + + +def get_cpu_count(default=None): + """ + Get the multiplexer CPU count from the MITOGEN_CPU_COUNT environment + variable, returning `default` if one isn't set, or is out of range. + + :param int default: + Default CPU, or :data:`None` to use all available CPUs. + """ + max_cpus = multiprocessing.cpu_count() + if default is None: + default = max_cpus + + cpu_count = getenv_int('MITOGEN_CPU_COUNT', default=default) + if cpu_count < 1 or cpu_count > max_cpus: + cpu_count = default + + return cpu_count + + +class Broker(mitogen.master.Broker): + """ + WorkerProcess maintains at most 2 file descriptors, therefore does not need + the exuberant syscall expense of EpollPoller, so override it and restore + the poll() poller. + """ + poller_class = mitogen.core.Poller + + +class Binding(object): + """ + Represent a bound connection for a particular inventory hostname. When + operating in sharded mode, the actual MuxProcess implementing a connection + varies according to the target machine. Depending on the particular + implementation, this class represents a binding to the correct MuxProcess. + """ + def get_child_service_context(self): + """ + Return the :class:`mitogen.core.Context` to which children should + direct requests for services such as FileService, or :data:`None` for + the local process. + + This can be different from :meth:`get_service_context` where MuxProcess + and WorkerProcess are combined, and it is discovered a task is + delegated after being assigned to its initial worker for the original + un-delegated hostname. In that case, connection management and + expensive services like file transfer must be implemented by the + MuxProcess connected to the target, rather than routed to the + MuxProcess responsible for executing the task. + """ + raise NotImplementedError() + + def get_service_context(self): + """ + Return the :class:`mitogen.core.Context` to which this process should + direct ContextService requests, or :data:`None` for the local process. + """ + raise NotImplementedError() + + def close(self): + """ + Finalize any associated resources. + """ + raise NotImplementedError() + + +class WorkerModel(object): + """ + Interface used by StrategyMixin to manage various Mitogen services, by + default running in one or more connection multiplexer subprocesses spawned + off the top-level Ansible process. + """ + def on_strategy_start(self): + """ + Called prior to strategy start in the top-level process. Responsible + for preparing any worker/connection multiplexer state. + """ + raise NotImplementedError() + + def on_strategy_complete(self): + """ + Called after strategy completion in the top-level process. Must place + Ansible back in a "compatible" state where any other strategy plug-in + may execute. + """ + raise NotImplementedError() + + def get_binding(self, inventory_name): + """ + Return a :class:`Binding` to access Mitogen services for + `inventory_name`. Usually called from worker processes, but may also be + called from top-level process to handle "meta: reset_connection". + """ + raise NotImplementedError() + + +class ClassicBinding(Binding): + """ + Only one connection may be active at a time in a classic worker, so its + binding just provides forwarders back to :class:`ClassicWorkerModel`. + """ + def __init__(self, model): + self.model = model + + def get_service_context(self): + """ + See Binding.get_service_context(). + """ + return self.model.parent + + def get_child_service_context(self): + """ + See Binding.get_child_service_context(). + """ + return self.model.parent + + def close(self): + """ + See Binding.close(). + """ + self.model.on_binding_close() + + +class ClassicWorkerModel(WorkerModel): + #: In the top-level process, this references one end of a socketpair(), + #: whose other end child MuxProcesses block reading from to determine when + #: the master process dies. When the top-level exits abnormally, or + #: normally but where :func:`_on_process_exit` has been called, this socket + #: will be closed, causing all the children to wake. + parent_sock = None + + #: In the mux process, this is the other end of :attr:`cls_parent_sock`. + #: The main thread blocks on a read from it until :attr:`cls_parent_sock` + #: is closed. + child_sock = None + + #: mitogen.master.Router for this worker. + router = None + + #: mitogen.master.Broker for this worker. + broker = None + + #: Name of multiplexer process socket we are currently connected to. + listener_path = None + + #: mitogen.parent.Context representing the parent Context, which is the + #: connection multiplexer process when running in classic mode, or the + #: top-level process when running a new-style mode. + parent = None + + def __init__(self, _init_logging=True): + """ + Arrange for classic model multiplexers to be started. The parent choses + UNIX socket paths each child will use prior to fork, creates a + socketpair used essentially as a semaphore, then blocks waiting for the + child to indicate the UNIX socket is ready for use. + + :param bool _init_logging: + For testing, if :data:`False`, don't initialize logging. + """ + # #573: The process ID that installed the :mod:`atexit` handler. If + # some unknown Ansible plug-in forks the Ansible top-level process and + # later performs a graceful Python exit, it may try to wait for child + # PIDs it never owned, causing a crash. We want to avoid that. + self._pid = os.getpid() + + common_setup(_init_logging=_init_logging) + + self.parent_sock, self.child_sock = socket.socketpair() + mitogen.core.set_cloexec(self.parent_sock.fileno()) + mitogen.core.set_cloexec(self.child_sock.fileno()) + + self._muxes = [ + MuxProcess(self, index) + for index in range(get_cpu_count(default=1)) + ] + for mux in self._muxes: + mux.start() + + atexit.register(self._on_process_exit) + self.child_sock.close() + self.child_sock = None + + def _listener_for_name(self, name): + """ + Given an inventory hostname, return the UNIX listener that should + communicate with it. This is a simple hash of the inventory name. + """ + mux = self._muxes[abs(hash(name)) % len(self._muxes)] + LOG.debug('will use multiplexer %d (%s) to connect to "%s"', + mux.index, mux.path, name) + return mux.path + + def _reconnect(self, path): + if self.router is not None: + # Router can just be overwritten, but the previous parent + # connection must explicitly be removed from the broker first. + self.router.disconnect(self.parent) + self.parent = None + self.router = None + + try: + self.router, self.parent = mitogen.unix.connect( + path=path, + broker=self.broker, + ) + except mitogen.unix.ConnectError as e: + # This is not AnsibleConnectionFailure since we want to break + # with_items loops. + raise ansible.errors.AnsibleError(shutting_down_msg % (e,)) + + self.router.max_message_size = MAX_MESSAGE_SIZE + self.listener_path = path + + def _on_process_exit(self): + """ + This is an :mod:`atexit` handler installed in the top-level process. + + Shut the write end of `sock`, causing the receive side of the socket in + every :class:`MuxProcess` to return 0-byte reads, and causing their + main threads to wake and initiate shutdown. After shutting the socket + down, wait on each child to finish exiting. + + This is done using :mod:`atexit` since Ansible lacks any better hook to + run code during exit, and unless some synchronization exists with + MuxProcess, debug logs may appear on the user's terminal *after* the + prompt has been printed. + """ + if self._pid != os.getpid(): + return + + try: + self.parent_sock.shutdown(socket.SHUT_WR) + except socket.error: + # Already closed. This is possible when tests are running. + LOG.debug('_on_process_exit: ignoring duplicate call') + return + + mitogen.core.io_op(self.parent_sock.recv, 1) + self.parent_sock.close() + + for mux in self._muxes: + _, status = os.waitpid(mux.pid, 0) + status = mitogen.fork._convert_exit_status(status) + LOG.debug('multiplexer %d PID %d %s', mux.index, mux.pid, + mitogen.parent.returncode_to_str(status)) + + def _test_reset(self): + """ + Used to clean up in unit tests. + """ + self.on_binding_close() + self._on_process_exit() + set_worker_model(None) + + global _classic_worker_model + _classic_worker_model = None + + def on_strategy_start(self): + """ + See WorkerModel.on_strategy_start(). + """ + + def on_strategy_complete(self): + """ + See WorkerModel.on_strategy_complete(). + """ + + def get_binding(self, inventory_name): + """ + See WorkerModel.get_binding(). + """ + if self.broker is None: + self.broker = Broker() + + path = self._listener_for_name(inventory_name) + if path != self.listener_path: + self._reconnect(path) + + return ClassicBinding(self) + + def on_binding_close(self): + if not self.broker: + return + + self.broker.shutdown() + self.broker.join() + self.router = None + self.broker = None + self.parent = None + self.listener_path = None + + # #420: Ansible executes "meta" actions in the top-level process, + # meaning "reset_connection" will cause :class:`mitogen.core.Latch` FDs + # to be cached and erroneously shared by children on subsequent + # WorkerProcess forks. To handle that, call on_fork() to ensure any + # shared state is discarded. + # #490: only attempt to clean up when it's known that some resources + # exist to cleanup, otherwise later __del__ double-call to close() due + # to GC at random moment may obliterate an unrelated Connection's + # related resources. + mitogen.fork.on_fork() + + class MuxProcess(object): """ Implement a subprocess forked from the Ansible top-level, as a safe place to contain the Mitogen IO multiplexer thread, keeping its use of the logging package (and the logging package's heavy use of locks) far away - from the clutches of os.fork(), which is used continuously by the - multiprocessing package in the top-level process. + from os.fork(), which is used continuously by the multiprocessing package + in the top-level process. The problem with running the multiplexer in that process is that should the multiplexer thread be in the process of emitting a log entry (and holding @@ -129,97 +615,68 @@ class MuxProcess(object): See https://bugs.python.org/issue6721 for a thorough description of the class of problems this worker is intended to avoid. """ - - #: In the top-level process, this references one end of a socketpair(), - #: which the MuxProcess blocks reading from in order to determine when - #: the master process dies. Once the read returns, the MuxProcess will - #: begin shutting itself down. - worker_sock = None - - #: In the worker process, this references the other end of - #: :py:attr:`worker_sock`. - child_sock = None - - #: In the top-level process, this is the PID of the single MuxProcess - #: that was spawned. - worker_pid = None - #: A copy of :data:`os.environ` at the time the multiplexer process was #: started. It's used by mitogen_local.py to find changes made to the #: top-level environment (e.g. vars plugins -- issue #297) that must be #: applied to locally executed commands and modules. - original_env = None - - #: In both processes, this is the temporary UNIX socket used for - #: forked WorkerProcesses to contact the MuxProcess - unix_listener_path = None - - #: Singleton. - _instance = None - - @classmethod - def start(cls, _init_logging=True): - """ - Arrange for the subprocess to be started, if it is not already running. - - The parent process picks a UNIX socket path the child will use prior to - fork, creates a socketpair used essentially as a semaphore, then blocks - waiting for the child to indicate the UNIX socket is ready for use. - - :param bool _init_logging: - For testing, if :data:`False`, don't initialize logging. - """ - if cls.worker_sock is not None: + cls_original_env = None + + def __init__(self, model, index): + #: :class:`ClassicWorkerModel` instance we were created by. + self.model = model + #: MuxProcess CPU index. + self.index = index + #: Individual path of this process. + self.path = mitogen.unix.make_socket_path() + + def start(self): + self.pid = os.fork() + if self.pid: + # Wait for child to boot before continuing. + mitogen.core.io_op(self.model.parent_sock.recv, 1) return - if faulthandler is not None: - faulthandler.enable() - - mitogen.utils.setup_gil() - cls.unix_listener_path = mitogen.unix.make_socket_path() - cls.worker_sock, cls.child_sock = socket.socketpair() - atexit.register(lambda: clean_shutdown(cls.worker_sock)) - mitogen.core.set_cloexec(cls.worker_sock.fileno()) - mitogen.core.set_cloexec(cls.child_sock.fileno()) - - cls.profiling = os.environ.get('MITOGEN_PROFILING') is not None - if cls.profiling: - mitogen.core.enable_profiling() - if _init_logging: - ansible_mitogen.logging.setup() - - cls.original_env = dict(os.environ) - cls.child_pid = os.fork() - if cls.child_pid: - save_pid('controller') - ansible_mitogen.logging.set_process_name('top') - ansible_mitogen.affinity.policy.assign_controller() - cls.child_sock.close() - cls.child_sock = None - mitogen.core.io_op(cls.worker_sock.recv, 1) - else: - save_pid('mux') - ansible_mitogen.logging.set_process_name('mux') - ansible_mitogen.affinity.policy.assign_muxprocess() - cls.worker_sock.close() - cls.worker_sock = None - self = cls() - self.worker_main() + ansible_mitogen.logging.set_process_name('mux:' + str(self.index)) + if setproctitle: + setproctitle.setproctitle('mitogen mux:%s (%s)' % ( + self.index, + os.path.basename(self.path), + )) + + self.model.parent_sock.close() + self.model.parent_sock = None + try: + try: + self.worker_main() + except Exception: + LOG.exception('worker_main() crashed') + finally: + sys.exit() def worker_main(self): """ - The main function of for the mux process: setup the Mitogen broker - thread and ansible_mitogen services, then sleep waiting for the socket + The main function of the mux process: setup the Mitogen broker thread + and ansible_mitogen services, then sleep waiting for the socket connected to the parent to be closed (indicating the parent has died). """ + save_pid('mux') + + # #623: MuxProcess ignores SIGINT because it wants to live until every + # Ansible worker process has been cleaned up by + # TaskQueueManager.cleanup(), otherwise harmles yet scary warnings + # about being unable connect to MuxProess could be printed. + signal.signal(signal.SIGINT, signal.SIG_IGN) + ansible_mitogen.logging.set_process_name('mux') + ansible_mitogen.affinity.policy.assign_muxprocess(self.index) + self._setup_master() self._setup_services() try: # Let the parent know our listening socket is ready. - mitogen.core.io_op(self.child_sock.send, b('1')) + mitogen.core.io_op(self.model.child_sock.send, b('1')) # Block until the socket is closed, which happens on parent exit. - mitogen.core.io_op(self.child_sock.recv, 1) + mitogen.core.io_op(self.model.child_sock.recv, 1) finally: self.broker.shutdown() self.broker.join() @@ -238,64 +695,6 @@ class MuxProcess(object): if secs: mitogen.debug.dump_to_logger(secs=secs) - def _setup_simplejson(self, responder): - """ - We support serving simplejson for Python 2.4 targets on Ansible 2.3, at - least so the package's own CI Docker scripts can run without external - help, however newer versions of simplejson no longer support Python - 2.4. Therefore override any installed/loaded version with a - 2.4-compatible version we ship in the compat/ directory. - """ - responder.whitelist_prefix('simplejson') - - # issue #536: must be at end of sys.path, in case existing newer - # version is already loaded. - compat_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.append(compat_path) - - for fullname, is_pkg, suffix in ( - (u'simplejson', True, '__init__.py'), - (u'simplejson.decoder', False, 'decoder.py'), - (u'simplejson.encoder', False, 'encoder.py'), - (u'simplejson.scanner', False, 'scanner.py'), - ): - path = os.path.join(compat_path, 'simplejson', suffix) - fp = open(path, 'rb') - try: - source = fp.read() - finally: - fp.close() - - responder.add_source_override( - fullname=fullname, - path=path, - source=source, - is_pkg=is_pkg, - ) - - def _setup_responder(self, responder): - """ - Configure :class:`mitogen.master.ModuleResponder` to only permit - certain packages, and to generate custom responses for certain modules. - """ - responder.whitelist_prefix('ansible') - responder.whitelist_prefix('ansible_mitogen') - self._setup_simplejson(responder) - - # Ansible 2.3 is compatible with Python 2.4 targets, however - # ansible/__init__.py is not. Instead, executor/module_common.py writes - # out a 2.4-compatible namespace package for unknown reasons. So we - # copy it here. - responder.add_source_override( - fullname='ansible', - path=ansible.__file__, - source=(ANSIBLE_PKG_OVERRIDE % ( - ansible.__version__, - ansible.__author__, - )).encode(), - is_pkg=True, - ) - def _setup_master(self): """ Construct a Router, Broker, and mitogen.unix listener @@ -303,14 +702,14 @@ class MuxProcess(object): self.broker = mitogen.master.Broker(install_watcher=False) self.router = mitogen.master.Router( broker=self.broker, - max_message_size=4096 * 1048576, + max_message_size=MAX_MESSAGE_SIZE, ) - self._setup_responder(self.router.responder) - mitogen.core.listen(self.broker, 'shutdown', self.on_broker_shutdown) - mitogen.core.listen(self.broker, 'exit', self.on_broker_exit) - self.listener = mitogen.unix.Listener( + _setup_responder(self.router.responder) + mitogen.core.listen(self.broker, 'shutdown', self._on_broker_shutdown) + mitogen.core.listen(self.broker, 'exit', self._on_broker_exit) + self.listener = mitogen.unix.Listener.build_stream( router=self.router, - path=self.unix_listener_path, + path=self.path, backlog=C.DEFAULT_FORKS, ) self._enable_router_debug() @@ -323,36 +722,24 @@ class MuxProcess(object): """ self.pool = mitogen.service.Pool( router=self.router, - services=[ - mitogen.service.FileService(router=self.router), - mitogen.service.PushFileService(router=self.router), - ansible_mitogen.services.ContextService(self.router), - ansible_mitogen.services.ModuleDepService(self.router), - ], size=getenv_int('MITOGEN_POOL_SIZE', default=32), ) - LOG.debug('Service pool configured: size=%d', self.pool.size) + setup_pool(self.pool) - def on_broker_shutdown(self): + def _on_broker_shutdown(self): """ - Respond to broker shutdown by beginning service pool shutdown. Do not - join on the pool yet, since that would block the broker thread which - then cannot clean up pending handlers, which is required for the - threads to exit gracefully. + Respond to broker shutdown by shutting down the pool. Do not join on it + yet, since that would block the broker thread which then cannot clean + up pending handlers and connections, which is required for the threads + to exit gracefully. """ - # In normal operation we presently kill the process because there is - # not yet any way to cancel connect(). - self.pool.stop(join=self.profiling) + self.pool.stop(join=False) - def on_broker_exit(self): + def _on_broker_exit(self): """ - Respond to the broker thread about to exit by sending SIGTERM to - ourself. In future this should gracefully join the pool, but TERM is - fine for now. + Respond to the broker thread about to exit by finally joining on the + pool. This is safe since pools only block in connection attempts, and + connection attempts fail with CancelledError when broker shutdown + begins. """ - if not self.profiling: - # In normal operation we presently kill the process because there is - # not yet any way to cancel connect(). When profiling, threads - # including the broker must shut down gracefully, otherwise pstats - # won't be written. - os.kill(os.getpid(), signal.SIGTERM) + self.pool.join() diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 30c36be7..5cf171b6 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -37,7 +37,6 @@ how to build arguments for it, preseed related data, etc. """ import atexit -import codecs import imp import os import re @@ -52,7 +51,6 @@ import mitogen.core import ansible_mitogen.target # TODO: circular import from mitogen.core import b from mitogen.core import bytes_partition -from mitogen.core import str_partition from mitogen.core import str_rpartition from mitogen.core import to_text @@ -104,12 +102,59 @@ iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) -if mitogen.core.PY3: - shlex_split = shlex.split -else: - def shlex_split(s, comments=False): - return [mitogen.core.to_text(token) - for token in shlex.split(str(s), comments=comments)] +def shlex_split_b(s): + """ + Use shlex.split() to split characters in some single-byte encoding, without + knowing what that encoding is. The input is bytes, the output is a list of + bytes. + """ + assert isinstance(s, mitogen.core.BytesType) + if mitogen.core.PY3: + return [ + t.encode('latin1') + for t in shlex.split(s.decode('latin1'), comments=True) + ] + + return [t for t in shlex.split(s, comments=True)] + + +class TempFileWatcher(object): + """ + Since Ansible 2.7.0, lineinfile leaks file descriptors returned by + :func:`tempfile.mkstemp` (ansible/ansible#57327). Handle this and all + similar cases by recording descriptors produced by mkstemp during module + execution, and cleaning up any leaked descriptors on completion. + """ + def __init__(self): + self._real_mkstemp = tempfile.mkstemp + # (fd, st.st_dev, st.st_ino) + self._fd_dev_inode = [] + tempfile.mkstemp = self._wrap_mkstemp + + def _wrap_mkstemp(self, *args, **kwargs): + fd, path = self._real_mkstemp(*args, **kwargs) + st = os.fstat(fd) + self._fd_dev_inode.append((fd, st.st_dev, st.st_ino)) + return fd, path + + def revert(self): + tempfile.mkstemp = self._real_mkstemp + for tup in self._fd_dev_inode: + self._revert_one(*tup) + + def _revert_one(self, fd, st_dev, st_ino): + try: + st = os.fstat(fd) + except OSError: + # FD no longer exists. + return + + if not (st.st_dev == st_dev and st.st_ino == st_ino): + # FD reused. + return + + LOG.info("a tempfile.mkstemp() FD was leaked during the last task") + os.close(fd) class EnvironmentFileWatcher(object): @@ -126,13 +171,19 @@ class EnvironmentFileWatcher(object): A more robust future approach may simply be to arrange for the persistent interpreter to restart when a change is detected. """ + # We know nothing about the character set of /etc/environment or the + # process environment. + environ = getattr(os, 'environb', os.environ) + def __init__(self, path): self.path = os.path.expanduser(path) #: Inode data at time of last check. self._st = self._stat() #: List of inherited keys appearing to originated from this file. - self._keys = [key for key, value in self._load() - if value == os.environ.get(key)] + self._keys = [ + key for key, value in self._load() + if value == self.environ.get(key) + ] LOG.debug('%r installed; existing keys: %r', self, self._keys) def __repr__(self): @@ -146,7 +197,7 @@ class EnvironmentFileWatcher(object): def _load(self): try: - fp = codecs.open(self.path, 'r', encoding='utf-8') + fp = open(self.path, 'rb') try: return list(self._parse(fp)) finally: @@ -160,36 +211,36 @@ class EnvironmentFileWatcher(object): """ for line in fp: # ' #export foo=some var ' -> ['#export', 'foo=some var '] - bits = shlex_split(line, comments=True) - if (not bits) or bits[0].startswith('#'): + bits = shlex_split_b(line) + if (not bits) or bits[0].startswith(b('#')): continue - if bits[0] == u'export': + if bits[0] == b('export'): bits.pop(0) - key, sep, value = str_partition(u' '.join(bits), u'=') + key, sep, value = bytes_partition(b(' ').join(bits), b('=')) if key and sep: yield key, value def _on_file_changed(self): LOG.debug('%r: file changed, reloading', self) for key, value in self._load(): - if key in os.environ: + if key in self.environ: LOG.debug('%r: existing key %r=%r exists, not setting %r', - self, key, os.environ[key], value) + self, key, self.environ[key], value) else: LOG.debug('%r: setting key %r to %r', self, key, value) self._keys.append(key) - os.environ[key] = value + self.environ[key] = value def _remove_existing(self): """ When a change is detected, remove keys that existed in the old file. """ for key in self._keys: - if key in os.environ: + if key in self.environ: LOG.debug('%r: removing old key %r', self, key) - del os.environ[key] + del self.environ[key] self._keys = [] def check(self): @@ -344,11 +395,22 @@ class Runner(object): env.update(self.env) self._env = TemporaryEnvironment(env) + def _revert_cwd(self): + """ + #591: make a best-effort attempt to return to :attr:`good_temp_dir`. + """ + try: + os.chdir(self.good_temp_dir) + except OSError: + LOG.debug('%r: could not restore CWD to %r', + self, self.good_temp_dir) + def revert(self): """ Revert any changes made to the process after running a module. The base implementation simply restores the original environment. """ + self._revert_cwd() self._env.revert() self.revert_temp_dir() @@ -760,7 +822,21 @@ class NewStyleRunner(ScriptRunner): for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) for fullname in self.module_map['builtin']: - mitogen.core.import_module(fullname) + try: + mitogen.core.import_module(fullname) + except ImportError: + # #590: Ansible 2.8 module_utils.distro is a package that + # replaces itself in sys.modules with a non-package during + # import. Prior to replacement, it is a real package containing + # a '_distro' submodule which is used on 2.x. Given a 2.x + # controller and 3.x target, the import hook never needs to run + # again before this replacement occurs, and 'distro' is + # replaced with a module from the stdlib. In this case as this + # loop progresses to the next entry and attempts to preload + # 'distro._distro', the import mechanism will fail. So here we + # silently ignore any failure for it. + if fullname != 'ansible.module_utils.distro._distro': + raise def _setup_excepthook(self): """ @@ -778,6 +854,7 @@ class NewStyleRunner(ScriptRunner): # module, but this has never been a bug report. Instead act like an # interpreter that had its script piped on stdin. self._argv = TemporaryArgv(['']) + self._temp_watcher = TempFileWatcher() self._importer = ModuleUtilsImporter( context=self.service_context, module_utils=self.module_map['custom'], @@ -793,6 +870,7 @@ class NewStyleRunner(ScriptRunner): def revert(self): self.atexit_wrapper.revert() + self._temp_watcher.revert() self._argv.revert() self._stdio.revert() self._revert_excepthook() diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index a7c0e46f..e6c41e5b 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -180,7 +180,7 @@ class ContextService(mitogen.service.Service): Return a reference, making it eligable for recycling once its reference count reaches zero. """ - LOG.debug('%r.put(%r)', self, context) + LOG.debug('decrementing reference count for %r', context) self._lock.acquire() try: if self._refs_by_context.get(context, 0) == 0: @@ -372,7 +372,7 @@ class ContextService(mitogen.service.Service): try: method = getattr(self.router, spec['method']) except AttributeError: - raise Error('unsupported method: %(transport)s' % spec) + raise Error('unsupported method: %(method)s' % spec) context = method(via=via, unidirectional=True, **spec['kwargs']) if via and spec.get('enable_lru'): @@ -443,7 +443,7 @@ class ContextService(mitogen.service.Service): @mitogen.service.arg_spec({ 'stack': list }) - def get(self, msg, stack): + def get(self, stack): """ Return a Context referring to an established connection with the given configuration, establishing new connections as necessary. diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index b9211fcc..8f093999 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -31,6 +31,11 @@ import os import signal import threading +try: + import setproctitle +except ImportError: + setproctitle = None + import mitogen.core import ansible_mitogen.affinity import ansible_mitogen.loaders @@ -40,9 +45,15 @@ import ansible_mitogen.process import ansible import ansible.executor.process.worker +try: + # 2.8+ has a standardized "unset" object. + from ansible.utils.sentinel import Sentinel +except ImportError: + Sentinel = None + ANSIBLE_VERSION_MIN = '2.3' -ANSIBLE_VERSION_MAX = '2.7' +ANSIBLE_VERSION_MAX = '2.8' NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" @@ -113,9 +124,15 @@ def wrap_action_loader__get(name, *args, **kwargs): the use of shell fragments wherever possible. This is used instead of static subclassing as it generalizes to third party - action modules outside the Ansible tree. + action plugins outside the Ansible tree. """ - klass = action_loader__get(name, class_only=True) + get_kwargs = {'class_only': True} + if name in ('fetch',): + name = 'mitogen_' + name + if ansible.__version__ >= '2.8': + get_kwargs['collection_list'] = kwargs.pop('collection_list', None) + + klass = action_loader__get(name, **get_kwargs) if klass: bases = (ansible_mitogen.mixins.ActionModuleMixin, klass) adorned_klass = type(str(name), bases, {}) @@ -126,20 +143,28 @@ def wrap_action_loader__get(name, *args, **kwargs): def wrap_connection_loader__get(name, *args, **kwargs): """ - While the strategy is active, rewrite connection_loader.get() calls for - some transports into requests for a compatible Mitogen transport. + While a Mitogen strategy is active, rewrite connection_loader.get() calls + for some transports into requests for a compatible Mitogen transport. """ - if name in ('docker', 'kubectl', 'jail', 'local', 'lxc', - 'lxd', 'machinectl', 'setns', 'ssh'): + if name in ('buildah', 'docker', 'kubectl', 'jail', 'local', + 'lxc', 'lxd', 'machinectl', 'setns', 'ssh'): name = 'mitogen_' + name return connection_loader__get(name, *args, **kwargs) -def wrap_worker__run(*args, **kwargs): +def wrap_worker__run(self): """ - While the strategy is active, rewrite connection_loader.get() calls for - some transports into requests for a compatible Mitogen transport. + While a Mitogen strategy is active, trap WorkerProcess.run() calls and use + the opportunity to set the worker's name in the process list and log + output, activate profiling if requested, and bind the worker to a specific + CPU. """ + if setproctitle: + setproctitle.setproctitle('worker:%s task:%s' % ( + self._host.name, + self._task.action, + )) + # Ignore parent's attempts to murder us when we still need to write # profiling output. if mitogen.core._profile_hook.__name__ != '_profile_hook': @@ -148,16 +173,70 @@ def wrap_worker__run(*args, **kwargs): ansible_mitogen.logging.set_process_name('task') ansible_mitogen.affinity.policy.assign_worker() return mitogen.core._profile_hook('WorkerProcess', - lambda: worker__run(*args, **kwargs) + lambda: worker__run(self) ) +class AnsibleWrappers(object): + """ + Manage add/removal of various Ansible runtime hooks. + """ + def _add_plugin_paths(self): + """ + Add the Mitogen plug-in directories to the ModuleLoader path, avoiding + the need for manual configuration. + """ + base_dir = os.path.join(os.path.dirname(__file__), 'plugins') + ansible_mitogen.loaders.connection_loader.add_directory( + os.path.join(base_dir, 'connection') + ) + ansible_mitogen.loaders.action_loader.add_directory( + os.path.join(base_dir, 'action') + ) + + def _install_wrappers(self): + """ + Install our PluginLoader monkey patches and update global variables + with references to the real functions. + """ + global action_loader__get + action_loader__get = ansible_mitogen.loaders.action_loader.get + ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get + + global connection_loader__get + connection_loader__get = ansible_mitogen.loaders.connection_loader.get + ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get + + global worker__run + worker__run = ansible.executor.process.worker.WorkerProcess.run + ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run + + def _remove_wrappers(self): + """ + Uninstall the PluginLoader monkey patches. + """ + ansible_mitogen.loaders.action_loader.get = action_loader__get + ansible_mitogen.loaders.connection_loader.get = connection_loader__get + ansible.executor.process.worker.WorkerProcess.run = worker__run + + def install(self): + self._add_plugin_paths() + self._install_wrappers() + + def remove(self): + self._remove_wrappers() + + class StrategyMixin(object): """ - This mix-in enhances any built-in strategy by arranging for various Mitogen - services to be initialized in the Ansible top-level process, and for worker - processes to grow support for using those top-level services to communicate - with and execute modules on remote hosts. + This mix-in enhances any built-in strategy by arranging for an appropriate + WorkerModel instance to be constructed as necessary, or for the existing + one to be reused. + + The WorkerModel in turn arranges for a connection multiplexer to be started + somewhere (by default in an external process), and for WorkerProcesses to + grow support for using those top-level services to communicate with remote + hosts. Mitogen: @@ -175,18 +254,19 @@ class StrategyMixin(object): services, review the Standard Handles section of the How It Works guide in the documentation. - A ContextService is installed as a message handler in the master - process and run on a private thread. It is responsible for accepting - requests to establish new SSH connections from worker processes, and - ensuring precisely one connection exists and is reused for subsequent - playbook steps. The service presently runs in a single thread, so to - begin with, new SSH connections are serialized. + A ContextService is installed as a message handler in the connection + mutliplexer subprocess and run on a private thread. It is responsible + for accepting requests to establish new SSH connections from worker + processes, and ensuring precisely one connection exists and is reused + for subsequent playbook steps. The service presently runs in a single + thread, so to begin with, new SSH connections are serialized. Finally a mitogen.unix listener is created through which WorkerProcess - can establish a connection back into the master process, in order to - avail of ContextService. A UNIX listener socket is necessary as there - is no more sane mechanism to arrange for IPC between the Router in the - master process, and the corresponding Router in the worker process. + can establish a connection back into the connection multiplexer, in + order to avail of ContextService. A UNIX listener socket is necessary + as there is no more sane mechanism to arrange for IPC between the + Router in the connection multiplexer, and the corresponding Router in + the worker process. Ansible: @@ -194,10 +274,10 @@ class StrategyMixin(object): connection and action plug-ins. For connection plug-ins, if the desired method is "local" or "ssh", it - is redirected to the "mitogen" connection plug-in. That plug-in - implements communication via a UNIX socket connection to the top-level - Ansible process, and uses ContextService running in the top-level - process to actually establish and manage the connection. + is redirected to one of the "mitogen_*" connection plug-ins. That + plug-in implements communication via a UNIX socket connection to the + connection multiplexer process, and uses ContextService running there + to establish a persistent connection to the target. For action plug-ins, the original class is looked up as usual, but a new subclass is created dynamically in order to mix-in @@ -213,43 +293,6 @@ class StrategyMixin(object): remote process, all the heavy lifting of transferring the action module and its dependencies are automatically handled by Mitogen. """ - def _install_wrappers(self): - """ - Install our PluginLoader monkey patches and update global variables - with references to the real functions. - """ - global action_loader__get - action_loader__get = ansible_mitogen.loaders.action_loader.get - ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get - - global connection_loader__get - connection_loader__get = ansible_mitogen.loaders.connection_loader.get - ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get - - global worker__run - worker__run = ansible.executor.process.worker.WorkerProcess.run - ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run - - def _remove_wrappers(self): - """ - Uninstall the PluginLoader monkey patches. - """ - ansible_mitogen.loaders.action_loader.get = action_loader__get - ansible_mitogen.loaders.connection_loader.get = connection_loader__get - ansible.executor.process.worker.WorkerProcess.run = worker__run - - def _add_plugin_paths(self): - """ - Add the Mitogen plug-in directories to the ModuleLoader path, avoiding - the need for manual configuration. - """ - base_dir = os.path.join(os.path.dirname(__file__), 'plugins') - ansible_mitogen.loaders.connection_loader.add_directory( - os.path.join(base_dir, 'connection') - ) - ansible_mitogen.loaders.action_loader.add_directory( - os.path.join(base_dir, 'action') - ) def _queue_task(self, host, task, task_vars, play_context): """ @@ -261,14 +304,17 @@ class StrategyMixin(object): name=task.action, mod_type='', ) - ansible_mitogen.loaders.connection_loader.get( - name=play_context.connection, - class_only=True, - ) ansible_mitogen.loaders.action_loader.get( name=task.action, class_only=True, ) + if play_context.connection is not Sentinel: + # 2.8 appears to defer computing this until inside the worker. + # TODO: figure out where it has moved. + ansible_mitogen.loaders.connection_loader.get( + name=play_context.connection, + class_only=True, + ) return super(StrategyMixin, self)._queue_task( host=host, @@ -277,20 +323,35 @@ class StrategyMixin(object): play_context=play_context, ) + def _get_worker_model(self): + """ + In classic mode a single :class:`WorkerModel` exists, which manages + references and configuration of the associated connection multiplexer + process. + """ + return ansible_mitogen.process.get_classic_worker_model() + def run(self, iterator, play_context, result=0): """ - Arrange for a mitogen.master.Router to be available for the duration of - the strategy's real run() method. + Wrap :meth:`run` to ensure requisite infrastructure and modifications + are configured for the duration of the call. """ _assert_supported_release() - - ansible_mitogen.process.MuxProcess.start() - run = super(StrategyMixin, self).run - self._add_plugin_paths() - self._install_wrappers() + wrappers = AnsibleWrappers() + self._worker_model = self._get_worker_model() + ansible_mitogen.process.set_worker_model(self._worker_model) try: - return mitogen.core._profile_hook('Strategy', - lambda: run(iterator, play_context) - ) + self._worker_model.on_strategy_start() + try: + wrappers.install() + try: + run = super(StrategyMixin, self).run + return mitogen.core._profile_hook('Strategy', + lambda: run(iterator, play_context) + ) + finally: + wrappers.remove() + finally: + self._worker_model.on_strategy_complete() finally: - self._remove_wrappers() + ansible_mitogen.process.set_worker_model(None) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index ad1cab3e..aa4a16d0 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -240,6 +240,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)): undesirable in some circumstances. """ + @abc.abstractmethod + def mitogen_buildah_path(self): + """ + The path to the "buildah" program for the 'buildah' transport. + """ + @abc.abstractmethod def mitogen_docker_path(self): """ @@ -276,6 +282,18 @@ class Spec(with_metaclass(abc.ABCMeta, object)): The path to the "machinectl" program for the 'setns' transport. """ + @abc.abstractmethod + def mitogen_ssh_keepalive_interval(self): + """ + The SSH ServerAliveInterval. + """ + + @abc.abstractmethod + def mitogen_ssh_keepalive_count(self): + """ + The SSH ServerAliveCount. + """ + @abc.abstractmethod def mitogen_ssh_debug_level(self): """ @@ -294,6 +312,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)): Connection-specific arguments. """ + @abc.abstractmethod + def ansible_doas_exe(self): + """ + Value of "ansible_doas_exe" variable. + """ + class PlayContextSpec(Spec): """ @@ -372,7 +396,15 @@ class PlayContextSpec(Spec): ] def become_exe(self): - return self._play_context.become_exe + # In Ansible 2.8, PlayContext.become_exe always has a default value due + # to the new options mechanism. Previously it was only set if a value + # ("somewhere") had been specified for the task. + # For consistency in the tests, here we make older Ansibles behave like + # newer Ansibles. + exe = self._play_context.become_exe + if exe is None and self._play_context.become_method == 'sudo': + exe = 'sudo' + return exe def sudo_args(self): return [ @@ -380,8 +412,9 @@ class PlayContextSpec(Spec): for term in ansible.utils.shlex.shlex_split( first_true(( self._play_context.become_flags, - self._play_context.sudo_flags, - # Ansible 2.3. + # Ansible <=2.7. + getattr(self._play_context, 'sudo_flags', ''), + # Ansible <=2.3. getattr(C, 'DEFAULT_BECOME_FLAGS', ''), getattr(C, 'DEFAULT_SUDO_FLAGS', '') ), default='') @@ -397,6 +430,9 @@ class PlayContextSpec(Spec): def mitogen_mask_remote_name(self): return self._connection.get_task_var('mitogen_mask_remote_name') + def mitogen_buildah_path(self): + return self._connection.get_task_var('mitogen_buildah_path') + def mitogen_docker_path(self): return self._connection.get_task_var('mitogen_docker_path') @@ -412,6 +448,12 @@ class PlayContextSpec(Spec): def mitogen_lxc_info_path(self): return self._connection.get_task_var('mitogen_lxc_info_path') + def mitogen_ssh_keepalive_interval(self): + return self._connection.get_task_var('mitogen_ssh_keepalive_interval') + + def mitogen_ssh_keepalive_count(self): + return self._connection.get_task_var('mitogen_ssh_keepalive_count') + def mitogen_machinectl_path(self): return self._connection.get_task_var('mitogen_machinectl_path') @@ -424,6 +466,12 @@ class PlayContextSpec(Spec): def extra_args(self): return self._connection.get_extra_args() + def ansible_doas_exe(self): + return ( + self._connection.get_task_var('ansible_doas_exe') or + os.environ.get('ANSIBLE_DOAS_EXE') + ) + class MitogenViaSpec(Spec): """ @@ -608,6 +656,9 @@ class MitogenViaSpec(Spec): def mitogen_mask_remote_name(self): return self._host_vars.get('mitogen_mask_remote_name') + def mitogen_buildah_path(self): + return self._host_vars.get('mitogen_buildah_path') + def mitogen_docker_path(self): return self._host_vars.get('mitogen_docker_path') @@ -623,6 +674,12 @@ class MitogenViaSpec(Spec): def mitogen_lxc_info_path(self): return self._host_vars.get('mitogen_lxc_info_path') + def mitogen_ssh_keepalive_interval(self): + return self._host_vars.get('mitogen_ssh_keepalive_interval') + + def mitogen_ssh_keepalive_count(self): + return self._host_vars.get('mitogen_ssh_keepalive_count') + def mitogen_machinectl_path(self): return self._host_vars.get('mitogen_machinectl_path') @@ -634,3 +691,9 @@ class MitogenViaSpec(Spec): def extra_args(self): return [] # TODO + + def ansible_doas_exe(self): + return ( + self._host_vars.get('ansible_doas_exe') or + os.environ.get('ANSIBLE_DOAS_EXE') + ) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 297ab9ef..f5fe42b0 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,19 +1,35 @@ {% extends "!layout.html" %} {% set css_files = css_files + ['_static/style.css'] %} +{# We don't support Sphinx search, so don't let its JS either. #} +{% block scripts %} +{% endblock %} + +{# Alabaster ships a completely useless custom.css, suppress it. #} +{%- block extrahead %} + + +{% endblock %} + {% block footer %} {{ super() }} - + + {% endblock %} diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 5b541a14..22d7223f 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -75,33 +75,28 @@ Installation ``mitogen_host_pinned`` strategies exists to mimic the ``free`` and ``host_pinned`` strategies. -4. If targets have a restrictive ``sudoers`` file, add a rule like: - - :: - - deploy = (ALL) NOPASSWD:/usr/bin/python -c* - -5. +4. .. raw:: html -
- Releases occur frequently and often include important fixes. Subscribe - to the mitogen-announce - mailing list be notified of new releases. + + + + Get notified of new releases and important fixes.

- - -

-
+ + +

+ Demo @@ -145,7 +140,7 @@ Testimonials Noteworthy Differences ---------------------- -* Ansible 2.3-2.7 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify +* Ansible 2.3-2.8 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify your installation is running one of these versions by checking ``ansible --version`` output. @@ -169,18 +164,27 @@ Noteworthy Differences - initech_app - y2k_fix +* Ansible 2.8 `interpreter discovery + `_ + and `become plugins + `_ are not yet + supported. + * The ``doas``, ``su`` and ``sudo`` become methods are available. File bugs to register interest in more. -* The `docker `_, - `jail `_, - `kubectl `_, - `local `_, - `lxc `_, - `lxd `_, - and `ssh `_ - built-in connection types are supported, along with Mitogen-specific - :ref:`machinectl `, :ref:`mitogen_doas `, +* The ``sudo`` comands executed differ slightly compared to Ansible. In some + cases where the target has a ``sudo`` configuration that restricts the exact + commands allowed to run, it may be necessary to add a ``sudoers`` rule like: + + :: + + your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c* + +* The :ans:conn:`~buildah`, :ans:conn:`~docker`, :ans:conn:`~jail`, + :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, and + :ans:conn:`~ssh` built-in connection types are supported, along with + Mitogen-specific :ref:`machinectl `, :ref:`mitogen_doas `, :ref:`mitogen_su `, :ref:`mitogen_sudo `, and :ref:`setns ` types. File bugs to register interest in others. @@ -195,16 +199,14 @@ Noteworthy Differences artificial serialization, causing slowdown equivalent to `task_duration * num_targets`. This will be addressed soon. -* The Ansible 2.7 `reboot - `_ module - may require a ``pre_reboot_delay`` on systemd hosts, as insufficient time - exists for the reboot command's exit status to be reported before necessary - processes are torn down. +* The Ansible 2.7 :ans:mod:`reboot` may require a ``pre_reboot_delay`` on + systemd hosts, as insufficient time exists for the reboot command's exit + status to be reported before necessary processes are torn down. * On OS X when a SSH password is specified and the default connection type of - ``smart`` is used, Ansible may select the Paramiko plug-in rather than - Mitogen. If you specify a password on OS X, ensure ``connection: ssh`` - appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the + :ans:conn:`~smart` is used, Ansible may select the :ans:conn:`paramiko_ssh` + rather than Mitogen. If you specify a password on OS X, ensure ``connection: + ssh`` appears in your playbook, ``ansible.cfg``, or as ``-c ssh`` on the command-line. * Ansible permits up to ``forks`` connections to be setup in parallel, whereas @@ -341,19 +343,12 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally `sftp(1)`_ or `scp(1)`_ are used to copy files by the -`assemble `_, -`copy `_, -`patch `_, -`script `_, -`template `_, and -`unarchive `_ -actions, or when uploading modules with pipelining disabled. With Mitogen -copies are implemented natively using the same interpreters, connection tree, -and routed message bus that carries RPCs. - -.. _scp(1): https://linux.die.net/man/1/scp -.. _sftp(1): https://linux.die.net/man/1/sftp +Normally :linux:man1:`sftp` or :linux:man1:`scp` are used to copy files by the +:ans:mod:`~assemble`, :ans:mod:`~aws_s3`, :ans:mod:`~copy`, :ans:mod:`~patch`, +:ans:mod:`~script`, :ans:mod:`~template`, :ans:mod:`~unarchive`, and +:ans:mod:`~uri` actions, or when uploading modules with pipelining disabled. +With Mitogen copies are implemented natively using the same interpreters, +connection tree, and routed message bus that carries RPCs. This permits direct streaming between endpoints regardless of execution environment, without necessitating temporary copies in intermediary accounts or @@ -369,15 +364,15 @@ Safety ^^^^^^ Transfers proceed to a hidden file in the destination directory, with content -and metadata synced using `fsync(2) `_ prior -to rename over any existing file. This ensures the file remains consistent at -all times, in the event of a crash, or when overlapping `ansible-playbook` runs -deploy differing file contents. +and metadata synced using :linux:man2:`fsync` prior to rename over any existing +file. This ensures the file remains consistent at all times, in the event of a +crash, or when overlapping `ansible-playbook` runs deploy differing file +contents. -The `sftp(1)`_ and `scp(1)`_ tools may cause undetected data corruption -in the form of truncated files, or files containing intermingled data segments -from overlapping runs. As part of normal operation, both tools expose a window -where readers may observe inconsistent file contents. +The :linux:man1:`sftp` and :linux:man1:`scp` tools may cause undetected data +corruption in the form of truncated files, or files containing intermingled +data segments from overlapping runs. As part of normal operation, both tools +expose a window where readers may observe inconsistent file contents. Performance @@ -495,11 +490,11 @@ Ansible may: * Create a directory owned by the SSH user either under ``remote_tmp``, or a system-default directory, * Upload action dependencies such as non-new style modules or rendered - templates to that directory via `sftp(1)`_ or `scp(1)`_. + templates to that directory via :linux:man1:`sftp` or :linux:man1:`scp`. * Attempt to modify the directory's access control list to grant access to the - target user using `setfacl(1) `_, - requiring that tool to be installed and a supported filesystem to be in use, - or for the ``allow_world_readable_tmpfiles`` setting to be :data:`True`. + target user using :linux:man1:`setfacl`, requiring that tool to be installed + and a supported filesystem to be in use, or for the + ``allow_world_readable_tmpfiles`` setting to be :data:`True`. * Create a directory owned by the target user either under ``remote_tmp``, or a system-default directory, if a new-style module needs a temporary directory and one was not previously created for a supporting file earlier in the @@ -565,9 +560,9 @@ in regular Ansible: operations relating to modifying the directory to support cross-account access are avoided. -* An explicit work-around is included to avoid the `copy` and `template` - actions needlessly triggering a round-trip to set their temporary file as - executable. +* An explicit work-around is included to avoid the :ans:mod:`~copy` and + :ans:mod:`~template` actions needlessly triggering a round-trip to set their + temporary file as executable. * During task shutdown, it is not necessary to wait to learn if the target has succeeded in deleting a temporary directory, since any error that may occur @@ -597,10 +592,10 @@ DNS Resolution ^^^^^^^^^^^^^^ Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to -be reloaded via `res_init(3) `_. This -isn't necessary on some Linux distributions carrying glibc patches to -automatically check ``/etc/resolv.conf`` periodically, however it is necessary -on at least Debian and BSD derivatives. +be reloaded via :linux:man3:`res_init`. This isn't necessary on some Linux +distributions carrying glibc patches to automatically check +``/etc/resolv.conf`` periodically, however it is necessary on at least Debian +and BSD derivatives. ``/etc/environment`` @@ -719,6 +714,17 @@ establishment of additional reuseable interpreters as necessary to match the configuration of each task. +.. _method-buildah: + +Buildah +~~~~~~~ + +Like the :ans:conn:`buildah` except connection delegation is supported. + +* ``ansible_host``: Name of Buildah container (default: inventory hostname). +* ``ansible_user``: Name of user within the container to execute as. + + .. _doas: Doas @@ -730,7 +736,7 @@ as a become method. When used as a become method: * ``ansible_python_interpreter`` -* ``ansible_become_exe``: path to ``doas`` binary. +* ``ansible_become_exe`` / ``ansible_doas_exe``: path to ``doas`` binary. * ``ansible_become_user`` (default: ``root``) * ``ansible_become_pass`` (default: assume passwordless) * ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the @@ -745,6 +751,7 @@ When used as the ``mitogen_doas`` connection method: * The inventory hostname has no special meaning. * ``ansible_user``: username to use. * ``ansible_password``: password to use. +* ``ansible_doas_exe``: path to ``doas`` binary. * ``ansible_python_interpreter`` @@ -753,9 +760,7 @@ When used as the ``mitogen_doas`` connection method: Docker ~~~~~~ -Like `docker -`_ except -connection delegation is supported. +Like the :ans:conn:`docker` except connection delegation is supported. * ``ansible_host``: Name of Docker container (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. @@ -771,9 +776,7 @@ connection delegation is supported. FreeBSD Jail ~~~~~~~~~~~~ -Like `jail -`_ except -connection delegation is supported. +Like the :ans:conn:`jail` except connection delegation is supported. * ``ansible_host``: Name of jail (default: inventory hostname). * ``ansible_user``: Name of user within the jail to execute as. @@ -789,9 +792,7 @@ connection delegation is supported. Kubernetes Pod ~~~~~~~~~~~~~~ -Like `kubectl -`_ except -connection delegation is supported. +Like the :ans:conn:`kubectl` except connection delegation is supported. * ``ansible_host``: Name of pod (default: inventory hostname). * ``ansible_user``: Name of user to authenticate to API as. @@ -805,9 +806,7 @@ connection delegation is supported. Local ~~~~~ -Like `local -`_ except -connection delegation is supported. +Like the :ans:conn:`local` except connection delegation is supported. * ``ansible_python_interpreter`` @@ -834,10 +833,9 @@ additional differences exist that may break existing playbooks. LXC ~~~ -Connect to classic LXC containers, like `lxc -`_ except -connection delegation is supported, and ``lxc-attach`` is always used rather -than the LXC Python bindings, as is usual with ``lxc``. +Connect to classic LXC containers, like the :ans:conn:`lxc` except connection +delegation is supported, and ``lxc-attach`` is always used rather than the LXC +Python bindings, as is usual with ``lxc``. * ``ansible_python_interpreter`` * ``ansible_host``: Name of LXC container (default: inventory hostname). @@ -855,10 +853,9 @@ than the LXC Python bindings, as is usual with ``lxc``. LXD ~~~ -Connect to modern LXD containers, like `lxd -`_ except -connection delegation is supported. The ``lxc`` command must be available on -the host machine. +Connect to modern LXD containers, like the :ans:conn:`lxd` except connection +delegation is supported. The ``lxc`` command must be available on the host +machine. * ``ansible_python_interpreter`` * ``ansible_host``: Name of LXC container (default: inventory hostname). @@ -983,8 +980,7 @@ When used as the ``mitogen_sudo`` connection method: SSH ~~~ -Like `ssh `_ -except connection delegation is supported. +Like the :ans:conn:`ssh` except connection delegation is supported. * ``ansible_ssh_timeout`` * ``ansible_host``, ``ansible_ssh_host`` @@ -1005,6 +1001,11 @@ except connection delegation is supported. otherwise :data:`False`. This will change to off by default in a future release. If you are targetting many hosts on a fast network, please consider disabling SSH compression. +* ``mitogen_ssh_keepalive_count``: integer count of server keepalive messages to + which no reply is received before considering the SSH server dead. Defaults + to 10. +* ``mitogen_ssh_keepalive_count``: integer seconds delay between keepalive + messages. Defaults to 30. Debugging @@ -1368,3 +1369,19 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less bandwidth and 1.8x less time**. .. image:: images/ansible/pcaps/costapp-uk-india.svg + + +.. raw:: html + + + diff --git a/docs/api.rst b/docs/api.rst index db39ad99..7ab3274e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -2,11 +2,6 @@ API Reference ************* -.. toctree:: - :hidden: - - signals - Package Layout ============== @@ -31,29 +26,10 @@ mitogen.core .. automodule:: mitogen.core .. currentmodule:: mitogen.core -.. decorator:: takes_econtext - - Decorator that marks a function or class method to automatically receive a - kwarg named `econtext`, referencing the - :class:`mitogen.core.ExternalContext` active in the context in which the - function is being invoked in. The decorator is only meaningful when the - function is invoked via :data:`CALL_FUNCTION - `. - - When the function is invoked directly, `econtext` must still be passed to - it explicitly. +.. autodecorator:: takes_econtext .. currentmodule:: mitogen.core -.. decorator:: takes_router - - Decorator that marks a function or class method to automatically receive a - kwarg named `router`, referencing the :class:`mitogen.core.Router` - active in the context in which the function is being invoked in. The - decorator is only meaningful when the function is invoked via - :data:`CALL_FUNCTION `. - - When the function is invoked directly, `router` must still be passed to it - explicitly. +.. autodecorator:: takes_router mitogen.master @@ -96,8 +72,12 @@ Router Class :members: -.. currentmodule:: mitogen.master +.. currentmodule:: mitogen.parent +.. autoclass:: Router + :members: + +.. currentmodule:: mitogen.master .. autoclass:: Router (broker=None) :members: @@ -107,6 +87,20 @@ Router Class Connection Methods ================== +.. currentmodule:: mitogen.parent +.. method:: Router.buildah (container=None, buildah_path=None, username=None, \**kwargs) + + Construct a context on the local machine over a ``buildah`` invocation. + Accepts all parameters accepted by :meth:`local`, in addition to: + + :param str container: + The name of the Buildah container to connect to. + :param str doas_path: + Filename or complete path to the ``buildah`` binary. ``PATH`` will be + searched if given as a filename. Defaults to ``buildah``. + :param str username: + Username to use, defaults to unset. + .. currentmodule:: mitogen.parent .. method:: Router.fork (on_fork=None, on_start=None, debug=False, profiling=False, via=None) @@ -383,6 +377,9 @@ Connection Methods the root PID of a running Docker, LXC, LXD, or systemd-nspawn container. + The setns method depends on the built-in :mod:`ctypes` module, and thus + does not support Python 2.4. + A program is required only to find the root PID, after which management of the child Python interpreter is handled directly. @@ -550,11 +547,11 @@ Context Class .. currentmodule:: mitogen.parent - -.. autoclass:: CallChain +.. autoclass:: Context :members: -.. autoclass:: Context +.. currentmodule:: mitogen.parent +.. autoclass:: CallChain :members: @@ -620,6 +617,14 @@ Fork Safety Utility Functions ================= +.. currentmodule:: mitogen.core +.. function:: now + + A reference to :func:`time.time` on Python 2, or :func:`time.monotonic` on + Python >3.3. We prefer :func:`time.monotonic` when available to ensure + timers are not impacted by system clock changes. + + .. module:: mitogen.utils A random assortment of utility functions useful on masters and children. @@ -659,3 +664,7 @@ Exceptions .. autoclass:: LatchError .. autoclass:: StreamError .. autoclass:: TimeoutError + +.. currentmodule:: mitogen.parent +.. autoclass:: EofError +.. autoclass:: CancelledError diff --git a/docs/changelog.rst b/docs/changelog.rst index 0b20a852..fe15ca27 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,12 +15,286 @@ Release Notes -v0.2.8 (unreleased) +v0.2.9 (unreleased) ------------------- To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +*(no changes)* + + +v0.2.8 (2019-08-18) +------------------- + +This release includes Ansible 2.8 and SELinux support, fixes for two deadlocks, +and major internal design overhauls in preparation for future functionality. + + +Enhancements +~~~~~~~~~~~~ + +* :gh:issue:`556`, + :gh:issue:`587`: Ansible 2.8 is supported. `Become plugins + `_ and + `interpreter discovery + `_ + are not yet handled. + +* :gh:issue:`419`, :gh:issue:`470`, file descriptor usage is approximately + halved, as it is no longer necessary to separately manage read and write + sides to work around a design problem. + +* :gh:issue:`419`: setup for all connections happens almost entirely on one + thread, reducing contention and context switching early in a run. + +* :gh:issue:`419`: Connection setup is better pipelined, eliminating some + network round-trips. Most infrastructure is in place to support future + removal of the final round-trips between a target booting and receiving + function calls. + +* :gh:pull:`595`: the :meth:`~mitogen.parent.Router.buildah` connection method + is available to manipulate `Buildah `_ containers, and + is exposed to Ansible as the :ans:conn:`buildah`. + +* :gh:issue:`615`: a modified :ans:mod:`fetch` implements streaming transfer + even when ``become`` is active, avoiding excess CPU usage and memory spikes, + and improving performance. A copy of two 512 MiB files drops from 47 seconds + to 7 seconds, with peak memory usage dropping from 10.7 GiB to 64.8 MiB. + +* `Operon `_ no longer requires a custom + library installation, both Ansible and Operon are supported by a single + Mitogen release. + +* The ``MITOGEN_CPU_COUNT`` variable shards the connection multiplexer into + per-CPU workers. This may improve throughput for large runs involving file + transfer, and is required for future functionality. One multiplexer starts by + default, to match existing behaviour. + +* :gh:commit:`d6faff06`, :gh:commit:`807cbef9`, :gh:commit:`e93762b3`, + :gh:commit:`50bfe4c7`: locking is avoided on hot paths, and some locks are + released before waking a thread that must immediately acquire the same lock. + + +Mitogen for Ansible +~~~~~~~~~~~~~~~~~~~ + +* :gh:issue:`363`: fix an obscure race matching *Permission denied* errors from + some versions of ``su`` running on heavily loaded machines. + +* :gh:issue:`410`: Uses of :linux:man7:`unix` sockets are replaced with + traditional :linux:man7:`pipe` pairs when SELinux is detected, to work around + a broken heuristic in common SELinux policies that prevents inheriting + :linux:man7:`unix` sockets across privilege domains. + +* `#467 `_: an incompatibility + running Mitogen under `Molecule + `_ was resolved. + +* :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of + connections, ``async`` tasks, tasks using custom :mod:`module_utils`, + ``mitogen_task_isolation: fork`` modules, and modules present on an internal + blacklist. This would manifest as a timeout or hang, was easily hit, had been + present since 0.2.0, and likely impacted many users. + +* :gh:issue:`549`: the open file limit is increased to the permitted hard + limit. It is common for distributions to ship with a higher hard limit than + the default soft limit, allowing *"too many open files"* errors to be avoided + more often in large runs without user intervention. + +* :gh:issue:`558`, :gh:issue:`582`: on Ansible 2.3 a directory was + unconditionally deleted after the first module belonging to an action plug-in + had executed, causing the :ans:mod:`unarchive` to fail. + +* :gh:issue:`578`: the extension could crash while rendering an error due to an + incorrect format string. + +* :gh:issue:`590`: the importer can handle modules that replace themselves in + :data:`sys.modules` with completely unrelated modules during import, as in + the case of Ansible 2.8 :mod:`ansible.module_utils.distro`. + +* :gh:issue:`591`: the working directory is reset between tasks to ensure + :func:`os.getcwd` cannot fail, in the same way :class:`AnsibleModule` + resets it during initialization. However this restore happens before the + module executes, ensuring code that calls :func:`os.getcwd` prior to + :class:`AnsibleModule` initialization, such as the Ansible 2.7 + :ans:mod:`pip`, cannot fail due to the actions of a prior task. + +* :gh:issue:`593`: the SSH connection method exposes + ``mitogen_ssh_keepalive_interval`` and ``mitogen_ssh_keepalive_count`` + variables, and the default timeout for an SSH server has been increased from + `15*3` seconds to `30*10` seconds. + +* :gh:issue:`600`: functionality to reflect changes to ``/etc/environment`` did + not account for Unicode file contents. The file may now use any single byte + encoding. + +* :gh:issue:`602`: connection configuration is more accurately inferred for + :ans:mod:`meta: reset_connection ` the :ans:mod:`synchronize`, and for + any action plug-ins that establish additional connections. + +* :gh:issue:`598`, :gh:issue:`605`: fix a deadlock managing a shared counter + used for load balancing, present since 0.2.4. + +* :gh:issue:`615`: streaming is implemented for the :ans:mod:`fetch` and other + actions that transfer files from targets to the controller. Previously files + delivered were sent in one message, requiring them to fit in RAM and be + smaller than an internal message size sanity check. Transfers from controller + to targets have been streaming since 0.2.0. + +* :gh:commit:`7ae926b3`: the :ans:mod:`lineinfile` leaks writable temporary + file descriptors since Ansible 2.7.0. When :ans:mod:`~lineinfile` created or + modified a script, and that script was later executed, the execution could + fail with "*text file busy*". Temporary descriptors are now tracked and + cleaned up on exit for all modules. + + +Core Library +~~~~~~~~~~~~ + +* Log readability is improving and many :func:`repr` strings are more + descriptive. The old pseudo-function-call format is migrating to + readable output where possible. For example, *"Stream(ssh:123).connect()"* + might be written *"connecting to ssh:123"*. + +* In preparation for reducing default log output, many messages are delivered + to per-component loggers, including messages originating from children, + enabling :mod:`logging` aggregation to function as designed. An importer + message like:: + + 12:00:00 D mitogen.ctx.remotehost mitogen: loading module "foo" + + Might instead be logged to the ``mitogen.importer.[remotehost]`` logger:: + + 12:00:00 D mitogen.importer.[remotehost] loading module "foo" + + Allowing a filter or handler for ``mitogen.importer`` to select that logger + in every process. This introduces a small risk of leaking memory in + long-lived programs, as logger objects are internally persistent. + +* :func:`bytearray` was removed from the list of supported serialization types. + It was never portable between Python versions, unused, and never made much + sense to support. + +* :gh:issue:`170`: to improve subprocess + management and asynchronous connect, a :class:`~mitogen.parent.TimerList` + interface is available, accessible as :attr:`Broker.timers` in an + asynchronous context. + +* :gh:issue:`419`: the internal + :class:`~mitogen.core.Stream` has been refactored into many new classes, + modularizing protocol behaviour, output buffering, line-oriented input + parsing, option handling and connection management. Connection setup is + internally asynchronous, laying most groundwork for fully asynchronous + connect, proxied Ansible become plug-ins, and in-process SSH. + +* :gh:issue:`169`, + :gh:issue:`419`: zombie subprocess reaping + has vastly improved, by using timers to efficiently poll for a child to exit, + and delaying shutdown while any subprocess remains. Polling avoids + process-global configuration such as a `SIGCHLD` handler, or + :func:`signal.set_wakeup_fd` available in modern Python. + +* :gh:issue:`256`, :gh:issue:`419`: most :func:`os.dup` use was eliminated, + along with most manual file descriptor management. Descriptors are trapped in + :func:`os.fdopen` objects at creation, ensuring a leaked object will close + itself, and ensuring every descriptor is fused to a `closed` flag, preventing + historical bugs where a double close could destroy unrelated descriptors. + +* :gh:issue:`533`: routing accounts for + a race between a parent (or cousin) sending a message to a child via an + intermediary, where the child had recently disconnected, and + :data:`~mitogen.core.DEL_ROUTE` propagating from the intermediary + to the sender, informing it that the child no longer exists. This condition + is detected at the intermediary and a :ref:`dead message ` is + returned to the sender. + + Previously since the intermediary had already removed its route for the + child, the *route messages upwards* rule would be triggered, causing the + message (with a privileged :ref:`src_id/auth_id `) to be + sent upstream, resulting in a ``bad auth_id`` error logged at the first + upstream parent, and a possible hang due to a request message being dropped. + +* :gh:issue:`586`: fix import of + :mod:`__main__` on later versions of Python 3 when running from the + interactive console. + +* :gh:issue:`606`: fix example code on the + documentation front page. + +* :gh:issue:`612`: fix various errors + introduced by stream refactoring. + +* :gh:issue:`615`: when routing fails to + deliver a message for some reason other than the sender cannot or should not + reach the recipient, and no reply-to address is present on the message, + instead send a :ref:`dead message ` to the original recipient. This + ensures a descriptive message is delivered to a thread sleeping on the reply + to a function call, where the reply might be dropped due to exceeding the + maximum configured message size. + +* :gh:issue:`624`: the number of threads used for a child's automatically + initialized service thread pool has been reduced from 16 to 2. This may drop + to 1 in future, and become configurable via a :class:`Router` option. + +* :gh:commit:`a5536c35`: avoid quadratic + buffer management when logging lines received from a child's redirected + standard IO. + +* :gh:commit:`49a6446a`: the + :meth:`empty` methods of :class:`~mitogen.core.Latch`, + :class:`~mitogen.core.Receiver` and :class:`~mitogen.select.Select` are + obsoleted by a more general :meth:`size` method. :meth:`empty` will be + removed in 0.3 + +* :gh:commit:`ecc570cb`: previously + :meth:`mitogen.select.Select.add` would enqueue one wake event when adding an + existing receiver, latch or subselect that contained multiple buffered items, + causing :meth:`get` calls to block or fail even though data existed to return. + +* :gh:commit:`5924af15`: *[security]* + unidirectional routing, where contexts may optionally only communicate with + parents and never siblings (so that air-gapped networks cannot be + unintentionally bridged) was not inherited when a child was initiated + directly from an another child. This did not effect Ansible, since the + controller initiates any new child used for routing, only forked tasks are + initiated by children. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +bug reports, testing, features and fixes in this release contributed by +`Andreas Hubert `_. +`Anton Markelov `_, +`Dan `_, +`Dave Cottlehuber `_, +`Denis Krienbühl `_, +`El Mehdi CHAOUKI `_, +`Florent Dutheil `_, +`James Hogarth `_, +`Jordan Webb `_, +`Julian Andres Klode `_, +`Marc Hartmayer `_, +`Nigel Metheringham `_, +`Orion Poplawski `_, +`Pieter Voet `_, +`Stefane Fermigier `_, +`Szabó Dániel Ernő `_, +`Ulrich Schreiner `_, +`Vincent S. Cojot `_, +`yen `_, +`Yuki Nishida `_, +`@alexhexabeam `_, +`@DavidVentura `_, +`@dbiegunski `_, +`@ghp-rr `_, +`@migalsp `_, +`@rizzly `_, +`@SQGE `_, and +`@tho86 `_. + v0.2.7 (2019-05-19) ------------------- @@ -31,27 +305,27 @@ on Ansible 2.8, which is not yet supported. Fixes ~~~~~ -* `#557 `_: fix a crash when running +* :gh:issue:`557`: fix a crash when running on machines with high CPU counts. -* `#570 `_: the ``firewalld`` module - internally caches a dbus name that changes across ``firewalld`` restarts, - causing a failure if the service is restarted between ``firewalld`` module invocations. +* :gh:issue:`570`: the :ans:mod:`firewalld` internally caches a dbus name that + changes across :ans:mod:`~firewalld` restarts, causing a failure if the + service is restarted between :ans:mod:`~firewalld` module invocations. -* `#575 `_: fix a crash when +* :gh:issue:`575`: fix a crash when rendering an error message to indicate no usable temporary directories could be found. -* `#576 `_: fix a crash during +* :gh:issue:`576`: fix a crash during startup on SuSE Linux 11, due to an incorrect version compatibility check in the Mitogen code. -* `#581 `_: a +* :gh:issue:`581`: a ``mitogen_mask_remote_name`` Ansible variable is exposed, to allow masking the username, hostname and process ID of ``ansible-playbook`` running on the controller machine. -* `#587 `_: display a friendly +* :gh:issue:`587`: display a friendly message when running on an unsupported version of Ansible, to cope with potential influx of 2.8-related bug reports. @@ -73,53 +347,53 @@ v0.2.6 (2019-03-06) Fixes ~~~~~ -* `#542 `_: some versions of OS X +* :gh:issue:`542`: some versions of OS X ship a default Python that does not support :func:`select.poll`. Restore the 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer :func:`select.poll` if it is available. -* `#545 `_: an optimization - introduced in `#493 `_ caused a +* :gh:issue:`545`: an optimization + introduced in :gh:issue:`493` caused a 64-bit integer to be assigned to a 32-bit field on ARM 32-bit targets, causing runs to fail. -* `#548 `_: `mitogen_via=` could fail +* :gh:issue:`548`: `mitogen_via=` could fail when the selected transport was set to ``smart``. -* `#550 `_: avoid some broken +* :gh:issue:`550`: avoid some broken TTY-related `ioctl()` calls on Windows Subsystem for Linux 2016 Anniversary Update. -* `#554 `_: third party Ansible +* :gh:issue:`554`: third party Ansible action plug-ins that invoked :func:`_make_tmp_path` repeatedly could trigger an assertion failure. -* `#555 `_: work around an old idiom +* :gh:issue:`555`: work around an old idiom that reloaded :mod:`sys` in order to change the interpreter's default encoding. -* `ffae0355 `_: needless +* :gh:commit:`ffae0355`: needless information was removed from the documentation and installation procedure. Core Library ~~~~~~~~~~~~ -* `#535 `_: to support function calls +* :gh:issue:`535`: to support function calls on a service pool from another thread, :class:`mitogen.select.Select` additionally permits waiting on :class:`mitogen.core.Latch`. -* `#535 `_: +* :gh:issue:`535`: :class:`mitogen.service.Pool.defer` allows any function to be enqueued for the thread pool from another thread. -* `#535 `_: a new +* :gh:issue:`535`: a new :mod:`mitogen.os_fork` module provides a :func:`os.fork` wrapper that pauses thread activity during fork. On Python<2.6, :class:`mitogen.core.Broker` and :class:`mitogen.service.Pool` automatically record their existence so that a :func:`os.fork` monkey-patch can automatically pause them for any attempt to start a subprocess. -* `ca63c26e `_: +* :gh:commit:`ca63c26e`: :meth:`mitogen.core.Latch.put`'s `obj` argument was made optional. @@ -144,47 +418,47 @@ v0.2.5 (2019-02-14) Fixes ~~~~~ -* `#511 `_, - `#536 `_: changes in 0.2.4 to +* :gh:issue:`511`, + :gh:issue:`536`: changes in 0.2.4 to repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. -* `#532 `_: fix a race in the service +* :gh:issue:`532`: fix a race in the service used to propagate Ansible modules, that could easily manifest when starting asynchronous tasks in a loop. -* `#536 `_: changes in 0.2.4 to +* :gh:issue:`536`: changes in 0.2.4 to support Python 2.4 interacted poorly with modules that imported ``simplejson`` from a controller that also loaded an incompatible newer version of ``simplejson``. -* `#537 `_: a swapped operator in the +* :gh:issue:`537`: a swapped operator in the CPU affinity logic meant 2 cores were reserved on 1`_: the source distribution +* :gh:issue:`538`: the source distribution includes a ``LICENSE`` file. -* `#539 `_: log output is no longer +* :gh:issue:`539`: log output is no longer duplicated when the Ansible ``log_path`` setting is enabled. -* `#540 `_: the ``stderr`` stream of +* :gh:issue:`540`: the ``stderr`` stream of async module invocations was previously discarded. -* `#541 `_: Python error logs +* :gh:issue:`541`: Python error logs originating from the ``boto`` package are quiesced, and only appear in ``-vvv`` output. This is since EC2 modules may trigger errors during normal operation, when retrying transiently failing requests. -* `748f5f67 `_, - `21ad299d `_, - `8ae6ca1d `_, - `7fd0d349 `_: +* :gh:commit:`748f5f67`, + :gh:commit:`21ad299d`, + :gh:commit:`8ae6ca1d`, + :gh:commit:`7fd0d349`: the ``ansible_ssh_host``, ``ansible_ssh_user``, ``ansible_user``, ``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly match typical behaviour when ``mitogen_via=`` is active. -* `2a8567b4 `_: fix a race +* :gh:commit:`2a8567b4`: fix a race initializing a child's service thread pool on Python 3.4+, due to a change in locking scheme used by the Python import mechanism. @@ -212,57 +486,54 @@ on the connection multiplexer. Enhancements ^^^^^^^^^^^^ -* `#76 `_, - `#351 `_, - `#352 `_: disconnect propagation +* :gh:issue:`76`, + :gh:issue:`351`, + :gh:issue:`352`: disconnect propagation has improved, allowing Ansible to cancel waits for responses from abruptly disconnected targets. This ensures a task will reliably fail rather than hang, for example on network failure or EC2 instance maintenance. -* `#369 `_, - `#407 `_: :meth:`Connection.reset` - is implemented, allowing `meta: reset_connection - `_ to shut +* :gh:issue:`369`, + :gh:issue:`407`: :meth:`Connection.reset` + is implemented, allowing :ans:mod:`meta: reset_connection ` to shut down the remote interpreter as documented, and improving support for the - `reboot - `_ - module. + :ans:mod:`reboot`. -* `09aa27a6 `_: the +* :gh:commit:`09aa27a6`: the ``mitogen_host_pinned`` strategy wraps the ``host_pinned`` strategy introduced in Ansible 2.7. -* `#477 `_: Python 2.4 is fully +* :gh:issue:`477`: Python 2.4 is fully supported by the core library and tested automatically, in any parent/child combination of 2.4, 2.6, 2.7 and 3.6 interpreters. -* `#477 `_: Ansible 2.3 is fully +* :gh:issue:`477`: Ansible 2.3 is fully supported and tested automatically. In combination with the core library Python 2.4 support, this allows Red Hat Enterprise Linux 5 targets to be managed with Mitogen. The ``simplejson`` package need not be installed on such targets, as is usually required by Ansible. -* `#412 `_: to simplify diagnosing +* :gh:issue:`412`: to simplify diagnosing connection configuration problems, Mitogen ships a ``mitogen_get_stack`` action that is automatically added to the action plug-in path. See :ref:`mitogen-get-stack` for more information. -* `152effc2 `_, - `bd4b04ae `_: a CPU affinity +* :gh:commit:`152effc2`, + :gh:commit:`bd4b04ae`: a CPU affinity policy was added for Linux controllers, reducing latency and SMP overhead on hot paths exercised for every task. This yielded a 19% speedup in a 64-target job composed of many short tasks, and should easily be visible as a runtime improvement in many-host runs. -* `2b44d598 `_: work around a +* :gh:commit:`2b44d598`: work around a defective caching mechanism by pre-heating it before spawning workers. This saves 40% runtime on a synthetic repetitive task. -* `0979422a `_: an expensive +* :gh:commit:`0979422a`: an expensive dependency scanning step was redundantly invoked for every task, bottlenecking the connection multiplexer. -* `eaa990a97 `_: a new +* :gh:commit:`eaa990a97`: a new ``mitogen_ssh_compression`` variable is supported, allowing Mitogen's default SSH compression to be disabled. SSH compression is a large contributor to CPU usage in many-target runs, and severely limits file transfer. On a `"shell: @@ -270,124 +541,115 @@ Enhancements task with compression, rising to 3 KiB without. File transfer throughput rises from ~25MiB/s when enabled to ~200MiB/s when disabled. -* `#260 `_, - `a18a083c `_: brokers no +* :gh:issue:`260`, + :gh:commit:`a18a083c`: brokers no longer wait for readiness indication to transmit, and instead assume transmission will succeed. As this is usually true, one loop iteration and two poller reconfigurations are avoided, yielding a significant reduction in interprocess round-trip latency. -* `#415 `_, - `#491 `_, - `#493 `_: the interface employed - for in-process queues changed from `kqueue - `_ / `epoll - `_ to `poll() - `_, which requires no setup - or teardown, yielding a 38% latency reduction for inter-thread communication. +* :gh:issue:`415`, :gh:issue:`491`, :gh:issue:`493`: the interface employed + for in-process queues changed from :freebsd:man2:`kqueue` / + :linux:man7:`epoll` to :linux:man2:`poll`, which requires no setup or + teardown, yielding a 38% latency reduction for inter-thread communication. Fixes ^^^^^ -* `#251 `_, - `#359 `_, - `#396 `_, - `#401 `_, - `#404 `_, - `#412 `_, - `#434 `_, - `#436 `_, - `#465 `_: connection delegation and +* :gh:issue:`251`, + :gh:issue:`359`, + :gh:issue:`396`, + :gh:issue:`401`, + :gh:issue:`404`, + :gh:issue:`412`, + :gh:issue:`434`, + :gh:issue:`436`, + :gh:issue:`465`: connection delegation and ``delegate_to:`` handling suffered a major regression in 0.2.3. The 0.2.2 behaviour has been restored, and further work has been made to improve the compatibility of connection delegation's configuration building methods. -* `#323 `_, - `#333 `_: work around a Windows +* :gh:issue:`323`, + :gh:issue:`333`: work around a Windows Subsystem for Linux bug that caused tracebacks to appear during shutdown. -* `#334 `_: the SSH method +* :gh:issue:`334`: the SSH method tilde-expands private key paths using Ansible's logic. Previously the path was passed unmodified to SSH, which expanded it using :func:`pwd.getpwnam`. This differs from :func:`os.path.expanduser`, which uses the ``HOME`` environment variable if it is set, causing behaviour to diverge when Ansible was invoked across user accounts via ``sudo``. -* `#364 `_: file transfers from +* :gh:issue:`364`: file transfers from controllers running Python 2.7.2 or earlier could be interrupted due to a forking bug in the :mod:`tempfile` module. -* `#370 `_: the Ansible - `reboot `_ - module is supported. +* :gh:issue:`370`: the Ansible :ans:mod:`reboot` is supported. -* `#373 `_: the LXC and LXD methods - print a useful hint on failure, as no useful error is normally logged to the - console by these tools. +* :gh:issue:`373`: the LXC and LXD methods print a useful hint on failure, as + no useful error is normally logged to the console by these tools. -* `#374 `_, - `#391 `_: file transfer and module +* :gh:issue:`374`, + :gh:issue:`391`: file transfer and module execution from 2.x controllers to 3.x targets was broken due to a regression - caused by refactoring, and compounded by `#426 - `_. + caused by refactoring, and compounded by :gh:issue:`426`. -* `#400 `_: work around a threading +* :gh:issue:`400`: work around a threading bug in the AWX display callback when running with high verbosity setting. -* `#409 `_: the setns method was +* :gh:issue:`409`: the setns method was silently broken due to missing tests. Basic coverage was added to prevent a recurrence. -* `#409 `_: the LXC and LXD methods +* :gh:issue:`409`: the LXC and LXD methods support ``mitogen_lxc_path`` and ``mitogen_lxc_attach_path`` variables to control the location of third pary utilities. -* `#410 `_: the sudo method supports +* :gh:issue:`410`: the sudo method supports the SELinux ``--type`` and ``--role`` options. -* `#420 `_: if a :class:`Connection` +* :gh:issue:`420`: if a :class:`Connection` was constructed in the Ansible top-level process, for example while executing ``meta: reset_connection``, resources could become undesirably shared in subsequent children. -* `#426 `_: an oversight while +* :gh:issue:`426`: an oversight while porting to Python 3 meant no automated 2->3 tests were running. A significant number of 2->3 bugs were fixed, mostly in the form of Unicode/bytes mismatches. -* `#429 `_: the ``sudo`` method can +* :gh:issue:`429`: the ``sudo`` method can now recognize internationalized password prompts. -* `#362 `_, - `#435 `_: the previous fix for slow +* :gh:issue:`362`, + :gh:issue:`435`: the previous fix for slow Python 2.x subprocess creation on Red Hat caused newly spawned children to have a reduced open files limit. A more intrusive fix has been added to directly address the problem without modifying the subprocess environment. -* `#397 `_, - `#454 `_: the previous approach to +* :gh:issue:`397`, + :gh:issue:`454`: the previous approach to handling modern Ansible temporary file cleanup was too aggressive, and could trigger early finalization of Cython-based extension modules, leading to segmentation faults. -* `#499 `_: the ``allow_same_user`` +* :gh:issue:`499`: the ``allow_same_user`` Ansible configuration setting is respected. -* `#527 `_: crashes in modules are +* :gh:issue:`527`: crashes in modules are trapped and reported in a manner that matches Ansible. In particular, a module crash no longer leads to an exception that may crash the corresponding action plug-in. -* `dc1d4251 `_: the - ``synchronize`` module could fail with the Docker transport due to a missing - attribute. +* :gh:commit:`dc1d4251`: the :ans:mod:`synchronize` could fail with the Docker + transport due to a missing attribute. -* `599da068 `_: fix a race +* :gh:commit:`599da068`: fix a race when starting async tasks, where it was possible for the controller to observe no status file on disk before the task had a chance to write one. -* `2c7af9f04 `_: Ansible +* :gh:commit:`2c7af9f04`: Ansible modules were repeatedly re-transferred. The bug was hidden by the previously mandatorily enabled SSH compression. @@ -395,7 +657,7 @@ Fixes Core Library ~~~~~~~~~~~~ -* `#76 `_: routing records the +* :gh:issue:`76`: routing records the destination context IDs ever received on each stream, and when disconnection occurs, propagates :data:`mitogen.core.DEL_ROUTE` messages towards every stream that ever communicated with the disappearing peer, rather than simply @@ -404,166 +666,166 @@ Core Library receivers to wake with :class:`mitogen.core.ChannelError`, even when one participant is not a parent of the other. -* `#109 `_, - `57504ba6 `_: newer Python 3 +* :gh:issue:`109`, + :gh:commit:`57504ba6`: newer Python 3 releases explicitly populate :data:`sys.meta_path` with importer internals, causing Mitogen to install itself at the end of the importer chain rather than the front. -* `#310 `_: support has returned for +* :gh:issue:`310`: support has returned for trying to figure out the real source of non-module objects installed in :data:`sys.modules`, so they can be imported. This is needed to handle syntax sugar used by packages like :mod:`plumbum`. -* `#349 `_: an incorrect format +* :gh:issue:`349`: an incorrect format string could cause large stack traces when attempting to import built-in modules on Python 3. -* `#387 `_, - `#413 `_: dead messages include an +* :gh:issue:`387`, + :gh:issue:`413`: dead messages include an optional reason in their body. This is used to cause :class:`mitogen.core.ChannelError` to report far more useful diagnostics at the point the error occurs that previously would have been buried in debug log output from an unrelated context. -* `#408 `_: a variety of fixes were +* :gh:issue:`408`: a variety of fixes were made to restore Python 2.4 compatibility. -* `#399 `_, - `#437 `_: ignore a +* :gh:issue:`399`, + :gh:issue:`437`: ignore a :class:`DeprecationWarning` to avoid failure of the ``su`` method on Python 3.7. -* `#405 `_: if an oversized message +* :gh:issue:`405`: if an oversized message is rejected, and it has a ``reply_to`` set, a dead message is returned to the sender. This ensures function calls exceeding the configured maximum size crash rather than hang. -* `#406 `_: +* :gh:issue:`406`: :class:`mitogen.core.Broker` did not call :meth:`mitogen.core.Poller.close` during shutdown, leaking the underlying poller FD in masters and parents. -* `#406 `_: connections could leak +* :gh:issue:`406`: connections could leak FDs when a child process failed to start. -* `#288 `_, - `#406 `_, - `#417 `_: connections could leave +* :gh:issue:`288`, + :gh:issue:`406`, + :gh:issue:`417`: connections could leave FD wrapper objects that had not been closed lying around to be closed during garbage collection, causing reused FD numbers to be closed at random moments. -* `#411 `_: the SSH method typed +* :gh:issue:`411`: the SSH method typed "``y``" rather than the requisite "``yes``" when `check_host_keys="accept"` was configured. This would lead to connection timeouts due to the hung response. -* `#414 `_, - `#425 `_: avoid deadlock of forked +* :gh:issue:`414`, + :gh:issue:`425`: avoid deadlock of forked children by reinitializing the :mod:`mitogen.service` pool lock. -* `#416 `_: around 1.4KiB of memory +* :gh:issue:`416`: around 1.4KiB of memory was leaked on every RPC, due to a list of strong references keeping alive any handler ever registered for disconnect notification. -* `#418 `_: the +* :gh:issue:`418`: the :func:`mitogen.parent.iter_read` helper would leak poller FDs, because execution of its :keyword:`finally` block was delayed on Python 3. Now callers explicitly close the generator when finished. -* `#422 `_: the fork method could +* :gh:issue:`422`: the fork method could fail to start if :data:`sys.stdout` was opened in block buffered mode, and buffered data was pending in the parent prior to fork. -* `#438 `_: a descriptive error is +* :gh:issue:`438`: a descriptive error is logged when stream corruption is detected. -* `#439 `_: descriptive errors are +* :gh:issue:`439`: descriptive errors are raised when attempting to invoke unsupported function types. -* `#444 `_: messages regarding +* :gh:issue:`444`: messages regarding unforwardable extension module are no longer logged as errors. -* `#445 `_: service pools unregister +* :gh:issue:`445`: service pools unregister the :data:`mitogen.core.CALL_SERVICE` handle at shutdown, ensuring any outstanding messages are either processed by the pool as it shuts down, or have dead messages sent in reply to them, preventing peer contexts from hanging due to a forgotten buffered message. -* `#446 `_: given thread A calling +* :gh:issue:`446`: given thread A calling :meth:`mitogen.core.Receiver.close`, and thread B, C, and D sleeping in :meth:`mitogen.core.Receiver.get`, previously only one sleeping thread would be woken with :class:`mitogen.core.ChannelError` when the receiver was closed. Now all threads are woken per the docstring. -* `#447 `_: duplicate attempts to +* :gh:issue:`447`: duplicate attempts to invoke :meth:`mitogen.core.Router.add_handler` cause an error to be raised, ensuring accidental re-registration of service pools are reported correctly. -* `#448 `_: the import hook +* :gh:issue:`448`: the import hook implementation now raises :class:`ModuleNotFoundError` instead of :class:`ImportError` in Python 3.6 and above, to cope with an upcoming version of the :mod:`subprocess` module requiring this new subclass to be raised. -* `#453 `_: the loggers used in +* :gh:issue:`453`: the loggers used in children for standard IO redirection have propagation disabled, preventing accidental reconfiguration of the :mod:`logging` package in a child from setting up a feedback loop. -* `#456 `_: a descriptive error is +* :gh:issue:`456`: a descriptive error is logged when :meth:`mitogen.core.Broker.defer` is called after the broker has shut down, preventing new messages being enqueued that will never be sent, and subsequently producing a program hang. -* `#459 `_: the beginnings of a +* :gh:issue:`459`: the beginnings of a :meth:`mitogen.master.Router.get_stats` call has been added. The initial statistics cover the module loader only. -* `#462 `_: Mitogen could fail to +* :gh:issue:`462`: Mitogen could fail to open a PTY on broken Linux systems due to a bad interaction between the glibc :func:`grantpt` function and an incorrectly mounted ``/dev/pts`` filesystem. Since correct group ownership is not required in most scenarios, when this problem is detected, the PTY is allocated and opened directly by the library. -* `#479 `_: Mitogen could fail to +* :gh:issue:`479`: Mitogen could fail to import :mod:`__main__` on Python 3.4 and newer due to a breaking change in the :mod:`pkgutil` API. The program's main script is now handled specially. -* `#481 `_: the version of `sudo` +* :gh:issue:`481`: the version of `sudo` that shipped with CentOS 5 replaced itself with the program to be executed, and therefore did not hold any child PTY open on our behalf. The child context is updated to preserve any PTY FD in order to avoid the kernel sending `SIGHUP` early during startup. -* `#523 `_: the test suite didn't +* :gh:issue:`523`: the test suite didn't generate a code coverage report if any test failed. -* `#524 `_: Python 3.6+ emitted a +* :gh:issue:`524`: Python 3.6+ emitted a :class:`DeprecationWarning` for :func:`mitogen.utils.run_with_router`. -* `#529 `_: Code coverage of the +* :gh:issue:`529`: Code coverage of the test suite was not measured across all Python versions. -* `16ca111e `_: handle OpenSSH +* :gh:commit:`16ca111e`: handle OpenSSH 7.5 permission denied prompts when ``~/.ssh/config`` rewrites are present. -* `9ec360c2 `_: a new +* :gh:commit:`9ec360c2`: a new :meth:`mitogen.core.Broker.defer_sync` utility function is provided. -* `f20e0bba `_: +* :gh:commit:`f20e0bba`: :meth:`mitogen.service.FileService.register_prefix` permits granting unprivileged access to whole filesystem subtrees, rather than single files at a time. -* `8f85ee03 `_: +* :gh:commit:`8f85ee03`: :meth:`mitogen.core.Router.myself` returns a :class:`mitogen.core.Context` referring to the current process. -* `824c7931 `_: exceptions +* :gh:commit:`824c7931`: exceptions raised by the import hook were updated to include probable reasons for a failure. -* `57b652ed `_: a stray import +* :gh:commit:`57b652ed`: a stray import meant an extra roundtrip and ~4KiB of data was wasted for any context that imported :mod:`mitogen.parent`. @@ -620,51 +882,40 @@ Mitogen for Ansible Enhancements ^^^^^^^^^^^^ -* `#315 `_, - `#392 `_: Ansible 2.6 and 2.7 are +* :gh:pull:`315`, + :gh:issue:`392`: Ansible 2.6 and 2.7 are supported. -* `#321 `_, - `#336 `_: temporary file handling - was simplified, undoing earlier damage caused by compatibility fixes, - improving 2.6 compatibility, and avoiding two network roundtrips for every - related action - (`assemble `_, - `aws_s3 `_, - `copy `_, - `patch `_, - `script `_, - `template `_, - `unarchive `_, - `uri `_). See - :ref:`ansible_tempfiles` for a complete description. - -* `#376 `_, - `#377 `_: the ``kubectl`` connection - type is now supported. Contributed by Yannig Perré. - -* `084c0ac0 `_: avoid a - roundtrip in - `copy `_ and - `template `_ - due to an unfortunate default. - -* `7458dfae `_: avoid a +* :gh:issue:`321`, :gh:issue:`336`: temporary file handling was simplified, + undoing earlier damage caused by compatibility fixes, improving 2.6 + compatibility, and avoiding two network roundtrips for every related action + (:ans:mod:`~assemble`, :ans:mod:`~aws_s3`, :ans:mod:`~copy`, + :ans:mod:`~patch`, :ans:mod:`~script`, :ans:mod:`~template`, + :ans:mod:`~unarchive`, :ans:mod:`~uri`). See :ref:`ansible_tempfiles` for a + complete description. + +* :gh:pull:`376`, :gh:pull:`377`: the ``kubectl`` connection type is now + supported. Contributed by Yannig Perré. + +* :gh:commit:`084c0ac0`: avoid a roundtrip in :ans:mod:`~copy` and + :ans:mod:`~template` due to an unfortunate default. + +* :gh:commit:`7458dfae`: avoid a roundtrip when transferring files smaller than 124KiB. Copy and template actions are now 2-RTT, reducing runtime for a 20-iteration template loop over a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 seconds compared to vanilla. -* `#337 `_: To avoid a scaling +* :gh:issue:`337`: To avoid a scaling limitation, a PTY is no longer allocated for an SSH connection unless the configuration specifies a password. -* `d62e6e2a `_: many-target +* :gh:commit:`d62e6e2a`: many-target runs executed the dependency scanner redundantly due to missing synchronization, wasting significant runtime in the connection multiplexer. In one case work was reduced by 95%, which may manifest as faster runs. -* `5189408e `_: threads are +* :gh:commit:`5189408e`: threads are cooperatively scheduled, minimizing `GIL `_ contention, and reducing context switching by around 90%. This manifests as an overall @@ -683,62 +934,62 @@ Enhancements Fixes ^^^^^ -* `#251 `_, - `#340 `_: Connection Delegation +* :gh:issue:`251`, + :gh:issue:`340`: Connection Delegation could establish connections to the wrong target when ``delegate_to:`` is present. -* `#291 `_: when Mitogen had +* :gh:issue:`291`: when Mitogen had previously been installed using ``pip`` or ``setuptools``, the globally installed version could conflict with a newer version bundled with an extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. -* `#324 `_: plays with a +* :gh:issue:`324`: plays with a `custom module_utils `_ would fail due to fallout from the Python 3 port and related tests being disabled. -* `#331 `_: the connection +* :gh:issue:`331`: the connection multiplexer subprocess always exits before the main Ansible process, ensuring logs generated by it do not overwrite the user's prompt when ``-vvv`` is enabled. -* `#332 `_: support a new +* :gh:issue:`332`: support a new :func:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. -* `#338 `_: compatibility: changes to +* :gh:issue:`338`: compatibility: changes to ``/etc/environment`` and ``~/.pam_environment`` made by a task are reflected in the runtime environment of subsequent tasks. See :ref:`ansible_process_env` for a complete description. -* `#343 `_: the sudo ``--login`` +* :gh:issue:`343`: the sudo ``--login`` option is supported. -* `#344 `_: connections no longer +* :gh:issue:`344`: connections no longer fail when the controller's login username contains slashes. -* `#345 `_: the ``IdentitiesOnly +* :gh:issue:`345`: the ``IdentitiesOnly yes`` option is no longer supplied to OpenSSH by default, better matching Ansible's behaviour. -* `#355 `_: tasks configured to run +* :gh:issue:`355`: tasks configured to run in an isolated forked subprocess were forked from the wrong parent context. This meant built-in modules overridden via a custom ``module_utils`` search path may not have had any effect. -* `#362 `_: to work around a slow +* :gh:issue:`362`: to work around a slow algorithm in the :mod:`subprocess` module, the maximum number of open files in processes running on the target is capped to 512, reducing the work required to start a subprocess by >2000x in default CentOS configurations. -* `#397 `_: recent Mitogen master +* :gh:issue:`397`: recent Mitogen master versions could fail to clean up temporary directories in a number of circumstances, and newer Ansibles moved to using :mod:`atexit` to effect temporary directory cleanup in some circumstances. -* `b9112a9c `_, - `2c287801 `_: OpenSSH 7.5 +* :gh:commit:`b9112a9c`, + :gh:commit:`2c287801`: OpenSSH 7.5 permission denied prompts are now recognized. Contributed by Alex Willmer. * A missing check caused an exception traceback to appear when using the @@ -758,53 +1009,53 @@ Core Library related function calls to a target context, cancelling the chain if an exception occurs. -* `#305 `_: fix a long-standing minor +* :gh:issue:`305`: fix a long-standing minor race relating to the logging framework, where *no route for Message..* would frequently appear during startup. -* `#313 `_: +* :gh:issue:`313`: :meth:`mitogen.parent.Context.call` was documented as capable of accepting static methods. While possible on Python 2.x the result is ugly, and in every case it should be trivial to replace with a classmethod. The documentation was fixed. -* `#337 `_: to avoid a scaling +* :gh:issue:`337`: to avoid a scaling limitation, a PTY is no longer allocated for each OpenSSH client if it can be avoided. PTYs are only allocated if a password is supplied, or when `host_key_checking=accept`. This is since Linux has a default of 4096 PTYs (``kernel.pty.max``), while OS X has a default of 127 and an absolute maximum of 999 (``kern.tty.ptmx_max``). -* `#339 `_: the LXD connection method +* :gh:issue:`339`: the LXD connection method was erroneously executing LXC Classic commands. -* `#345 `_: the SSH connection method +* :gh:issue:`345`: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. -* `#356 `_: if the master Python +* :gh:issue:`356`: if the master Python process does not have :data:`sys.executable` set, the default Python interpreter used for new children on the local machine defaults to ``"/usr/bin/python"``. -* `#366 `_, - `#380 `_: attempts by children to +* :gh:issue:`366`, + :gh:issue:`380`: attempts by children to import :mod:`__main__` where the main program module lacks an execution guard are refused, and an error is logged. This prevents a common and highly confusing error when prototyping new scripts. -* `#371 `_: the LXC connection method +* :gh:pull:`371`: the LXC connection method uses a more compatible method to establish an non-interactive session. Contributed by Brian Candler. -* `af2ded66 `_: add +* :gh:commit:`af2ded66`: add :func:`mitogen.fork.on_fork` to allow non-Mitogen managed process forks to clean up Mitogen resources in the child. -* `d6784242 `_: the setns method +* :gh:commit:`d6784242`: the setns method always resets ``HOME``, ``SHELL``, ``LOGNAME`` and ``USER`` environment variables to an account in the target container, defaulting to ``root``. -* `830966bf `_: the UNIX +* :gh:commit:`830966bf`: the UNIX listener no longer crashes if the peer process disappears in the middle of connection setup. @@ -827,7 +1078,7 @@ bug reports, testing, features and fixes in this release contributed by `Josh Smift `_, `Luca Nunzi `_, `Orion Poplawski `_, -`Peter V. Saveliev `_, +`Peter V. Saveliev `_, `Pierre-Henry Muller `_, `Pierre-Louis Bonicoli `_, `Prateek Jain `_, @@ -845,28 +1096,28 @@ v0.2.2 (2018-07-26) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#291 `_: ``ansible_*_interpreter`` +* :gh:issue:`291`: ``ansible_*_interpreter`` variables are parsed using a restrictive shell-like syntax, supporting a common idiom where ``ansible_python_interpreter`` is set to ``/usr/bin/env python``. -* `#299 `_: fix the ``network_cli`` +* :gh:issue:`299`: fix the ``network_cli`` connection type when the Mitogen strategy is active. Mitogen cannot help network device connections, however it should still be possible to use device connections while Mitogen is active. -* `#301 `_: variables like ``$HOME`` in +* :gh:pull:`301`: variables like ``$HOME`` in the ``remote_tmp`` setting are evaluated correctly. -* `#303 `_: the :ref:`doas` become method +* :gh:pull:`303`: the :ref:`doas` become method is supported. Contributed by `Mike Walker `_. -* `#309 `_: fix a regression to +* :gh:issue:`309`: fix a regression to process environment cleanup, caused by the change in v0.2.1 to run local tasks with the correct environment. -* `#317 `_: respect the verbosity +* :gh:issue:`317`: respect the verbosity setting when writing to Ansible's ``log_path``, if it is enabled. Child log filtering was also incorrect, causing the master to needlessly wake many times. This nets a 3.5% runtime improvement running against the local @@ -879,32 +1130,32 @@ Mitogen for Ansible Core Library ~~~~~~~~~~~~ -* `#291 `_: the ``python_path`` +* :gh:issue:`291`: the ``python_path`` parameter may specify an argument vector prefix rather than a string program path. -* `#300 `_: the broker could crash on +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to scheduled `kqueue - `_ filter changes for + `_ filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. -* `#303 `_: the :ref:`doas` become method +* :gh:pull:`303`: the :ref:`doas` become method is now supported. Contributed by `Mike Walker `_. -* `#307 `_: SSH login banner output +* :gh:issue:`307`: SSH login banner output containing the word 'password' is no longer confused for a password prompt. -* `#319 `_: SSH connections would +* :gh:issue:`319`: SSH connections would fail immediately on Windows Subsystem for Linux, due to use of `TCSAFLUSH` with :func:`termios.tcsetattr`. The flag is omitted if WSL is detected. -* `#320 `_: The OS X poller +* :gh:issue:`320`: The OS X poller could spuriously wake up due to ignoring an error bit set on events returned by the kernel, manifesting as a failure to read from an unrelated descriptor. -* `#342 `_: The ``network_cli`` +* :gh:issue:`342`: The ``network_cli`` connection type would fail due to a missing internal SSH plugin method. * Standard IO forwarding accidentally configured the replacement ``stdout`` and @@ -950,7 +1201,7 @@ v0.2.1 (2018-07-10) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#297 `_: compatibility: local +* :gh:issue:`297`: compatibility: local actions set their working directory to that of their defining playbook, and inherit a process environment as if they were executed as a subprocess of the forked task worker. diff --git a/docs/conf.py b/docs/conf.py index 7f03e451..1a6a117b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,13 +2,19 @@ import os import sys sys.path.append('..') +sys.path.append('.') import mitogen VERSION = '%s.%s.%s' % mitogen.__version__ -author = u'David Wilson' -copyright = u'2019, David Wilson' -exclude_patterns = ['_build'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput'] +author = u'Network Genomics' +copyright = u'2019, Network Genomics' +exclude_patterns = ['_build', '.venv'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput', 'domainrefs'] + +# get rid of version from , it messes with piwik +html_title = 'Mitogen Documentation' + +html_show_copyright = False html_show_sourcelink = False html_show_sphinx = False html_sidebars = {'**': ['globaltoc.html', 'github.html']} @@ -35,10 +41,53 @@ templates_path = ['_templates'] todo_include_todos = False version = VERSION +domainrefs = { + 'gh:commit': { + 'text': '%s', + 'url': 'https://github.com/dw/mitogen/commit/%s', + }, + 'gh:issue': { + 'text': '#%s', + 'url': 'https://github.com/dw/mitogen/issues/%s', + }, + 'gh:pull': { + 'text': '#%s', + 'url': 'https://github.com/dw/mitogen/pull/%s', + }, + 'ans:mod': { + 'text': '%s module', + 'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html', + }, + 'ans:conn': { + 'text': '%s connection plug-in', + 'url': 'https://docs.ansible.com/ansible/latest/plugins/connection/%s.html', + }, + 'freebsd:man2': { + 'text': '%s(2)', + 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + }, + 'linux:man1': { + 'text': '%s(1)', + 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + }, + 'linux:man2': { + 'text': '%s(2)', + 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + }, + 'linux:man3': { + 'text': '%s(3)', + 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + }, + 'linux:man7': { + 'text': '%s(7)', + 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + }, +} + rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s -.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__ +.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__ """ % locals() diff --git a/docs/contributors.rst b/docs/contributors.rst index dcfb50fa..584c4cd4 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -88,6 +88,9 @@ sponsorship and outstanding future-thinking of its early adopters. <h3>Private Sponsors</h3> <ul style="line-height: 120% !important;"> + <li><a href="https://skunkwerks.at/">SkunkWerks</a> — + <em>Mitogen on FreeBSD runs like a kid in a candy store: fast & + sweet.</em></li> <li>Donald Clark Jackson — <em>Mitogen is an exciting project, and I am happy to support its development.</em></li> diff --git a/docs/domainrefs.py b/docs/domainrefs.py new file mode 100644 index 00000000..4ff29dc5 --- /dev/null +++ b/docs/domainrefs.py @@ -0,0 +1,41 @@ + +import functools +import re + +import docutils.nodes +import docutils.utils + + +CUSTOM_RE = re.compile('(.*) <(.*)>') + + +def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]): + template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html' + + match = CUSTOM_RE.match(text) + if match: # "custom text <real link>" + title = match.group(1) + text = match.group(2) + elif text.startswith('~'): # brief + text = text[1:] + title = config.get('brief', '%s') % ( + docutils.utils.unescape(text), + ) + else: + title = config.get('text', '%s') % ( + docutils.utils.unescape(text), + ) + + node = docutils.nodes.reference( + rawsource=rawtext, + text=title, + refuri=config['url'] % (text,), + **options + ) + + return [node], [] + + +def setup(app): + for name, info in app.config._raw_config['domainrefs'].items(): + app.add_role(name, functools.partial(role, info)) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 020760bc..12056c55 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -265,7 +265,7 @@ We must therefore continue by writing our code as a script:: print(local.call(my_first_function)) if __name__ == '__main__': - mitogen.utils.log_to_file(main) + mitogen.utils.log_to_file("mitogen.log") mitogen.utils.run_with_router(main) Let's try running it: @@ -341,15 +341,13 @@ The following built-in types may be used as parameters or return values in remote procedure calls: * :class:`bool` -* :class:`bytearray` -* :func:`bytes` +* :func:`bytes` (:class:`str` on Python 2.x) * :class:`dict` * :class:`int` * :func:`list` * :class:`long` -* :class:`str` * :func:`tuple` -* :func:`unicode` +* :func:`unicode` (:class:`str` on Python 3.x) User-defined types may not be used, except for: diff --git a/docs/howitworks.rst b/docs/howitworks.rst index b4a53810..05c097e5 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -346,11 +346,15 @@ Masters listen on the following handles: .. currentmodule:: mitogen.core .. data:: ALLOCATE_ID - Replies to any message sent to it with a newly allocated range of context - IDs, to allow children to safely start their own contexts. Presently IDs - are allocated in batches of 1000 from a 32 bit range, allowing up to 4.2 - million parent contexts to be created and destroyed before the associated - Router must be recreated. + Replies to any message sent to it with a newly allocated range of context + IDs, to allow children to safely start their own contexts. Presently IDs are + allocated in batches of 1000 from a 32 bit range, allowing up to 4.2 million + parent contexts to be created and destroyed before the associated Router + must be recreated. + + This is handled by :class:`mitogen.master.IdAllocator` in the master + process, and messages are sent to it from + :class:`mitogen.parent.ChildIdAllocator` in children. Children listen on the following handles: @@ -430,8 +434,9 @@ also listen on the following handles: Receives `target_id` integer from downstream, verifies a route exists to `target_id` via the stream on which the message was received, removes that - route from its local table, then propagates the message upward towards its - own parent. + route from its local table, triggers the ``disconnect`` signal on any + :class:`mitogen.core.Context` instance in the local process, then + propagates the message upward towards its own parent. .. currentmodule:: mitogen.core .. data:: DETACHING @@ -625,7 +630,8 @@ The `auth_id` field is separate from `src_id` in order to support granting privilege to contexts that do not follow the tree's natural trust chain. This supports cases where siblings are permitted to execute code on one another, or where isolated processes can connect to a listener and communicate with an -already established established tree. +already established established tree, such as where a :mod:`mitogen.unix` +client receives the same privilege as the process it connects to. Differences Between Master And Child Brokers @@ -669,8 +675,12 @@ code occurring after the first conditional that looks like a standard if __name__ == '__main__': run_some_code() -This is a hack, but it's the least annoying hack I've found for the problem -yet. +To further avoid accidental execution, Mitogen will refuse to serve +:mod:`__main__` to children if no execution guard is found, as it is common +that no guard is present during early script prototyping. + +These are hacks, but they are the safest and least annoying found to solve the +problem. Avoiding Negative Imports diff --git a/docs/index.rst b/docs/index.rst index 6b5deb71..3cd53d32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,8 +27,8 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_, ultimately it is not intended for direct use by consumer software. .. _Salt: https://docs.saltstack.com/en/latest/ -.. _Ansible: http://docs.ansible.com/ -.. _Fabric: http://www.fabfile.org/ +.. _Ansible: https://docs.ansible.com/ +.. _Fabric: https://www.fabfile.org/ The focus is to centralize and perfect the intricate dance required to run Python code safely and efficiently on a remote machine, while **avoiding @@ -132,7 +132,7 @@ any tool such as `py2exe`_ that correctly implement the protocols in PEP-302, allowing truly single file applications to run across multiple machines without further effort. -.. _py2exe: http://www.py2exe.org/ +.. _py2exe: https://www.py2exe.org/ Common sources of import latency and bandwidth consumption are mitigated: @@ -155,40 +155,6 @@ Common sources of import latency and bandwidth consumption are mitigated: representing 1.7MiB of uncompressed source split across 148 modules. -SSH Client Emulation -#################### - -.. image:: images/fakessh.svg - :class: mitogen-right-300 - -Support is included for starting subprocesses with a modified environment, that -cause their attempt to use SSH to be redirected back into the host program. In -this way tools like `rsync`, `git`, `sftp`, and `scp` can efficiently reuse the -host program's existing connection to the remote machine, including any -firewall/user account hopping in use, with no additional configuration. - -Scenarios that were not previously possible with these tools are enabled, such -as running `sftp` and `rsync` over a `sudo` session, to an account the user -cannot otherwise directly log into, including in restrictive environments that -for example enforce an interactive TTY and account password. - -.. raw:: html - - <div style="clear: both;"></div> - -.. code-block:: python - - bastion = router.ssh(hostname='bastion.mycorp.com') - webserver = router.ssh(via=bastion, hostname='webserver') - webapp = router.sudo(via=webserver, username='webapp') - fileserver = router.ssh(via=bastion, hostname='fileserver') - - # Transparently tunnelled over fileserver -> .. -> sudo.webapp link - fileserver.call(mitogen.fakessh.run, webapp, [ - 'rsync', 'appdata', 'appserver:appdata' - ]) - - Message Routing ############### @@ -329,36 +295,7 @@ External contexts are configured such that any attempt to execute a function from the main Python script will correctly cause that script to be imported as usual into the slave process. -.. code-block:: python - - #!/usr/bin/env python - """ - Install our application on a remote machine. - - Usage: - install_app.py <hostname> - - Where: - <hostname> Hostname to install to. - """ - import os - import sys - - import mitogen - - - def install_app(): - os.system('tar zxvf my_app.tar.gz') - - - @mitogen.main() - def main(broker): - if len(sys.argv) != 2: - print(__doc__) - sys.exit(1) - - context = mitogen.ssh.connect(broker, sys.argv[1]) - context.call(install_app) +.. literalinclude:: ../examples/install_app.py Event-driven IO diff --git a/docs/internals.rst b/docs/internals.rst index e1dd4a41..c3247be0 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2,10 +2,10 @@ Internal API Reference ********************** -.. toctree:: - :hidden: +.. note:: - signals + Internal APIs are subject to rapid change even across minor releases. This + page exists to help users modify and extend the library. Constants @@ -15,114 +15,170 @@ Constants .. autodata:: CHUNK_SIZE -Poller Classes -============== +Pollers +======= .. currentmodule:: mitogen.core .. autoclass:: Poller - :members: + :members: + +.. currentmodule:: mitogen.parent +.. autoclass:: KqueuePoller .. currentmodule:: mitogen.parent .. autoclass:: EpollPoller .. currentmodule:: mitogen.parent -.. autoclass:: KqueuePoller +.. autoclass:: PollPoller -Latch Class -=========== +Latch +===== .. currentmodule:: mitogen.core .. autoclass:: Latch :members: -PidfulStreamHandler Class -========================= +Logging +======= + +See also :class:`mitogen.core.IoLoggerProtocol`. + +.. currentmodule:: mitogen.core +.. autoclass:: LogHandler + :members: + +.. currentmodule:: mitogen.master +.. autoclass:: LogForwarder + :members: .. currentmodule:: mitogen.core .. autoclass:: PidfulStreamHandler :members: -Side Class -========== +Stream, Side & Protocol +======================= .. currentmodule:: mitogen.core -.. autoclass:: Side +.. autoclass:: Stream :members: +.. currentmodule:: mitogen.core +.. autoclass:: BufferedWriter + :members: -Stream Classes -============== +.. currentmodule:: mitogen.core +.. autoclass:: Side + :members: .. currentmodule:: mitogen.core -.. autoclass:: BasicStream +.. autoclass:: Protocol :members: -.. autoclass:: Stream +.. currentmodule:: mitogen.parent +.. autoclass:: BootstrapProtocol :members: -.. currentmodule:: mitogen.fork -.. autoclass:: Stream +.. currentmodule:: mitogen.core +.. autoclass:: DelimitedProtocol :members: .. currentmodule:: mitogen.parent -.. autoclass:: Stream +.. autoclass:: LogProtocol :members: -.. currentmodule:: mitogen.ssh -.. autoclass:: Stream +.. currentmodule:: mitogen.core +.. autoclass:: IoLoggerProtocol :members: -.. currentmodule:: mitogen.sudo -.. autoclass:: Stream +.. currentmodule:: mitogen.core +.. autoclass:: MitogenProtocol :members: - -Other Stream Subclasses -======================= +.. currentmodule:: mitogen.parent +.. autoclass:: MitogenProtocol + :members: .. currentmodule:: mitogen.core +.. autoclass:: Waker + :members: + -.. autoclass:: IoLogger +Connection & Options +==================== + +.. currentmodule:: mitogen.fork +.. autoclass:: Options + :members: +.. autoclass:: Connection :members: -.. autoclass:: Waker +.. currentmodule:: mitogen.parent +.. autoclass:: Options + :members: +.. autoclass:: Connection :members: +.. currentmodule:: mitogen.ssh +.. autoclass:: Options + :members: +.. autoclass:: Connection + :members: -Poller Class -============ +.. currentmodule:: mitogen.sudo +.. autoclass:: Options + :members: +.. autoclass:: Connection + :members: + + +Import Mechanism +================ .. currentmodule:: mitogen.core -.. autoclass:: Poller - :members: +.. autoclass:: Importer + :members: -.. currentmodule:: mitogen.parent -.. autoclass:: KqueuePoller +.. currentmodule:: mitogen.master +.. autoclass:: ModuleResponder + :members: .. currentmodule:: mitogen.parent -.. autoclass:: EpollPoller +.. autoclass:: ModuleForwarder + :members: -Importer Class +Module Finders ============== -.. currentmodule:: mitogen.core -.. autoclass:: Importer +.. currentmodule:: mitogen.master +.. autoclass:: ModuleFinder :members: +.. currentmodule:: mitogen.master +.. autoclass:: FinderMethod + :members: -Responder Class -=============== +.. currentmodule:: mitogen.master +.. autoclass:: DefectivePython3xMainMethod + :members: .. currentmodule:: mitogen.master -.. autoclass:: ModuleResponder +.. autoclass:: PkgutilMethod :members: +.. currentmodule:: mitogen.master +.. autoclass:: SysModulesMethod + :members: + +.. currentmodule:: mitogen.master +.. autoclass:: ParentEnumerationMethod + :members: -RouteMonitor Class + +Routing Management ================== .. currentmodule:: mitogen.parent @@ -130,45 +186,68 @@ RouteMonitor Class :members: -Forwarder Class -=============== +Timer Management +================ .. currentmodule:: mitogen.parent -.. autoclass:: ModuleForwarder +.. autoclass:: TimerList + :members: + +.. currentmodule:: mitogen.parent +.. autoclass:: Timer :members: -ExternalContext Class +Context ID Allocation ===================== +.. currentmodule:: mitogen.master +.. autoclass:: IdAllocator + :members: + +.. currentmodule:: mitogen.parent +.. autoclass:: ChildIdAllocator + :members: + + +Child Implementation +==================== + .. currentmodule:: mitogen.core .. autoclass:: ExternalContext :members: +.. currentmodule:: mitogen.core +.. autoclass:: Dispatcher + :members: -mitogen.master -============== + +Process Management +================== .. currentmodule:: mitogen.parent -.. autoclass:: ProcessMonitor +.. autoclass:: Reaper :members: +.. currentmodule:: mitogen.parent +.. autoclass:: Process + :members: -Blocking I/O Functions -====================== +.. currentmodule:: mitogen.parent +.. autoclass:: PopenProcess + :members: -These functions exist to support the blocking phase of setting up a new -context. They will eventually be replaced with asynchronous equivalents. +.. currentmodule:: mitogen.fork +.. autoclass:: Process + :members: -.. currentmodule:: mitogen.parent -.. autofunction:: discard_until -.. autofunction:: iter_read -.. autofunction:: write_all +Helper Functions +================ -Subprocess Creation Functions -============================= +Subprocess Functions +--------------------- .. currentmodule:: mitogen.parent .. autofunction:: create_child @@ -176,19 +255,19 @@ Subprocess Creation Functions .. autofunction:: tty_create_child -Helper Functions -================ +Helpers +------- .. currentmodule:: mitogen.core -.. autofunction:: to_text .. autofunction:: has_parent_authority +.. autofunction:: io_op +.. autofunction:: pipe +.. autofunction:: set_block .. autofunction:: set_cloexec .. autofunction:: set_nonblock -.. autofunction:: set_block -.. autofunction:: io_op +.. autofunction:: to_text .. currentmodule:: mitogen.parent -.. autofunction:: close_nonstandard_fds .. autofunction:: create_socketpair .. currentmodule:: mitogen.master @@ -198,7 +277,67 @@ Helper Functions .. autofunction:: minimize_source +.. _signals: + Signals ======= -:ref:`Please refer to Signals <signals>`. +Mitogen contains a simplistic signal mechanism to decouple its components. When +a signal is fired by an instance of a class, functions registered to receive it +are called back. + +.. warning:: + + As signals execute on the Broker thread, and without exception handling, + they are generally unsafe for consumption by user code, as any bugs could + trigger crashes and hangs for which the broker is unable to forward logs, + or ensure the buggy context always shuts down on disconnect. + + +Functions +--------- + +.. currentmodule:: mitogen.core + +.. autofunction:: listen +.. autofunction:: unlisten +.. autofunction:: fire + + +List +---- + +These signals are used internally by Mitogen. + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - Class + - Name + - Description + + * - :py:class:`mitogen.core.Stream` + - ``disconnect`` + - Fired on the Broker thread when disconnection is detected. + + * - :py:class:`mitogen.core.Stream` + - ``shutdown`` + - Fired on the Broker thread when broker shutdown begins. + + * - :py:class:`mitogen.core.Context` + - ``disconnect`` + - Fired on the Broker thread during shutdown (???) + + * - :py:class:`mitogen.parent.Process` + - ``exit`` + - Fired when :class:`mitogen.parent.Reaper` detects subprocess has fully + exitted. + + * - :py:class:`mitogen.core.Broker` + - ``shutdown`` + - Fired after Broker.shutdown() is called. + + * - :py:class:`mitogen.core.Broker` + - ``exit`` + - Fired immediately prior to the broker thread exit. diff --git a/docs/requirements.txt b/docs/requirements.txt index a93c2140..3c4674fd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -Sphinx==1.7.1 -sphinxcontrib-programoutput==0.11 -alabaster==0.7.10 +Sphinx==2.1.2; python_version > '3.0' +sphinxcontrib-programoutput==0.14; python_version > '3.0' +alabaster==0.7.10; python_version > '3.0' diff --git a/docs/services.rst b/docs/services.rst index ef402214..85b08e6d 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -61,55 +61,7 @@ Pool Example ------- -.. code-block:: python - - import mitogen - import mitogen.service - - - class FileService(mitogen.service.Service): - """ - Simple file server, for demonstration purposes only! Use of this in - real code would be a security vulnerability as it would permit children - to read arbitrary files from the master's disk. - """ - handle = 500 - required_args = { - 'path': str - } - - def dispatch(self, args, msg): - with open(args['path'], 'r') as fp: - return fp.read() - - - def download_file(context, path): - s = mitogen.service.call(context, FileService.handle, { - 'path': path - }) - - with open(path, 'w') as fp: - fp.write(s) - - - @mitogen.core.takes_econtext - def download_some_files(paths, econtext): - for path in paths: - download_file(econtext.master, path) - - - @mitogen.main() - def main(router): - pool = mitogen.service.Pool(router, size=1, services=[ - FileService(router), - ]) - - remote = router.ssh(hostname='k3') - remote.call(download_some_files, [ - '/etc/passwd', - '/etc/hosts', - ]) - pool.stop() +.. literalinclude:: ../examples/service/self_contained.py Reference @@ -134,3 +86,12 @@ Reference .. autoclass:: mitogen.service.Pool :members: + +Built-in Services +----------------- + +.. autoclass:: mitogen.service.FileService + :members: + +.. autoclass:: mitogen.service.PushFileService + :members: diff --git a/docs/signals.rst b/docs/signals.rst deleted file mode 100644 index 19533bb1..00000000 --- a/docs/signals.rst +++ /dev/null @@ -1,60 +0,0 @@ - -.. _signals: - -Signals -======= - -Mitogen contains a simplistic signal mechanism to help decouple its internal -components. When a signal is fired by a particular instance of a class, any -functions registered to receive it will be called back. - -.. warning:: - - As signals execute on the Broker thread, and without exception handling, - they are generally unsafe for consumption by user code, as any bugs could - trigger crashes and hangs for which the broker is unable to forward logs, - or ensure the buggy context always shuts down on disconnect. - - -Functions ---------- - -.. currentmodule:: mitogen.core - -.. autofunction:: listen -.. autofunction:: fire - - -List ----- - -These signals are used internally by Mitogen. - -.. list-table:: - :header-rows: 1 - :widths: auto - - * - Class - - Name - - Description - - * - :py:class:`mitogen.core.Stream` - - ``disconnect`` - - Fired on the Broker thread when disconnection is detected. - - * - :py:class:`mitogen.core.Context` - - ``disconnect`` - - Fired on the Broker thread during shutdown (???) - - * - :py:class:`mitogen.core.Router` - - ``shutdown`` - - Fired on the Broker thread after Broker.shutdown() is called. - - * - :py:class:`mitogen.core.Broker` - - ``shutdown`` - - Fired after Broker.shutdown() is called. - - * - :py:class:`mitogen.core.Broker` - - ``exit`` - - Fired immediately prior to the broker thread exit. - diff --git a/docs/toc.rst b/docs/toc.rst index 2bbd0f9a..e43326f1 100644 --- a/docs/toc.rst +++ b/docs/toc.rst @@ -7,11 +7,11 @@ Table Of Contents index Mitogen for Ansible <ansible_detailed> - contributors changelog + contributors howitworks - getting_started api + getting_started examples internals diff --git a/examples/install_app.py b/examples/install_app.py new file mode 100644 index 00000000..566353a8 --- /dev/null +++ b/examples/install_app.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +""" +Install our application on a remote machine. + +Usage: + install_app.py <hostname> + +Where: + <hostname> Hostname to install to. +""" +import os +import sys + +import mitogen + + +def install_app(): + os.system('tar zxvf my_app.tar.gz') + + +@mitogen.main() +def main(router): + if len(sys.argv) != 2: + print(__doc__) + sys.exit(1) + + context = router.ssh(hostname=sys.argv[1]) + context.call(install_app) diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index d0cd9a3a..55b272d9 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -241,9 +241,13 @@ def main(router): print('usage: %s <host> <mountpoint>' % sys.argv[0]) sys.exit(1) - blerp = fuse.FUSE( + kwargs = {} + if sys.platform == 'darwin': + kwargs['volname'] = '%s (Mitogen)' % (sys.argv[1],) + + fuse.FUSE( operations=Operations(sys.argv[1]), mountpoint=sys.argv[2], foreground=True, - volname='%s (Mitogen)' % (sys.argv[1],), + **kwargs ) diff --git a/examples/service/client.py b/examples/service/client.py deleted file mode 100644 index fc2d8427..00000000 --- a/examples/service/client.py +++ /dev/null @@ -1,15 +0,0 @@ - -import mitogen.master -import mitogen.unix -import mitogen.service -import mitogen.utils - - -PING = 500 - - -mitogen.utils.log_to_file() - -router, parent = mitogen.unix.connect('/tmp/mitosock') -with router: - print(mitogen.service.call(parent, CONNECT_BY_ID, {})) diff --git a/examples/service/self_contained.py b/examples/service/self_contained.py new file mode 100644 index 00000000..332aa24e --- /dev/null +++ b/examples/service/self_contained.py @@ -0,0 +1,52 @@ +import mitogen +import mitogen.service + + +class FileService(mitogen.service.Service): + """ + Simple file server, for demonstration purposes only! Use of this in + real code would be a security vulnerability as it would permit children + to read any file from the master's disk. + """ + + @mitogen.service.expose(policy=mitogen.service.AllowAny()) + @mitogen.service.arg_spec(spec={ + 'path': str + }) + def read_file(self, path): + with open(path, 'rb') as fp: + return fp.read() + + +def download_file(source_context, path): + s = source_context.call_service( + service_name=FileService, # may also be string 'pkg.mod.FileService' + method_name='read_file', + path=path, + ) + + with open(path, 'w') as fp: + fp.write(s) + + +def download_some_files(source_context, paths): + for path in paths: + download_file(source_context, path) + + +@mitogen.main() +def main(router): + pool = mitogen.service.Pool(router, services=[ + FileService(router), + ]) + + remote = router.ssh(hostname='k3') + remote.call(download_some_files, + source_context=router.myself(), + paths=[ + '/etc/passwd', + '/etc/hosts', + ] + ) + pool.stop() + diff --git a/examples/service/server.py b/examples/service/server.py deleted file mode 100644 index 1f8c1475..00000000 --- a/examples/service/server.py +++ /dev/null @@ -1,20 +0,0 @@ - -# The service framework will fundamentally change (i.e. become much nicer, and -# hopefully lose those hard-coded magic numbers somehow), but meanwhile this is -# a taster of how it looks today. - -import mitogen -import mitogen.service -import mitogen.unix - - -class PingService(mitogen.service.Service): - def dispatch(self, dct, msg): - return 'Hello, world' - - -@mitogen.main() -def main(router): - listener = mitogen.unix.Listener(router, path='/tmp/mitosock') - service = PingService(router) - service.run() diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 47fe4d38..47e570ab 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 7) +__version__ = (0, 2, 8) #: This is :data:`False` in slave contexts. Previously it was used to prevent @@ -111,10 +111,10 @@ def main(log_level='INFO', profiling=_default_profiling): if profiling: mitogen.core.enable_profiling() mitogen.master.Router.profiling = profiling - utils.log_to_file(level=log_level) + mitogen.utils.log_to_file(level=log_level) return mitogen.core._profile_hook( 'app.main', - utils.run_with_router, + mitogen.utils.run_with_router, func, ) return wrapper diff --git a/mitogen/buildah.py b/mitogen/buildah.py new file mode 100644 index 00000000..f850234d --- /dev/null +++ b/mitogen/buildah.py @@ -0,0 +1,73 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Options(mitogen.parent.Options): + container = None + username = None + buildah_path = 'buildah' + + def __init__(self, container=None, buildah_path=None, username=None, + **kwargs): + super(Options, self).__init__(**kwargs) + assert container is not None + self.container = container + if buildah_path: + self.buildah_path = buildah_path + if username: + self.username = username + + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = False + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + + def _get_name(self): + return u'buildah.' + self.options.container + + def get_boot_command(self): + args = [self.options.buildah_path, 'run'] + if self.options.username: + args += ['--user=' + self.options.username] + args += ['--', self.options.container] + return args + super(Connection, self).get_boot_command() diff --git a/mitogen/compat/pkgutil.py b/mitogen/compat/pkgutil.py index 28e2aead..15eb2afa 100644 --- a/mitogen/compat/pkgutil.py +++ b/mitogen/compat/pkgutil.py @@ -542,7 +542,8 @@ def extend_path(path, name): if os.path.isfile(pkgfile): try: f = open(pkgfile) - except IOError, msg: + except IOError: + msg = sys.exc_info()[1] sys.stderr.write("Can't open %s: %s\n" % (pkgfile, msg)) else: diff --git a/mitogen/core.py b/mitogen/core.py index ff77bba9..87388620 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -37,6 +37,7 @@ bootstrap implementation sent to every new slave context. import binascii import collections import encodings.latin_1 +import encodings.utf_8 import errno import fcntl import itertools @@ -49,6 +50,7 @@ import signal import socket import struct import sys +import syslog import threading import time import traceback @@ -102,10 +104,9 @@ LOG = logging.getLogger('mitogen') IOLOG = logging.getLogger('mitogen.io') IOLOG.setLevel(logging.INFO) -LATIN1_CODEC = encodings.latin_1.Codec() # str.encode() may take import lock. Deadlock possible if broker calls # .encode() on behalf of thread currently waiting for module. -UTF8_CODEC = encodings.latin_1.Codec() +LATIN1_CODEC = encodings.latin_1.Codec() _v = False _vv = False @@ -121,6 +122,7 @@ LOAD_MODULE = 107 FORWARD_MODULE = 108 DETACHING = 109 CALL_SERVICE = 110 +STUB_CALL_SERVICE = 111 #: Special value used to signal disconnection or the inability to route a #: message, when it appears in the `reply_to` field. Usually causes @@ -214,7 +216,8 @@ else: class Error(Exception): - """Base for all exceptions raised by Mitogen. + """ + Base for all exceptions raised by Mitogen. :param str fmt: Exception text, or format string if `args` is non-empty. @@ -230,14 +233,18 @@ class Error(Exception): class LatchError(Error): - """Raised when an attempt is made to use a :class:`mitogen.core.Latch` - that has been marked closed.""" + """ + Raised when an attempt is made to use a :class:`mitogen.core.Latch` that + has been marked closed. + """ pass class Blob(BytesType): - """A serializable bytes subclass whose content is summarized in repr() - output, making it suitable for logging binary data.""" + """ + A serializable bytes subclass whose content is summarized in repr() output, + making it suitable for logging binary data. + """ def __repr__(self): return '[blob: %d bytes]' % len(self) @@ -246,8 +253,10 @@ class Blob(BytesType): class Secret(UnicodeType): - """A serializable unicode subclass whose content is masked in repr() - output, making it suitable for logging passwords.""" + """ + A serializable unicode subclass whose content is masked in repr() output, + making it suitable for logging passwords. + """ def __repr__(self): return '[secret]' @@ -281,7 +290,7 @@ class Kwargs(dict): def __init__(self, dct): for k, v in dct.iteritems(): if type(k) is unicode: - k, _ = UTF8_CODEC.encode(k) + k, _ = encodings.utf_8.encode(k) self[k] = v def __repr__(self): @@ -321,30 +330,42 @@ def _unpickle_call_error(s): class ChannelError(Error): - """Raised when a channel dies or has been closed.""" + """ + Raised when a channel dies or has been closed. + """ remote_msg = 'Channel closed by remote end.' local_msg = 'Channel closed by local end.' class StreamError(Error): - """Raised when a stream cannot be established.""" + """ + Raised when a stream cannot be established. + """ pass class TimeoutError(Error): - """Raised when a timeout occurs on a stream.""" + """ + Raised when a timeout occurs on a stream. + """ pass def to_text(o): - """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of + """ + Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of :class:`bytes`, otherwise pass it to the :class:`str` constructor. The - returned object is always a plain :class:`str`, any subclass is removed.""" + returned object is always a plain :class:`str`, any subclass is removed. + """ if isinstance(o, BytesType): return o.decode('utf-8') return UnicodeType(o) +# Documented in api.rst to work around Sphinx limitation. +now = getattr(time, 'monotonic', time.time) + + # Python 2.4 try: any @@ -378,41 +399,84 @@ else: return _partition(s, sep, s.find) or (s, '', '') +def _has_parent_authority(context_id): + return ( + (context_id == mitogen.context_id) or + (context_id in mitogen.parent_ids) + ) + def has_parent_authority(msg, _stream=None): - """Policy function for use with :class:`Receiver` and + """ + Policy function for use with :class:`Receiver` and :meth:`Router.add_handler` that requires incoming messages to originate from a parent context, or on a :class:`Stream` whose :attr:`auth_id <Stream.auth_id>` has been set to that of a parent context or the current - context.""" - return (msg.auth_id == mitogen.context_id or - msg.auth_id in mitogen.parent_ids) + context. + """ + return _has_parent_authority(msg.auth_id) + + +def _signals(obj, signal): + return ( + obj.__dict__ + .setdefault('_signals', {}) + .setdefault(signal, []) + ) def listen(obj, name, func): """ - Arrange for `func(*args, **kwargs)` to be invoked when the named signal is + Arrange for `func()` to be invoked when signal `name` is fired on `obj`. + """ + _signals(obj, name).append(func) + + +def unlisten(obj, name, func): + """ + Remove `func()` from the list of functions invoked when signal `name` is fired by `obj`. + + :raises ValueError: + `func()` was not on the list. """ - signals = vars(obj).setdefault('_signals', {}) - signals.setdefault(name, []).append(func) + _signals(obj, name).remove(func) def fire(obj, name, *args, **kwargs): """ Arrange for `func(*args, **kwargs)` to be invoked for every function - registered for the named signal on `obj`. + registered for signal `name` on `obj`. """ - signals = vars(obj).get('_signals', {}) - for func in signals.get(name, ()): + for func in _signals(obj, name): func(*args, **kwargs) def takes_econtext(func): + """ + Decorator that marks a function or class method to automatically receive a + kwarg named `econtext`, referencing the + :class:`mitogen.core.ExternalContext` active in the context in which the + function is being invoked in. The decorator is only meaningful when the + function is invoked via :data:`CALL_FUNCTION <mitogen.core.CALL_FUNCTION>`. + + When the function is invoked directly, `econtext` must still be passed to + it explicitly. + """ func.mitogen_takes_econtext = True return func def takes_router(func): + """ + Decorator that marks a function or class method to automatically receive a + kwarg named `router`, referencing the :class:`mitogen.core.Router` active + in the context in which the function is being invoked in. The decorator is + only meaningful when the function is invoked via :data:`CALL_FUNCTION + <mitogen.core.CALL_FUNCTION>`. + + When the function is invoked directly, `router` must still be passed to it + explicitly. + """ func.mitogen_takes_router = True return func @@ -432,35 +496,42 @@ def is_blacklisted_import(importer, fullname): def set_cloexec(fd): - """Set the file descriptor `fd` to automatically close on - :func:`os.execve`. This has no effect on file descriptors inherited across - :func:`os.fork`, they must be explicitly closed through some other means, - such as :func:`mitogen.fork.on_fork`.""" + """ + Set the file descriptor `fd` to automatically close on :func:`os.execve`. + This has no effect on file descriptors inherited across :func:`os.fork`, + they must be explicitly closed through some other means, such as + :func:`mitogen.fork.on_fork`. + """ flags = fcntl.fcntl(fd, fcntl.F_GETFD) assert fd > 2 fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def set_nonblock(fd): - """Set the file descriptor `fd` to non-blocking mode. For most underlying - file types, this causes :func:`os.read` or :func:`os.write` to raise + """ + Set the file descriptor `fd` to non-blocking mode. For most underlying file + types, this causes :func:`os.read` or :func:`os.write` to raise :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread - when the underlying kernel buffer is exhausted.""" + when the underlying kernel buffer is exhausted. + """ flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) def set_block(fd): - """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread - when the underlying kernel buffer is exhausted.""" + """ + Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread when + the underlying kernel buffer is exhausted. + """ flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) def io_op(func, *args): - """Wrap `func(*args)` that may raise :class:`select.error`, - :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating - to disconnection and retry events in various subsystems: + """ + Wrap `func(*args)` that may raise :class:`select.error`, :class:`IOError`, + or :class:`OSError`, trapping UNIX error codes relating to disconnection + and retry events in various subsystems: * When a signal is delivered to the process on Python 2, system call retry is signalled through :data:`errno.EINTR`. The invocation is automatically @@ -491,7 +562,8 @@ def io_op(func, *args): class PidfulStreamHandler(logging.StreamHandler): - """A :class:`logging.StreamHandler` subclass used when + """ + A :class:`logging.StreamHandler` subclass used when :meth:`Router.enable_debug() <mitogen.master.Router.enable_debug>` has been called, or the `debug` parameter was specified during context construction. Verifies the process ID has not changed on each call to :meth:`emit`, @@ -568,7 +640,7 @@ def _real_profile_hook(name, func, *args): return func(*args) finally: path = _profile_fmt % { - 'now': int(1e6 * time.time()), + 'now': int(1e6 * now()), 'identity': name, 'pid': os.getpid(), 'ext': '%s' @@ -596,6 +668,43 @@ def import_module(modname): return __import__(modname, None, None, ['']) +def pipe(): + """ + Create a UNIX pipe pair using :func:`os.pipe`, wrapping the returned + descriptors in Python file objects in order to manage their lifetime and + ensure they are closed when their last reference is discarded and they have + not been closed explicitly. + """ + rfd, wfd = os.pipe() + return ( + os.fdopen(rfd, 'rb', 0), + os.fdopen(wfd, 'wb', 0) + ) + + +def iter_split(buf, delim, func): + """ + Invoke `func(s)` for each `delim`-delimited chunk in the potentially large + `buf`, avoiding intermediate lists and quadratic string operations. Return + the trailing undelimited portion of `buf`, or any unprocessed portion of + `buf` after `func(s)` returned :data:`False`. + + :returns: + `(trailer, cont)`, where `cont` is :data:`False` if the last call to + `func(s)` returned :data:`False`. + """ + dlen = len(delim) + start = 0 + cont = True + while cont: + nl = buf.find(delim, start) + if nl == -1: + break + cont = not func(buf[start:nl]) is False + start = nl + dlen + return buf[start:], cont + + class Py24Pickler(py_pickle.Pickler): """ Exceptions were classic classes until Python 2.5. Sadly for 2.4, cPickle @@ -687,6 +796,10 @@ class Message(object): #: the :class:`mitogen.select.Select` interface. Defaults to :data:`None`. receiver = None + HEADER_FMT = '>hLLLLLL' + HEADER_LEN = struct.calcsize(HEADER_FMT) + HEADER_MAGIC = 0x4d49 # 'MI' + def __init__(self, **kwargs): """ Construct a message from from the supplied `kwargs`. :attr:`src_id` and @@ -697,6 +810,14 @@ class Message(object): vars(self).update(kwargs) assert isinstance(self.data, BytesType) + def pack(self): + return ( + struct.pack(self.HEADER_FMT, self.HEADER_MAGIC, self.dst_id, + self.src_id, self.auth_id, self.handle, + self.reply_to or 0, len(self.data)) + + self.data + ) + def _unpickle_context(self, context_id, name): return _unpickle_context(context_id, name, router=self.router) @@ -708,8 +829,10 @@ class Message(object): return s def _find_global(self, module, func): - """Return the class implementing `module_name.class_name` or raise - `StreamError` if the module is not whitelisted.""" + """ + Return the class implementing `module_name.class_name` or raise + `StreamError` if the module is not whitelisted. + """ if module == __name__: if func == '_unpickle_call_error' or func == 'CallError': return _unpickle_call_error @@ -744,7 +867,7 @@ class Message(object): """ Syntax helper to construct a dead message. """ - kwargs['data'], _ = UTF8_CODEC.encode(reason or u'') + kwargs['data'], _ = encodings.utf_8.encode(reason or u'') return cls(reply_to=IS_DEAD, **kwargs) @classmethod @@ -785,7 +908,8 @@ class Message(object): if msg.handle: (self.router or router).route(msg) else: - LOG.debug('Message.reply(): discarding due to zero handle: %r', msg) + LOG.debug('dropping reply to message with no return address: %r', + msg) if PY3: UNPICKLER_KWARGS = {'encoding': 'bytes'} @@ -824,7 +948,11 @@ class Message(object): unpickler.find_global = self._find_global try: # Must occur off the broker thread. - obj = unpickler.load() + try: + obj = unpickler.load() + except: + LOG.error('raw pickle was: %r', self.data) + raise self._unpickled = obj except (TypeError, ValueError): e = sys.exc_info()[1] @@ -851,7 +979,7 @@ class Sender(object): Senders may be serialized, making them convenient to wire up data flows. See :meth:`mitogen.core.Receiver.to_sender` for more information. - :param Context context: + :param mitogen.core.Context context: Context to send messages to. :param int dst_handle: Destination handle to send messages to. @@ -893,7 +1021,7 @@ def _unpickle_sender(router, context_id, dst_handle): if not (isinstance(router, Router) and isinstance(context_id, (int, long)) and context_id >= 0 and isinstance(dst_handle, (int, long)) and dst_handle > 0): - raise TypeError('cannot unpickle Sender: bad input') + raise TypeError('cannot unpickle Sender: bad input or missing router') return Sender(Context(router, context_id), dst_handle) @@ -920,11 +1048,11 @@ class Receiver(object): routed to the context due to disconnection, and ignores messages that did not originate from the respondent context. """ - #: If not :data:`None`, a reference to a function invoked as - #: `notify(receiver)` when a new message is delivered to this receiver. The - #: function is invoked on the broker thread, therefore it must not block. - #: Used by :class:`mitogen.select.Select` to implement waiting on multiple - #: receivers. + #: If not :data:`None`, a function invoked as `notify(receiver)` after a + #: message has been received. The function is invoked on :class:`Broker` + #: thread, therefore it must not block. Used by + #: :class:`mitogen.select.Select` to efficiently implement waiting on + #: multiple event sources. notify = None raise_channelerror = True @@ -997,13 +1125,32 @@ class Receiver(object): self.handle = None self._latch.close() + def size(self): + """ + Return the number of items currently buffered. + + As with :class:`Queue.Queue`, `0` may be returned even though a + subsequent call to :meth:`get` will succeed, since a message may be + posted at any moment between :meth:`size` and :meth:`get`. + + As with :class:`Queue.Queue`, `>0` may be returned even though a + subsequent call to :meth:`get` will block, since another waiting thread + may be woken at any moment between :meth:`size` and :meth:`get`. + + :raises LatchError: + The underlying latch has already been marked closed. + """ + return self._latch.size() + def empty(self): """ - Return :data:`True` if calling :meth:`get` would block. + Return `size() == 0`. + + .. deprecated:: 0.2.8 + Use :meth:`size` instead. - As with :class:`Queue.Queue`, :data:`True` may be returned even though - a subsequent call to :meth:`get` will succeed, since a message may be - posted at any moment between :meth:`empty` and :meth:`get`. + :raises LatchError: + The latch has already been marked closed. """ return self._latch.empty() @@ -1052,7 +1199,10 @@ class Channel(Sender, Receiver): A channel inherits from :class:`mitogen.core.Sender` and `mitogen.core.Receiver` to provide bidirectional functionality. - This class is incomplete and obsolete, it will be removed in Mitogen 0.3. + .. deprecated:: 0.2.0 + This class is incomplete and obsolete, it will be removed in Mitogen + 0.3. + Channels were an early attempt at syntax sugar. It is always easier to pass around unidirectional pairs of senders/receivers, even though the syntax is baroque: @@ -1089,6 +1239,7 @@ class Importer(object): # The Mitogen package is handled specially, since the child context must # construct it manually during startup. MITOGEN_PKG_CONTENT = [ + 'buildah', 'compat', 'debug', 'doas', @@ -1129,6 +1280,7 @@ class Importer(object): ALWAYS_BLACKLIST += ['cStringIO'] def __init__(self, router, context, core_src, whitelist=(), blacklist=()): + self._log = logging.getLogger('mitogen.importer') self._context = context self._present = {'mitogen': self.MITOGEN_PKG_CONTENT} self._lock = threading.Lock() @@ -1177,7 +1329,7 @@ class Importer(object): ) def __repr__(self): - return 'Importer()' + return 'Importer' def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a @@ -1202,18 +1354,18 @@ class Importer(object): _tls.running = True try: - _v and LOG.debug('%r.find_module(%r)', self, fullname) + #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) pkgname, dot, _ = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: - LOG.debug('%r: %r is submodule of a package we did not load', - self, fullname) + self._log.debug('%s is submodule of a locally loaded package', + fullname) return None suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): - LOG.debug('%r: master doesn\'t know %r', self, fullname) + self._log.debug('%s has no submodule %s', pkgname, suffix) return None # #114: explicitly whitelisted prefixes override any @@ -1224,10 +1376,9 @@ class Importer(object): try: self.builtin_find_module(fullname) - _vv and IOLOG.debug('%r: %r is available locally', - self, fullname) + _vv and self._log.debug('%r is available locally', fullname) except ImportError: - _vv and IOLOG.debug('find_module(%r) returning self', fullname) + _vv and self._log.debug('we will try to load %r', fullname) return self finally: del _tls.running @@ -1278,7 +1429,7 @@ class Importer(object): tup = msg.unpickle() fullname = tup[0] - _v and LOG.debug('Importer._on_load_module(%r)', fullname) + _v and self._log.debug('received %s', fullname) self._lock.acquire() try: @@ -1302,10 +1453,12 @@ class Importer(object): if not present: funcs = self._callbacks.get(fullname) if funcs is not None: - _v and LOG.debug('_request_module(%r): in flight', fullname) + _v and self._log.debug('existing request for %s in flight', + fullname) funcs.append(callback) else: - _v and LOG.debug('_request_module(%r): new request', fullname) + _v and self._log.debug('sending new %s request to parent', + fullname) self._callbacks[fullname] = [callback] self._context.send( Message(data=b(fullname), handle=GET_MODULE) @@ -1318,7 +1471,7 @@ class Importer(object): def load_module(self, fullname): fullname = to_text(fullname) - _v and LOG.debug('Importer.load_module(%r)', fullname) + _v and self._log.debug('requesting %s', fullname) self._refuse_imports(fullname) event = threading.Event() @@ -1342,7 +1495,7 @@ class Importer(object): if mod.__package__ and not PY3: # 2.x requires __package__ to be exactly a string. - mod.__package__, _ = UTF8_CODEC.encode(mod.__package__) + mod.__package__, _ = encodings.utf_8.encode(mod.__package__) source = self.get_source(fullname) try: @@ -1355,7 +1508,10 @@ class Importer(object): exec(code, vars(mod)) else: exec('exec code in vars(mod)') - return mod + + # #590: if a module replaces itself in sys.modules during import, below + # is necessary. This matches PyImport_ExecCodeModuleEx() + return sys.modules.get(fullname, mod) def get_filename(self, fullname): if fullname in self._cache: @@ -1381,11 +1537,30 @@ class Importer(object): class LogHandler(logging.Handler): + """ + A :class:`logging.Handler` subclass that arranges for :data:`FORWARD_LOG` + messages to be sent to a parent context in response to logging messages + generated by the current context. This is installed by default in child + contexts during bootstrap, so that :mod:`logging` events can be viewed and + managed centrally in the master process. + + The handler is initially *corked* after construction, such that it buffers + messages until :meth:`uncork` is called. This allows logging to be + installed prior to communication with the target being available, and + avoids any possible race where early log messages might be dropped. + + :param mitogen.core.Context context: + The context to send log messages towards. At present this is always + the master process. + """ def __init__(self, context): logging.Handler.__init__(self) self.context = context self.local = threading.local() self._buffer = [] + # Private synchronization is needed while corked, to ensure no + # concurrent call to _send() exists during uncork(). + self._buffer_lock = threading.Lock() def uncork(self): """ @@ -1393,15 +1568,30 @@ class LogHandler(logging.Handler): possible to route messages, therefore messages are buffered until :meth:`uncork` is called by :class:`ExternalContext`. """ - self._send = self.context.send - for msg in self._buffer: - self._send(msg) - self._buffer = None + self._buffer_lock.acquire() + try: + self._send = self.context.send + for msg in self._buffer: + self._send(msg) + self._buffer = None + finally: + self._buffer_lock.release() def _send(self, msg): - self._buffer.append(msg) + self._buffer_lock.acquire() + try: + if self._buffer is None: + # uncork() may run concurrent to _send() + self._send(msg) + else: + self._buffer.append(msg) + finally: + self._buffer_lock.release() def emit(self, rec): + """ + Send a :data:`FORWARD_LOG` message towards the target context. + """ if rec.name == 'mitogen.io' or \ getattr(self.local, 'in_emit', False): return @@ -1418,45 +1608,371 @@ class LogHandler(logging.Handler): self.local.in_emit = False +class Stream(object): + """ + A :class:`Stream` is one readable and optionally one writeable file + descriptor (represented by :class:`Side`) aggregated alongside an + associated :class:`Protocol` that knows how to respond to IO readiness + events for those descriptors. + + Streams are registered with :class:`Broker`, and callbacks are invoked on + the broker thread in response to IO activity. When registered using + :meth:`Broker.start_receive` or :meth:`Broker._start_transmit`, the broker + may call any of :meth:`on_receive`, :meth:`on_transmit`, + :meth:`on_shutdown` or :meth:`on_disconnect`. + + It is expected that the :class:`Protocol` associated with a stream will + change over its life. For example during connection setup, the initial + protocol may be :class:`mitogen.parent.BootstrapProtocol` that knows how to + enter SSH and sudo passwords and transmit the :mod:`mitogen.core` source to + the target, before handing off to :class:`MitogenProtocol` when the target + process is initialized. + + Streams connecting to children are in turn aggregated by + :class:`mitogen.parent.Connection`, which contains additional logic for + managing any child process, and a reference to any separate ``stderr`` + :class:`Stream` connected to that process. + """ + #: A :class:`Side` representing the stream's receive file descriptor. + receive_side = None + + #: A :class:`Side` representing the stream's transmit file descriptor. + transmit_side = None + + #: A :class:`Protocol` representing the protocol active on the stream. + protocol = None + + #: In parents, the :class:`mitogen.parent.Connection` instance. + conn = None + + #: The stream name. This is used in the :meth:`__repr__` output in any log + #: messages, it may be any descriptive string. + name = u'default' + + def set_protocol(self, protocol): + """ + Bind a :class:`Protocol` to this stream, by updating + :attr:`Protocol.stream` to refer to this stream, and updating this + stream's :attr:`Stream.protocol` to the refer to the protocol. Any + prior protocol's :attr:`Protocol.stream` is set to :data:`None`. + """ + if self.protocol: + self.protocol.stream = None + self.protocol = protocol + self.protocol.stream = self + + def accept(self, rfp, wfp): + """ + Attach a pair of file objects to :attr:`receive_side` and + :attr:`transmit_side`, after wrapping them in :class:`Side` instances. + :class:`Side` will call :func:`set_nonblock` and :func:`set_cloexec` + on the underlying file descriptors during construction. + + The same file object may be used for both sides. The default + :meth:`on_disconnect` is handles the possibility that only one + descriptor may need to be closed. + + :param file rfp: + The file object to receive from. + :param file wfp: + The file object to transmit to. + """ + self.receive_side = Side(self, rfp) + self.transmit_side = Side(self, wfp) + + def __repr__(self): + return "<Stream %s #%04x>" % (self.name, id(self) & 0xffff,) + + def on_receive(self, broker): + """ + Invoked by :class:`Broker` when the stream's :attr:`receive_side` has + been marked readable using :meth:`Broker.start_receive` and the broker + has detected the associated file descriptor is ready for reading. + + Subclasses must implement this if they are registered using + :meth:`Broker.start_receive`, and the method must invoke + :meth:`on_disconnect` if reading produces an empty string. + + The default implementation reads :attr:`Protocol.read_size` bytes and + passes the resulting bytestring to :meth:`Protocol.on_receive`. If the + bytestring is 0 bytes, invokes :meth:`on_disconnect` instead. + """ + buf = self.receive_side.read(self.protocol.read_size) + if not buf: + LOG.debug('%r: empty read, disconnecting', self.receive_side) + return self.on_disconnect(broker) + + self.protocol.on_receive(broker, buf) + + def on_transmit(self, broker): + """ + Invoked by :class:`Broker` when the stream's :attr:`transmit_side` has + been marked writeable using :meth:`Broker._start_transmit` and the + broker has detected the associated file descriptor is ready for + writing. + + Subclasses must implement they are ever registerd with + :meth:`Broker._start_transmit`. + + The default implementation invokes :meth:`Protocol.on_transmit`. + """ + self.protocol.on_transmit(broker) + + def on_shutdown(self, broker): + """ + Invoked by :meth:`Broker.shutdown` to allow the stream time to + gracefully shutdown. + + The default implementation emits a ``shutdown`` signal before + invoking :meth:`on_disconnect`. + """ + fire(self, 'shutdown') + self.protocol.on_shutdown(broker) + + def on_disconnect(self, broker): + """ + Invoked by :class:`Broker` to force disconnect the stream during + shutdown, invoked by the default :meth:`on_shutdown` implementation, + and usually invoked by any subclass :meth:`on_receive` implementation + in response to a 0-byte read. + + The base implementation fires a ``disconnect`` event, then closes + :attr:`receive_side` and :attr:`transmit_side` after unregistering the + stream from the broker. + """ + fire(self, 'disconnect') + self.protocol.on_disconnect(broker) + + +class Protocol(object): + """ + Implement the program behaviour associated with activity on a + :class:`Stream`. The protocol in use may vary over a stream's life, for + example to allow :class:`mitogen.parent.BootstrapProtocol` to initialize + the connected child before handing it off to :class:`MitogenProtocol`. A + stream's active protocol is tracked in the :attr:`Stream.protocol` + attribute, and modified via :meth:`Stream.set_protocol`. + + Protocols do not handle IO, they are entirely reliant on the interface + provided by :class:`Stream` and :class:`Side`, allowing the underlying IO + implementation to be replaced without modifying behavioural logic. + """ + stream_class = Stream + + #: The :class:`Stream` this protocol is currently bound to, or + #: :data:`None`. + stream = None + + #: The size of the read buffer used by :class:`Stream` when this is the + #: active protocol for the stream. + read_size = CHUNK_SIZE + + @classmethod + def build_stream(cls, *args, **kwargs): + stream = cls.stream_class() + stream.set_protocol(cls(*args, **kwargs)) + return stream + + def __repr__(self): + return '%s(%s)' % ( + self.__class__.__name__, + self.stream and self.stream.name, + ) + + def on_shutdown(self, broker): + _v and LOG.debug('%r: shutting down', self) + self.stream.on_disconnect(broker) + + def on_disconnect(self, broker): + # Normally both sides an FD, so it is important that tranmit_side is + # deregistered from Poller before closing the receive side, as pollers + # like epoll and kqueue unregister all events on FD close, causing + # subsequent attempt to unregister the transmit side to fail. + LOG.debug('%r: disconnecting', self) + broker.stop_receive(self.stream) + if self.stream.transmit_side: + broker._stop_transmit(self.stream) + + self.stream.receive_side.close() + if self.stream.transmit_side: + self.stream.transmit_side.close() + + +class DelimitedProtocol(Protocol): + """ + Provide a :meth:`Protocol.on_receive` implementation for protocols that are + delimited by a fixed string, like text based protocols. Each message is + passed to :meth:`on_line_received` as it arrives, with incomplete messages + passed to :meth:`on_partial_line_received`. + + When emulating user input it is often necessary to respond to incomplete + lines, such as when a "Password: " prompt is sent. + :meth:`on_partial_line_received` may be called repeatedly with an + increasingly complete message. When a complete message is finally received, + :meth:`on_line_received` will be called once for it before the buffer is + discarded. + + If :func:`on_line_received` returns :data:`False`, remaining data is passed + unprocessed to the stream's current protocol's :meth:`on_receive`. This + allows switching from line-oriented to binary while the input buffer + contains both kinds of data. + """ + #: The delimiter. Defaults to newline. + delimiter = b('\n') + _trailer = b('') + + def on_receive(self, broker, buf): + _vv and IOLOG.debug('%r.on_receive()', self) + stream = self.stream + self._trailer, cont = mitogen.core.iter_split( + buf=self._trailer + buf, + delim=self.delimiter, + func=self.on_line_received, + ) + + if self._trailer: + if cont: + self.on_partial_line_received(self._trailer) + else: + assert stream.protocol is not self + stream.protocol.on_receive(broker, self._trailer) + + def on_line_received(self, line): + """ + Receive a line from the stream. + + :param bytes line: + The encoded line, excluding the delimiter. + :returns: + :data:`False` to indicate this invocation modified the stream's + active protocol, and any remaining buffered data should be passed + to the new protocol's :meth:`on_receive` method. + + Any other return value is ignored. + """ + pass + + def on_partial_line_received(self, line): + """ + Receive a trailing unterminated partial line from the stream. + + :param bytes line: + The encoded partial line. + """ + pass + + +class BufferedWriter(object): + """ + Implement buffered output while avoiding quadratic string operations. This + is currently constructed by each protocol, in future it may become fixed + for each stream instead. + """ + def __init__(self, broker, protocol): + self._broker = broker + self._protocol = protocol + self._buf = collections.deque() + self._len = 0 + + def write(self, s): + """ + Transmit `s` immediately, falling back to enqueuing it and marking the + stream writeable if no OS buffer space is available. + """ + if not self._len: + # Modifying epoll/Kqueue state is expensive, as are needless broker + # loops. Rather than wait for writeability, just write immediately, + # and fall back to the broker loop on error or full buffer. + try: + n = self._protocol.stream.transmit_side.write(s) + if n: + if n == len(s): + return + s = s[n:] + except OSError: + pass + + self._broker._start_transmit(self._protocol.stream) + self._buf.append(s) + self._len += len(s) + + def on_transmit(self, broker): + """ + Respond to stream writeability by retrying previously buffered + :meth:`write` calls. + """ + if self._buf: + buf = self._buf.popleft() + written = self._protocol.stream.transmit_side.write(buf) + if not written: + _v and LOG.debug('disconnected during write to %r', self) + self._protocol.stream.on_disconnect(broker) + return + elif written != len(buf): + self._buf.appendleft(BufferType(buf, written)) + + _vv and IOLOG.debug('transmitted %d bytes to %r', written, self) + self._len -= written + + if not self._buf: + broker._stop_transmit(self._protocol.stream) + + class Side(object): """ - Represent a single side of a :class:`BasicStream`. This exists to allow - streams implemented using unidirectional (e.g. UNIX pipe) and bidirectional - (e.g. UNIX socket) file descriptors to operate identically. + Represent one side of a :class:`Stream`. This allows unidirectional (e.g. + pipe) and bidirectional (e.g. socket) streams to operate identically. + + Sides are also responsible for tracking the open/closed state of the + underlying FD, preventing erroneous duplicate calls to :func:`os.close` due + to duplicate :meth:`Stream.on_disconnect` calls, which would otherwise risk + silently succeeding by closing an unrelated descriptor. For this reason, it + is crucial only one file object exists per unique descriptor. :param mitogen.core.Stream stream: The stream this side is associated with. - - :param int fd: - Underlying file descriptor. - + :param object fp: + The file or socket object managing the underlying file descriptor. Any + object may be used that supports `fileno()` and `close()` methods. + :param bool cloexec: + If :data:`True`, the descriptor has its :data:`fcntl.FD_CLOEXEC` flag + enabled using :func:`fcntl.fcntl`. :param bool keep_alive: - Value for :attr:`keep_alive` - - During construction, the file descriptor has its :data:`os.O_NONBLOCK` flag - enabled using :func:`fcntl.fcntl`. + If :data:`True`, the continued existence of this side will extend the + shutdown grace period until it has been unregistered from the broker. + :param bool blocking: + If :data:`False`, the descriptor has its :data:`os.O_NONBLOCK` flag + enabled using :func:`fcntl.fcntl`. """ _fork_refs = weakref.WeakValueDictionary() + closed = False - def __init__(self, stream, fd, cloexec=True, keep_alive=True, blocking=False): + def __init__(self, stream, fp, cloexec=True, keep_alive=True, blocking=False): #: The :class:`Stream` for which this is a read or write side. self.stream = stream + # File or socket object responsible for the lifetime of its underlying + # file descriptor. + self.fp = fp #: Integer file descriptor to perform IO on, or :data:`None` if - #: :meth:`close` has been called. - self.fd = fd - self.closed = False + #: :meth:`close` has been called. This is saved separately from the + #: file object, since :meth:`file.fileno` cannot be called on it after + #: it has been closed. + self.fd = fp.fileno() #: If :data:`True`, causes presence of this side in #: :class:`Broker`'s active reader set to defer shutdown until the #: side is disconnected. self.keep_alive = keep_alive self._fork_refs[id(self)] = self if cloexec: - set_cloexec(fd) + set_cloexec(self.fd) if not blocking: - set_nonblock(fd) + set_nonblock(self.fd) def __repr__(self): - return '<Side of %r fd %s>' % (self.stream, self.fd) + return '<Side of %s fd %s>' % ( + self.stream.name or repr(self.stream), + self.fd + ) @classmethod def _on_fork(cls): @@ -1467,13 +1983,13 @@ class Side(object): def close(self): """ - Call :func:`os.close` on :attr:`fd` if it is not :data:`None`, + Call :meth:`file.close` on :attr:`fp` if it is not :data:`None`, then set it to :data:`None`. """ + _vv and IOLOG.debug('%r.close()', self) if not self.closed: - _vv and IOLOG.debug('%r.close()', self) self.closed = True - os.close(self.fd) + self.fp.close() def read(self, n=CHUNK_SIZE): """ @@ -1486,7 +2002,7 @@ class Side(object): in a 0-sized read like a regular file. :returns: - Bytes read, or the empty to string to indicate disconnection was + Bytes read, or the empty string to indicate disconnection was detected. """ if self.closed: @@ -1495,7 +2011,7 @@ class Side(object): return b('') s, disconnected = io_op(os.read, self.fd, n) if disconnected: - LOG.debug('%r.read(): disconnected: %s', self, disconnected) + LOG.debug('%r: disconnected during read: %s', self, disconnected) return b('') return s @@ -1509,107 +2025,59 @@ class Side(object): Number of bytes written, or :data:`None` if disconnection was detected. """ - if self.closed or self.fd is None: - # Refuse to touch the handle after closed, it may have been reused - # by another thread. + if self.closed: + # Don't touch the handle after close, it may be reused elsewhere. return None written, disconnected = io_op(os.write, self.fd, s) if disconnected: - LOG.debug('%r.write(): disconnected: %s', self, disconnected) + LOG.debug('%r: disconnected during write: %s', self, disconnected) return None return written -class BasicStream(object): - #: A :class:`Side` representing the stream's receive file descriptor. - receive_side = None - - #: A :class:`Side` representing the stream's transmit file descriptor. - transmit_side = None - - def on_receive(self, broker): - """ - Called by :class:`Broker` when the stream's :attr:`receive_side` has - been marked readable using :meth:`Broker.start_receive` and the broker - has detected the associated file descriptor is ready for reading. - - Subclasses must implement this if :meth:`Broker.start_receive` is ever - called on them, and the method must call :meth:`on_disconect` if - reading produces an empty string. - """ - pass - - def on_transmit(self, broker): - """ - Called by :class:`Broker` when the stream's :attr:`transmit_side` - has been marked writeable using :meth:`Broker._start_transmit` and - the broker has detected the associated file descriptor is ready for - writing. - - Subclasses must implement this if :meth:`Broker._start_transmit` is - ever called on them. - """ - pass - - def on_shutdown(self, broker): - """ - Called by :meth:`Broker.shutdown` to allow the stream time to - gracefully shutdown. The base implementation simply called - :meth:`on_disconnect`. - """ - _v and LOG.debug('%r.on_shutdown()', self) - fire(self, 'shutdown') - self.on_disconnect(broker) - - def on_disconnect(self, broker): - """ - Called by :class:`Broker` to force disconnect the stream. The base - implementation simply closes :attr:`receive_side` and - :attr:`transmit_side` and unregisters the stream from the broker. - """ - LOG.debug('%r.on_disconnect()', self) - if self.receive_side: - broker.stop_receive(self) - self.receive_side.close() - if self.transmit_side: - broker._stop_transmit(self) - self.transmit_side.close() - fire(self, 'disconnect') - - -class Stream(BasicStream): +class MitogenProtocol(Protocol): """ - :class:`BasicStream` subclass implementing mitogen's :ref:`stream - protocol <stream-protocol>`. + :class:`Protocol` implementing mitogen's :ref:`stream protocol + <stream-protocol>`. """ - #: If not :data:`None`, :class:`Router` stamps this into - #: :attr:`Message.auth_id` of every message received on this stream. - auth_id = None - #: If not :data:`False`, indicates the stream has :attr:`auth_id` set and #: its value is the same as :data:`mitogen.context_id` or appears in #: :data:`mitogen.parent_ids`. is_privileged = False - def __init__(self, router, remote_id, **kwargs): + def __init__(self, router, remote_id, auth_id=None, + local_id=None, parent_ids=None): self._router = router self.remote_id = remote_id - self.name = u'default' + #: If not :data:`None`, :class:`Router` stamps this into + #: :attr:`Message.auth_id` of every message received on this stream. + self.auth_id = auth_id + + if parent_ids is None: + parent_ids = mitogen.parent_ids + if local_id is None: + local_id = mitogen.context_id + + self.is_privileged = ( + (remote_id in parent_ids) or + auth_id in ([local_id] + parent_ids) + ) self.sent_modules = set(['mitogen', 'mitogen.core']) - self.construct(**kwargs) self._input_buf = collections.deque() - self._output_buf = collections.deque() self._input_buf_len = 0 - self._output_buf_len = 0 + self._writer = BufferedWriter(router.broker, self) + #: Routing records the dst_id of every message arriving from this #: stream. Any arriving DEL_ROUTE is rebroadcast for any such ID. self.egress_ids = set() - def construct(self): - pass - - def _internal_receive(self, broker, buf): + def on_receive(self, broker, buf): + """ + Handle the next complete message on the stream. Raise + :class:`StreamError` on failure. + """ + _vv and IOLOG.debug('%r.on_receive()', self) if self._input_buf and self._input_buf_len < 128: self._input_buf[0] += buf else: @@ -1619,60 +2087,45 @@ class Stream(BasicStream): while self._receive_one(broker): pass - def on_receive(self, broker): - """Handle the next complete message on the stream. Raise - :class:`StreamError` on failure.""" - _vv and IOLOG.debug('%r.on_receive()', self) - - buf = self.receive_side.read() - if not buf: - return self.on_disconnect(broker) - - self._internal_receive(broker, buf) - - HEADER_FMT = '>hLLLLLL' - HEADER_LEN = struct.calcsize(HEADER_FMT) - HEADER_MAGIC = 0x4d49 # 'MI' - corrupt_msg = ( - 'Corruption detected: frame signature incorrect. This likely means ' - 'some external process is interfering with the connection. Received:' + '%s: Corruption detected: frame signature incorrect. This likely means' + ' some external process is interfering with the connection. Received:' '\n\n' '%r' ) def _receive_one(self, broker): - if self._input_buf_len < self.HEADER_LEN: + if self._input_buf_len < Message.HEADER_LEN: return False msg = Message() msg.router = self._router (magic, msg.dst_id, msg.src_id, msg.auth_id, msg.handle, msg.reply_to, msg_len) = struct.unpack( - self.HEADER_FMT, - self._input_buf[0][:self.HEADER_LEN], + Message.HEADER_FMT, + self._input_buf[0][:Message.HEADER_LEN], ) - if magic != self.HEADER_MAGIC: - LOG.error(self.corrupt_msg, self._input_buf[0][:2048]) - self.on_disconnect(broker) + if magic != Message.HEADER_MAGIC: + LOG.error(self.corrupt_msg, self.stream.name, self._input_buf[0][:2048]) + self.stream.on_disconnect(broker) return False if msg_len > self._router.max_message_size: - LOG.error('Maximum message size exceeded (got %d, max %d)', - msg_len, self._router.max_message_size) - self.on_disconnect(broker) + LOG.error('%r: Maximum message size exceeded (got %d, max %d)', + self, msg_len, self._router.max_message_size) + self.stream.on_disconnect(broker) return False - total_len = msg_len + self.HEADER_LEN + total_len = msg_len + Message.HEADER_LEN if self._input_buf_len < total_len: _vv and IOLOG.debug( '%r: Input too short (want %d, got %d)', - self, msg_len, self._input_buf_len - self.HEADER_LEN + self, msg_len, self._input_buf_len - Message.HEADER_LEN ) return False - start = self.HEADER_LEN + start = Message.HEADER_LEN prev_start = start remain = total_len bits = [] @@ -1687,7 +2140,7 @@ class Stream(BasicStream): msg.data = b('').join(bits) self._input_buf.appendleft(buf[prev_start+len(bit):]) self._input_buf_len -= total_len - self._router._async_route(msg, self) + self._router._async_route(msg, self.stream) return True def pending_bytes(self): @@ -1699,68 +2152,31 @@ class Stream(BasicStream): For an accurate result, this method should be called from the Broker thread, for example by using :meth:`Broker.defer_sync`. """ - return self._output_buf_len + return self._writer._len def on_transmit(self, broker): - """Transmit buffered messages.""" + """ + Transmit buffered messages. + """ _vv and IOLOG.debug('%r.on_transmit()', self) - - if self._output_buf: - buf = self._output_buf.popleft() - written = self.transmit_side.write(buf) - if not written: - _v and LOG.debug('%r.on_transmit(): disconnection detected', self) - self.on_disconnect(broker) - return - elif written != len(buf): - self._output_buf.appendleft(BufferType(buf, written)) - - _vv and IOLOG.debug('%r.on_transmit() -> len %d', self, written) - self._output_buf_len -= written - - if not self._output_buf: - broker._stop_transmit(self) + self._writer.on_transmit(broker) def _send(self, msg): _vv and IOLOG.debug('%r._send(%r)', self, msg) - pkt = struct.pack(self.HEADER_FMT, self.HEADER_MAGIC, msg.dst_id, - msg.src_id, msg.auth_id, msg.handle, - msg.reply_to or 0, len(msg.data)) + msg.data - - if not self._output_buf_len: - # Modifying epoll/Kqueue state is expensive, as are needless broker - # loops. Rather than wait for writeability, just write immediately, - # and fall back to the broker loop on error or full buffer. - try: - n = self.transmit_side.write(pkt) - if n: - if n == len(pkt): - return - pkt = pkt[n:] - except OSError: - pass - - self._router.broker._start_transmit(self) - self._output_buf.append(pkt) - self._output_buf_len += len(pkt) + self._writer.write(msg.pack()) def send(self, msg): - """Send `data` to `handle`, and tell the broker we have output. May - be called from any thread.""" + """ + Send `data` to `handle`, and tell the broker we have output. May be + called from any thread. + """ self._router.broker.defer(self._send, msg) def on_shutdown(self, broker): - """Override BasicStream behaviour of immediately disconnecting.""" - _v and LOG.debug('%r.on_shutdown(%r)', self, broker) - - def accept(self, rfd, wfd): - # TODO: what is this os.dup for? - self.receive_side = Side(self, os.dup(rfd)) - self.transmit_side = Side(self, os.dup(wfd)) - - def __repr__(self): - cls = type(self) - return "%s.%s('%s')" % (cls.__module__, cls.__name__, self.name) + """ + Disable :class:`Protocol` immediate disconnect behaviour. + """ + _v and LOG.debug('%r: shutting down', self) class Context(object): @@ -1779,28 +2195,27 @@ class Context(object): explicitly, as that method is deduplicating, and returns the only context instance :ref:`signals` will be raised on. - :param Router router: + :param mitogen.core.Router router: Router to emit messages through. :param int context_id: Context ID. :param str name: Context name. """ + name = None remote_name = None def __init__(self, router, context_id, name=None): self.router = router self.context_id = context_id - self.name = name + if name: + self.name = to_text(name) def __reduce__(self): - name = self.name - if name and not isinstance(name, UnicodeType): - name = UnicodeType(name, 'utf-8') - return _unpickle_context, (self.context_id, name) + return _unpickle_context, (self.context_id, self.name) def on_disconnect(self): - _v and LOG.debug('%r.on_disconnect()', self) + _v and LOG.debug('%r: disconnecting', self) fire(self, 'disconnect') def send_async(self, msg, persist=False): @@ -1821,20 +2236,17 @@ class Context(object): :class:`Receiver` configured to receive any replies sent to the message's `reply_to` handle. """ - if self.router.broker._thread == threading.currentThread(): # TODO - raise SystemError('Cannot making blocking call on broker thread') - receiver = Receiver(self.router, persist=persist, respondent=self) msg.dst_id = self.context_id msg.reply_to = receiver.handle - _v and LOG.debug('%r.send_async(%r)', self, msg) + _v and LOG.debug('sending message to %r: %r', self, msg) self.send(msg) return receiver def call_service_async(self, service_name, method_name, **kwargs): - _v and LOG.debug('%r.call_service_async(%r, %r, %r)', - self, service_name, method_name, kwargs) + _v and LOG.debug('calling service %s.%s of %r, args: %r', + service_name, method_name, self, kwargs) if isinstance(service_name, BytesType): service_name = service_name.encode('utf-8') elif not isinstance(service_name, UnicodeType): @@ -1942,7 +2354,7 @@ class Poller(object): self._wfds = {} def __repr__(self): - return '%s(%#x)' % (type(self).__name__, id(self)) + return '%s' % (type(self).__name__,) def _update(self, fd): """ @@ -2025,9 +2437,6 @@ class Poller(object): if gen and gen < self._generation: yield data - if timeout: - timeout *= 1000 - def poll(self, timeout=None): """ Block the calling thread until one or more FDs are ready for IO. @@ -2059,8 +2468,18 @@ class Latch(object): See :ref:`waking-sleeping-threads` for further discussion. """ + #: The :class:`Poller` implementation to use for waiting. Since the poller + #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller` + #: if it is available, or :class:`mitogen.core.Poller` otherwise, since + #: these implementations require no system calls to create, configure or + #: destroy. poller_class = Poller + #: If not :data:`None`, a function invoked as `notify(latch)` after a + #: successful call to :meth:`put`. The function is invoked on the + #: :meth:`put` caller's thread, which may be the :class:`Broker` thread, + #: therefore it must not block. Used by :class:`mitogen.select.Select` to + #: efficiently implement waiting on multiple event sources. notify = None # The _cls_ prefixes here are to make it crystal clear in the code which @@ -2113,19 +2532,17 @@ class Latch(object): finally: self._lock.release() - def empty(self): + def size(self): """ - Return :data:`True` if calling :meth:`get` would block. + Return the number of items currently buffered. - As with :class:`Queue.Queue`, :data:`True` may be returned even - though a subsequent call to :meth:`get` will succeed, since a - message may be posted at any moment between :meth:`empty` and - :meth:`get`. + As with :class:`Queue.Queue`, `0` may be returned even though a + subsequent call to :meth:`get` will succeed, since a message may be + posted at any moment between :meth:`size` and :meth:`get`. - As with :class:`Queue.Queue`, :data:`False` may be returned even - though a subsequent call to :meth:`get` will block, since another - waiting thread may be woken at any moment between :meth:`empty` and - :meth:`get`. + As with :class:`Queue.Queue`, `>0` may be returned even though a + subsequent call to :meth:`get` will block, since another waiting thread + may be woken at any moment between :meth:`size` and :meth:`get`. :raises LatchError: The latch has already been marked closed. @@ -2134,10 +2551,22 @@ class Latch(object): try: if self.closed: raise LatchError() - return len(self._queue) == 0 + return len(self._queue) finally: self._lock.release() + def empty(self): + """ + Return `size() == 0`. + + .. deprecated:: 0.2.8 + Use :meth:`size` instead. + + :raises LatchError: + The latch has already been marked closed. + """ + return self.size() == 0 + def _get_socketpair(self): """ Return an unused socketpair, creating one if none exist. @@ -2215,7 +2644,7 @@ class Latch(object): :meth:`put` to write a byte to our socket pair. """ _vv and IOLOG.debug( - '%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)', + '%r._get_sleep(timeout=%r, block=%r, fd=%d/%d)', self, timeout, block, rsock.fileno(), wsock.fileno() ) @@ -2270,17 +2699,20 @@ class Latch(object): raise LatchError() self._queue.append(obj) + wsock = None if self._waking < len(self._sleeping): wsock, cookie = self._sleeping[self._waking] self._waking += 1 _vv and IOLOG.debug('%r.put() -> waking wfd=%r', self, wsock.fileno()) - self._wake(wsock, cookie) elif self.notify: self.notify(self) finally: self._lock.release() + if wsock: + self._wake(wsock, cookie) + def _wake(self, wsock, cookie): written, disconnected = io_op(os.write, wsock.fileno(), cookie) assert written == len(cookie) and not disconnected @@ -2293,30 +2725,32 @@ class Latch(object): ) -class Waker(BasicStream): +class Waker(Protocol): """ - :class:`BasicStream` subclass implementing the `UNIX self-pipe trick`_. - Used to wake the multiplexer when another thread needs to modify its state - (via a cross-thread function call). + :class:`Protocol` implementing the `UNIX self-pipe trick`_. Used to wake + :class:`Broker` when another thread needs to modify its state, by enqueing + a function call to run on the :class:`Broker` thread. .. _UNIX self-pipe trick: https://cr.yp.to/docs/selfpipe.html """ + read_size = 1 broker_ident = None + @classmethod + def build_stream(cls, broker): + stream = super(Waker, cls).build_stream(broker) + stream.accept(*pipe()) + return stream + def __init__(self, broker): self._broker = broker self._lock = threading.Lock() self._deferred = [] - rfd, wfd = os.pipe() - self.receive_side = Side(self, rfd) - self.transmit_side = Side(self, wfd) - def __repr__(self): - return 'Waker(%r rfd=%r, wfd=%r)' % ( - self._broker, - self.receive_side and self.receive_side.fd, - self.transmit_side and self.transmit_side.fd, + return 'Waker(fd=%r/%r)' % ( + self.stream.receive_side and self.stream.receive_side.fd, + self.stream.transmit_side and self.stream.transmit_side.fd, ) @property @@ -2330,7 +2764,7 @@ class Waker(BasicStream): finally: self._lock.release() - def on_receive(self, broker): + def on_receive(self, broker, buf): """ Drain the pipe and fire callbacks. Since :attr:`_deferred` is synchronized, :meth:`defer` and :meth:`on_receive` can conspire to @@ -2339,7 +2773,6 @@ class Waker(BasicStream): _vv and IOLOG.debug('%r.on_receive()', self) self._lock.acquire() try: - self.receive_side.read(1) deferred = self._deferred self._deferred = [] finally: @@ -2351,7 +2784,7 @@ class Waker(BasicStream): except Exception: LOG.exception('defer() crashed: %r(*%r, **%r)', func, args, kwargs) - self._broker.shutdown() + broker.shutdown() def _wake(self): """ @@ -2359,7 +2792,7 @@ class Waker(BasicStream): teardown, the FD may already be closed, so ignore EBADF. """ try: - self.transmit_side.write(b(' ')) + self.stream.transmit_side.write(b(' ')) except OSError: e = sys.exc_info()[1] if e.args[0] != errno.EBADF: @@ -2386,65 +2819,74 @@ class Waker(BasicStream): if self._broker._exitted: raise Error(self.broker_shutdown_msg) - _vv and IOLOG.debug('%r.defer() [fd=%r]', self, self.transmit_side.fd) + _vv and IOLOG.debug('%r.defer() [fd=%r]', self, + self.stream.transmit_side.fd) self._lock.acquire() try: - if not self._deferred: - self._wake() + should_wake = not self._deferred self._deferred.append((func, args, kwargs)) finally: self._lock.release() + if should_wake: + self._wake() -class IoLogger(BasicStream): - """ - :class:`BasicStream` subclass that sets up redirection of a standard - UNIX file descriptor back into the Python :mod:`logging` package. - """ - _buf = '' - def __init__(self, broker, name, dest_fd): - self._broker = broker - self._name = name - self._rsock, self._wsock = socket.socketpair() - os.dup2(self._wsock.fileno(), dest_fd) - set_cloexec(self._wsock.fileno()) +class IoLoggerProtocol(DelimitedProtocol): + """ + Attached to one end of a socket pair whose other end overwrites one of the + standard ``stdout`` or ``stderr`` file descriptors in a child context. + Received data is split up into lines, decoded as UTF-8 and logged to the + :mod:`logging` package as either the ``stdout`` or ``stderr`` logger. + Logging in child contexts is in turn forwarded to the master process using + :class:`LogHandler`. + """ + @classmethod + def build_stream(cls, name, dest_fd): + """ + Even though the file descriptor `dest_fd` will hold the opposite end of + the socket open, we must keep a separate dup() of it (i.e. wsock) in + case some code decides to overwrite `dest_fd` later, which would + prevent break :meth:`on_shutdown` from calling :meth:`shutdown() + <socket.socket.shutdown>` on it. + """ + rsock, wsock = socket.socketpair() + os.dup2(wsock.fileno(), dest_fd) + stream = super(IoLoggerProtocol, cls).build_stream(name) + stream.name = name + stream.accept(rsock, wsock) + return stream + + def __init__(self, name): self._log = logging.getLogger(name) # #453: prevent accidental log initialization in a child creating a # feedback loop. self._log.propagate = False self._log.handlers = logging.getLogger().handlers[:] - self.receive_side = Side(self, self._rsock.fileno()) - self.transmit_side = Side(self, dest_fd, cloexec=False, blocking=True) - self._broker.start_receive(self) - - def __repr__(self): - return '<IoLogger %s>' % (self._name,) - - def _log_lines(self): - while self._buf.find('\n') != -1: - line, _, self._buf = str_partition(self._buf, '\n') - self._log.info('%s', line.rstrip('\n')) - def on_shutdown(self, broker): - """Shut down the write end of the logging socket.""" - _v and LOG.debug('%r.on_shutdown()', self) + """ + Shut down the write end of the socket, preventing any further writes to + it by this process, or subprocess that inherited it. This allows any + remaining kernel-buffered data to be drained during graceful shutdown + without the buffer continuously refilling due to some out of control + child process. + """ + _v and LOG.debug('%r: shutting down', self) if not IS_WSL: - # #333: WSL generates invalid readiness indication on shutdown() - self._wsock.shutdown(socket.SHUT_WR) - self._wsock.close() - self.transmit_side.close() - - def on_receive(self, broker): - _vv and IOLOG.debug('%r.on_receive()', self) - buf = self.receive_side.read() - if not buf: - return self.on_disconnect(broker) + # #333: WSL generates invalid readiness indication on shutdown(). + # This modifies the *kernel object* inherited by children, causing + # EPIPE on subsequent writes to any dupped FD in any process. The + # read side can then drain completely of prior buffered data. + self.stream.transmit_side.fp.shutdown(socket.SHUT_WR) + self.stream.transmit_side.close() - self._buf += buf.decode('latin1') - self._log_lines() + def on_line_received(self, line): + """ + Decode the received line as UTF-8 and pass it to the logging framework. + """ + self._log.info('%s', line.decode('utf-8', 'replace')) class Router(object): @@ -2456,7 +2898,12 @@ class Router(object): **Note:** This is the somewhat limited core version of the Router class used by child contexts. The master subclass is documented below this one. """ + #: The :class:`mitogen.core.Context` subclass to use when constructing new + #: :class:`Context` objects in :meth:`myself` and :meth:`context_by_id`. + #: Permits :class:`Router` subclasses to extend the :class:`Context` + #: interface, as done in :class:`mitogen.parent.Router`. context_class = Context + max_message_size = 128 * 1048576 #: When :data:`True`, permit children to only communicate with the current @@ -2476,6 +2923,18 @@ class Router(object): #: parameter. unidirectional = False + duplicate_handle_msg = 'cannot register a handle that already exists' + refused_msg = 'refused by policy' + invalid_handle_msg = 'invalid handle' + too_large_msg = 'message too large (max %d bytes)' + respondent_disconnect_msg = 'the respondent Context has disconnected' + broker_exit_msg = 'Broker has exitted' + no_route_msg = 'no route to %r, my ID is %r' + unidirectional_msg = ( + 'routing mode prevents forward of message from context %d to ' + 'context %d via context %d' + ) + def __init__(self, broker): self.broker = broker listen(broker, 'exit', self._on_broker_exit) @@ -2514,12 +2973,13 @@ class Router(object): corresponding :attr:`_context_by_id` member. This is replaced by :class:`mitogen.parent.RouteMonitor` in an upgraded context. """ - LOG.error('%r._on_del_route() %r', self, msg) if msg.is_dead: return target_id_s, _, name = bytes_partition(msg.data, b(':')) target_id = int(target_id_s, 10) + LOG.error('%r: deleting route to %s (%d)', + self, to_text(name), target_id) context = self._context_by_id.get(target_id) if context: fire(context, 'disconnect') @@ -2542,16 +3002,21 @@ class Router(object): for context in notify: context.on_disconnect() - broker_exit_msg = 'Broker has exitted' - def _on_broker_exit(self): + """ + Called prior to broker exit, informs callbacks registered with + :meth:`add_handler` the connection is dead. + """ + _v and LOG.debug('%r: broker has exitted', self) while self._handle_map: _, (_, func, _, _) = self._handle_map.popitem() func(Message.dead(self.broker_exit_msg)) def myself(self): """ - Return a :class:`Context` referring to the current process. + Return a :class:`Context` referring to the current process. Since + :class:`Context` is serializable, this is convenient to use in remote + function call parameter lists. """ return self.context_class( router=self, @@ -2561,8 +3026,25 @@ class Router(object): def context_by_id(self, context_id, via_id=None, create=True, name=None): """ - Messy factory/lookup function to find a context by its ID, or construct - it. This will eventually be replaced by a more sensible interface. + Return or construct a :class:`Context` given its ID. An internal + mapping of ID to the canonical :class:`Context` representing that ID, + so that :ref:`signals` can be raised. + + This may be called from any thread, lookup and construction are atomic. + + :param int context_id: + The context ID to look up. + :param int via_id: + If the :class:`Context` does not already exist, set its + :attr:`Context.via` to the :class:`Context` matching this ID. + :param bool create: + If the :class:`Context` does not already exist, create it. + :param str name: + If the :class:`Context` does not already exist, set its name. + + :returns: + :class:`Context`, or return :data:`None` if `create` is + :data:`False` and no :class:`Context` previously existed. """ context = self._context_by_id.get(context_id) if context: @@ -2591,7 +3073,8 @@ class Router(object): the stream's receive side to the I/O multiplexer. This method remains public while the design has not yet settled. """ - _v and LOG.debug('register(%r, %r)', context, stream) + _v and LOG.debug('%s: registering %r to stream %r', + self, context, stream) self._write_lock.acquire() try: self._stream_by_id[context.context_id] = stream @@ -2606,7 +3089,13 @@ class Router(object): """ Return the :class:`Stream` that should be used to communicate with `dst_id`. If a specific route for `dst_id` is not known, a reference to - the parent context's stream is returned. + the parent context's stream is returned. If the parent is disconnected, + or when running in the master context, return :data:`None` instead. + + This can be used from any thread, but its output is only meaningful + from the context of the :class:`Broker` thread, as disconnection or + replacement could happen in parallel on the broker thread at any + moment. """ return ( self._stream_by_id.get(dst_id) or @@ -2642,7 +3131,7 @@ class Router(object): If :data:`False`, the handler will be unregistered after a single message has been received. - :param Context respondent: + :param mitogen.core.Context respondent: Context that messages to this handle are expected to be sent from. If specified, arranges for a dead message to be delivered to `fn` when disconnection of the context is detected. @@ -2696,55 +3185,61 @@ class Router(object): return handle - duplicate_handle_msg = 'cannot register a handle that is already exists' - refused_msg = 'refused by policy' - invalid_handle_msg = 'invalid handle' - too_large_msg = 'message too large (max %d bytes)' - respondent_disconnect_msg = 'the respondent Context has disconnected' - broker_shutdown_msg = 'Broker is shutting down' - no_route_msg = 'no route to %r, my ID is %r' - unidirectional_msg = ( - 'routing mode prevents forward of message from context %d via ' - 'context %d' - ) - def _on_respondent_disconnect(self, context): for handle in self._handles_by_respondent.pop(context, ()): _, fn, _, _ = self._handle_map[handle] fn(Message.dead(self.respondent_disconnect_msg)) del self._handle_map[handle] - def on_shutdown(self, broker): - """Called during :meth:`Broker.shutdown`, informs callbacks registered - with :meth:`add_handle_cb` the connection is dead.""" - _v and LOG.debug('%r.on_shutdown(%r)', self, broker) - fire(self, 'shutdown') - for handle, (persist, fn) in self._handle_map.iteritems(): - _v and LOG.debug('%r.on_shutdown(): killing %r: %r', self, handle, fn) - fn(Message.dead(self.broker_shutdown_msg)) + def _maybe_send_dead(self, unreachable, msg, reason, *args): + """ + Send a dead message to either the original sender or the intended + recipient of `msg`, if the original sender was expecting a reply + (because its `reply_to` was set), otherwise assume the message is a + reply of some sort, and send the dead message to the original + destination. - def _maybe_send_dead(self, msg, reason, *args): + :param bool unreachable: + If :data:`True`, the recipient is known to be dead or routing + failed due to a security precaution, so don't attempt to fallback + to sending the dead message to the recipient if the original sender + did not include a reply address. + :param mitogen.core.Message msg: + Message that triggered the dead message. + :param str reason: + Human-readable error reason. + :param tuple args: + Elements to interpolate with `reason`. + """ if args: reason %= args LOG.debug('%r: %r is dead: %r', self, msg, reason) if msg.reply_to and not msg.is_dead: msg.reply(Message.dead(reason=reason), router=self) + elif not unreachable: + self._async_route( + Message.dead( + dst_id=msg.dst_id, + handle=msg.handle, + reason=reason, + ) + ) def _invoke(self, msg, stream): # IOLOG.debug('%r._invoke(%r)', self, msg) try: persist, fn, policy, respondent = self._handle_map[msg.handle] except KeyError: - self._maybe_send_dead(msg, reason=self.invalid_handle_msg) + self._maybe_send_dead(True, msg, reason=self.invalid_handle_msg) return if respondent and not (msg.is_dead or msg.src_id == respondent.context_id): - self._maybe_send_dead(msg, 'reply from unexpected context') + self._maybe_send_dead(True, msg, 'reply from unexpected context') return if policy and not policy(msg, stream): - self._maybe_send_dead(msg, self.refused_msg) + self._maybe_send_dead(True, msg, self.refused_msg) return if not persist: @@ -2772,52 +3267,69 @@ class Router(object): _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, in_stream) if len(msg.data) > self.max_message_size: - self._maybe_send_dead(msg, self.too_large_msg % ( + self._maybe_send_dead(False, msg, self.too_large_msg % ( self.max_message_size, )) return - # Perform source verification. + parent_stream = self._stream_by_id.get(mitogen.parent_id) + src_stream = self._stream_by_id.get(msg.src_id, parent_stream) + + # When the ingress stream is known, verify the message was received on + # the same as the stream we would expect to receive messages from the + # src_id and auth_id. This is like Reverse Path Filtering in IP, and + # ensures messages from a privileged context cannot be spoofed by a + # child. if in_stream: - parent = self._stream_by_id.get(mitogen.parent_id) - expect = self._stream_by_id.get(msg.auth_id, parent) - if in_stream != expect: + auth_stream = self._stream_by_id.get(msg.auth_id, parent_stream) + if in_stream != auth_stream: LOG.error('%r: bad auth_id: got %r via %r, not %r: %r', - self, msg.auth_id, in_stream, expect, msg) + self, msg.auth_id, in_stream, auth_stream, msg) return - if msg.src_id != msg.auth_id: - expect = self._stream_by_id.get(msg.src_id, parent) - if in_stream != expect: - LOG.error('%r: bad src_id: got %r via %r, not %r: %r', - self, msg.src_id, in_stream, expect, msg) - return + if msg.src_id != msg.auth_id and in_stream != src_stream: + LOG.error('%r: bad src_id: got %r via %r, not %r: %r', + self, msg.src_id, in_stream, src_stream, msg) + return - if in_stream.auth_id is not None: - msg.auth_id = in_stream.auth_id + # If the stream's MitogenProtocol has auth_id set, copy it to the + # message. This allows subtrees to become privileged by stamping a + # parent's context ID. It is used by mitogen.unix to mark client + # streams (like Ansible WorkerProcess) as having the same rights as + # the parent. + if in_stream.protocol.auth_id is not None: + msg.auth_id = in_stream.protocol.auth_id - # Maintain a set of IDs the source ever communicated with. - in_stream.egress_ids.add(msg.dst_id) + # Record the IDs the source ever communicated with. + in_stream.protocol.egress_ids.add(msg.dst_id) if msg.dst_id == mitogen.context_id: return self._invoke(msg, in_stream) out_stream = self._stream_by_id.get(msg.dst_id) - if out_stream is None: - out_stream = self._stream_by_id.get(mitogen.parent_id) + if (not out_stream) and (parent_stream != src_stream or not in_stream): + # No downstream route exists. The message could be from a child or + # ourselves for a parent, in which case we must forward it + # upstream, or it could be from a parent for a dead child, in which + # case its src_id/auth_id would fail verification if returned to + # the parent, so in that case reply with a dead message instead. + out_stream = parent_stream if out_stream is None: - self._maybe_send_dead(msg, self.no_route_msg, + self._maybe_send_dead(True, msg, self.no_route_msg, msg.dst_id, mitogen.context_id) return if in_stream and self.unidirectional and not \ - (in_stream.is_privileged or out_stream.is_privileged): - self._maybe_send_dead(msg, self.unidirectional_msg, - in_stream.remote_id, out_stream.remote_id) + (in_stream.protocol.is_privileged or + out_stream.protocol.is_privileged): + self._maybe_send_dead(True, msg, self.unidirectional_msg, + in_stream.protocol.remote_id, + out_stream.protocol.remote_id, + mitogen.context_id) return - out_stream._send(msg) + out_stream.protocol._send(msg) def route(self, msg): """ @@ -2832,17 +3344,26 @@ class Router(object): self.broker.defer(self._async_route, msg) +class NullTimerList(object): + def get_timeout(self): + return None + + class Broker(object): """ Responsible for handling I/O multiplexing in a private thread. - **Note:** This is the somewhat limited core version of the Broker class - used by child contexts. The master subclass is documented below. + **Note:** This somewhat limited core version is used by children. The + master subclass is documented below. """ poller_class = Poller _waker = None _thread = None + # :func:`mitogen.parent._upgrade_broker` replaces this with + # :class:`mitogen.parent.TimerList` during upgrade. + timers = NullTimerList() + #: Seconds grace to allow :class:`streams <Stream>` to shutdown gracefully #: before force-disconnecting them during :meth:`shutdown`. shutdown_timeout = 3.0 @@ -2850,11 +3371,11 @@ class Broker(object): def __init__(self, poller_class=None, activate_compat=True): self._alive = True self._exitted = False - self._waker = Waker(self) + self._waker = Waker.build_stream(self) #: Arrange for `func(\*args, \**kwargs)` to be executed on the broker #: thread, or immediately if the current thread is the broker thread. #: Safe to call from any thread. - self.defer = self._waker.defer + self.defer = self._waker.protocol.defer self.poller = self.poller_class() self.poller.start_receive( self._waker.receive_side.fd, @@ -2877,7 +3398,7 @@ class Broker(object): if sys.version_info < (2, 6): # import_module() is used to avoid dep scanner. os_fork = import_module('mitogen.os_fork') - mitogen.os_fork._notice_broker_or_pool(self) + os_fork._notice_broker_or_pool(self) def start_receive(self, stream): """ @@ -2888,7 +3409,7 @@ class Broker(object): """ _vv and IOLOG.debug('%r.start_receive(%r)', self, stream) side = stream.receive_side - assert side and side.fd is not None + assert side and not side.closed self.defer(self.poller.start_receive, side.fd, (side, stream.on_receive)) @@ -2909,7 +3430,7 @@ class Broker(object): """ _vv and IOLOG.debug('%r._start_transmit(%r)', self, stream) side = stream.transmit_side - assert side and side.fd is not None + assert side and not side.closed self.poller.start_transmit(side.fd, (side, stream.on_transmit)) def _stop_transmit(self, stream): @@ -2928,7 +3449,7 @@ class Broker(object): progress (e.g. log draining). """ it = (side.keep_alive for (_, (side, _)) in self.poller.readers) - return sum(it, 0) + return sum(it, 0) > 0 or self.timers.get_timeout() is not None def defer_sync(self, func): """ @@ -2971,10 +3492,19 @@ class Broker(object): """ _vv and IOLOG.debug('%r._loop_once(%r, %r)', self, timeout, self.poller) + + timer_to = self.timers.get_timeout() + if timeout is None: + timeout = timer_to + elif timer_to is not None and timer_to < timeout: + timeout = timer_to + #IOLOG.debug('readers =\n%s', pformat(self.poller.readers)) #IOLOG.debug('writers =\n%s', pformat(self.poller.writers)) for side, func in self.poller.poll(timeout): self._call(side.stream, func) + if timer_to is not None: + self.timers.expire() def _broker_exit(self): """ @@ -2982,7 +3512,7 @@ class Broker(object): to shut down gracefully, then discard the :class:`Poller`. """ for _, (side, _) in self.poller.readers + self.poller.writers: - LOG.debug('_broker_main() force disconnecting %r', side) + LOG.debug('%r: force disconnecting %r', self, side) side.stream.on_disconnect(self) self.poller.close() @@ -2997,15 +3527,15 @@ class Broker(object): for _, (side, _) in self.poller.readers + self.poller.writers: self._call(side.stream, side.stream.on_shutdown) - deadline = time.time() + self.shutdown_timeout - while self.keep_alive() and time.time() < deadline: - self._loop_once(max(0, deadline - time.time())) + deadline = now() + self.shutdown_timeout + while self.keep_alive() and now() < deadline: + self._loop_once(max(0, deadline - now())) if self.keep_alive(): - LOG.error('%r: some streams did not close gracefully. ' - 'The most likely cause for this is one or ' - 'more child processes still connected to ' - 'our stdout/stderr pipes.', self) + LOG.error('%r: pending work still existed %d seconds after ' + 'shutdown began. This may be due to a timer that is yet ' + 'to expire, or a child connection that did not fully ' + 'shut down.', self, self.shutdown_timeout) def _do_broker_main(self): """ @@ -3013,7 +3543,7 @@ class Broker(object): :meth:`shutdown` is called. """ # For Python 2.4, no way to retrieve ident except on thread. - self._waker.broker_ident = thread.get_ident() + self._waker.protocol.broker_ident = thread.get_ident() try: while self._alive: self._loop_once() @@ -3021,22 +3551,28 @@ class Broker(object): fire(self, 'shutdown') self._broker_shutdown() except Exception: - LOG.exception('_broker_main() crashed') + e = sys.exc_info()[1] + LOG.exception('broker crashed') + syslog.syslog(syslog.LOG_ERR, 'broker crashed: %s' % (e,)) + syslog.closelog() # prevent test 'fd leak'. self._alive = False # Ensure _alive is consistent on crash. self._exitted = True self._broker_exit() def _broker_main(self): - _profile_hook('mitogen.broker', self._do_broker_main) - fire(self, 'exit') + try: + _profile_hook('mitogen.broker', self._do_broker_main) + finally: + # 'finally' to ensure _on_broker_exit() can always SIGTERM. + fire(self, 'exit') def shutdown(self): """ Request broker gracefully disconnect streams and stop. Safe to call from any thread. """ - _v and LOG.debug('%r.shutdown()', self) + _v and LOG.debug('%r: shutting down', self) def _shutdown(): self._alive = False if self._alive and not self._exitted: @@ -3050,7 +3586,7 @@ class Broker(object): self._thread.join() def __repr__(self): - return 'Broker(%#x)' % (id(self),) + return 'Broker(%04x)' % (id(self) & 0xffff,) class Dispatcher(object): @@ -3064,13 +3600,31 @@ class Dispatcher(object): mode, any exception that occurs is recorded, and causes all subsequent calls with the same `chain_id` to fail with the same exception. """ + _service_recv = None + + def __repr__(self): + return 'Dispatcher' + def __init__(self, econtext): self.econtext = econtext #: Chain ID -> CallError if prior call failed. self._error_by_chain_id = {} - self.recv = Receiver(router=econtext.router, - handle=CALL_FUNCTION, - policy=has_parent_authority) + self.recv = Receiver( + router=econtext.router, + handle=CALL_FUNCTION, + policy=has_parent_authority, + ) + #: The :data:`CALL_SERVICE` :class:`Receiver` that will eventually be + #: reused by :class:`mitogen.service.Pool`, should it ever be loaded. + #: This is necessary for race-free reception of all service requests + #: delivered regardless of whether the stub or real service pool are + #: loaded. See #547 for related sorrows. + Dispatcher._service_recv = Receiver( + router=econtext.router, + handle=CALL_SERVICE, + policy=has_parent_authority, + ) + self._service_recv.notify = self._on_call_service listen(econtext.broker, 'shutdown', self.recv.close) @classmethod @@ -3080,7 +3634,7 @@ class Dispatcher(object): def _parse_request(self, msg): data = msg.unpickle(throw=False) - _v and LOG.debug('_dispatch_one(%r)', data) + _v and LOG.debug('%r: dispatching %r', self, data) chain_id, modname, klass, func, args, kwargs = data obj = import_module(modname) @@ -3111,10 +3665,47 @@ class Dispatcher(object): self._error_by_chain_id[chain_id] = e return chain_id, e + def _on_call_service(self, recv): + """ + Notifier for the :data:`CALL_SERVICE` receiver. This is called on the + :class:`Broker` thread for any service messages arriving at this + context, for as long as no real service pool implementation is loaded. + + In order to safely bootstrap the service pool implementation a sentinel + message is enqueued on the :data:`CALL_FUNCTION` receiver in order to + wake the main thread, where the importer can run without any + possibility of suffering deadlock due to concurrent uses of the + importer. + + Should the main thread be blocked indefinitely, preventing the import + from ever running, if it is blocked waiting on a service call, then it + means :mod:`mitogen.service` has already been imported and + :func:`mitogen.service.get_or_create_pool` has already run, meaning the + service pool is already active and the duplicate initialization was not + needed anyway. + + #547: This trickery is needed to avoid the alternate option of spinning + a temporary thread to import the service pool, which could deadlock if + a custom import hook executing on the main thread (under the importer + lock) would block waiting for some data that was in turn received by a + service. Main thread import lock can't be released until service is + running, service cannot satisfy request until import lock is released. + """ + self.recv._on_receive(Message(handle=STUB_CALL_SERVICE)) + + def _init_service_pool(self): + import mitogen.service + mitogen.service.get_or_create_pool(router=self.econtext.router) + def _dispatch_calls(self): for msg in self.recv: + if msg.handle == STUB_CALL_SERVICE: + if msg.src_id == mitogen.context_id: + self._init_service_pool() + continue + chain_id, ret = self._dispatch_one(msg) - _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) + _v and LOG.debug('%r: %r -> %r', self, msg, ret) if msg.reply_to: msg.reply(ret) elif isinstance(ret, CallError) and chain_id is None: @@ -3131,30 +3722,36 @@ class ExternalContext(object): """ External context implementation. + This class contains the main program implementation for new children. It is + responsible for setting up everything about the process environment, import + hooks, standard IO redirection, logging, configuring a :class:`Router` and + :class:`Broker`, and finally arranging for :class:`Dispatcher` to take over + the main thread after initialization is complete. + .. attribute:: broker + The :class:`mitogen.core.Broker` instance. .. attribute:: context + The :class:`mitogen.core.Context` instance. .. attribute:: channel + The :class:`mitogen.core.Channel` over which :data:`CALL_FUNCTION` requests are received. - .. attribute:: stdout_log - The :class:`mitogen.core.IoLogger` connected to ``stdout``. - .. attribute:: importer + The :class:`mitogen.core.Importer` instance. .. attribute:: stdout_log - The :class:`IoLogger` connected to ``stdout``. + + The :class:`IoLogger` connected to :data:`sys.stdout`. .. attribute:: stderr_log - The :class:`IoLogger` connected to ``stderr``. - .. method:: _dispatch_calls - Implementation for the main thread in every child context. + The :class:`IoLogger` connected to :data:`sys.stderr`. """ detached = False @@ -3165,37 +3762,9 @@ class ExternalContext(object): if not self.config['profiling']: os.kill(os.getpid(), signal.SIGTERM) - #: On Python >3.4, the global importer lock has been sharded into a - #: per-module lock, meaning there is no guarantee the import statement in - #: service_stub_main will be truly complete before a second thread - #: attempting the same import will see a partially initialized module. - #: Sigh. Therefore serialize execution of the stub itself. - service_stub_lock = threading.Lock() - - def _service_stub_main(self, msg): - self.service_stub_lock.acquire() - try: - import mitogen.service - pool = mitogen.service.get_or_create_pool(router=self.router) - pool._receiver._on_receive(msg) - finally: - self.service_stub_lock.release() - - def _on_call_service_msg(self, msg): - """ - Stub service handler. Start a thread to import the mitogen.service - implementation from, and deliver the message to the newly constructed - pool. This must be done as CALL_SERVICE for e.g. PushFileService may - race with a CALL_FUNCTION blocking the main thread waiting for a result - from that service. - """ - if not msg.is_dead: - th = threading.Thread(target=self._service_stub_main, args=(msg,)) - th.start() - def _on_shutdown_msg(self, msg): - _v and LOG.debug('_on_shutdown_msg(%r)', msg) if not msg.is_dead: + _v and LOG.debug('shutdown request from context %d', msg.src_id) self.broker.shutdown() def _on_parent_disconnect(self): @@ -3204,7 +3773,7 @@ class ExternalContext(object): mitogen.parent_id = None LOG.info('Detachment complete') else: - _v and LOG.debug('%r: parent stream is gone, dying.', self) + _v and LOG.debug('parent stream is gone, dying.') self.broker.shutdown() def detach(self): @@ -3215,7 +3784,7 @@ class ExternalContext(object): self.parent.send_await(Message(handle=DETACHING)) LOG.info('Detaching from %r; parent is %s', stream, self.parent) for x in range(20): - pending = self.broker.defer_sync(lambda: stream.pending_bytes()) + pending = self.broker.defer_sync(stream.protocol.pending_bytes) if not pending: break time.sleep(0.05) @@ -3230,17 +3799,12 @@ class ExternalContext(object): self.broker = Broker(activate_compat=False) self.router = Router(self.broker) self.router.debug = self.config.get('debug', False) - self.router.undirectional = self.config['unidirectional'] + self.router.unidirectional = self.config['unidirectional'] self.router.add_handler( fn=self._on_shutdown_msg, handle=SHUTDOWN, policy=has_parent_authority, ) - self.router.add_handler( - fn=self._on_call_service_msg, - handle=CALL_SERVICE, - policy=has_parent_authority, - ) self.master = Context(self.router, 0, 'master') parent_id = self.config['parent_ids'][0] if parent_id == 0: @@ -3249,17 +3813,23 @@ class ExternalContext(object): self.parent = Context(self.router, parent_id, 'parent') in_fd = self.config.get('in_fd', 100) - out_fd = self.config.get('out_fd', 1) - self.stream = Stream(self.router, parent_id) + in_fp = os.fdopen(os.dup(in_fd), 'rb', 0) + os.close(in_fd) + + out_fp = os.fdopen(os.dup(self.config.get('out_fd', 1)), 'wb', 0) + self.stream = MitogenProtocol.build_stream( + self.router, + parent_id, + local_id=self.config['context_id'], + parent_ids=self.config['parent_ids'] + ) + self.stream.accept(in_fp, out_fp) self.stream.name = 'parent' - self.stream.accept(in_fd, out_fd) self.stream.receive_side.keep_alive = False listen(self.stream, 'disconnect', self._on_parent_disconnect) listen(self.broker, 'exit', self._on_broker_exit) - os.close(in_fd) - def _reap_first_stage(self): try: os.wait() # Reap first stage. @@ -3327,31 +3897,33 @@ class ExternalContext(object): def _nullify_stdio(self): """ - Open /dev/null to replace stdin, and stdout/stderr temporarily. In case - of odd startup, assume we may be allocated a standard handle. + Open /dev/null to replace stdio temporarily. In case of odd startup, + assume we may be allocated a standard handle. """ - fd = os.open('/dev/null', os.O_RDWR) - try: - for stdfd in (0, 1, 2): - if fd != stdfd: - os.dup2(fd, stdfd) - finally: - if fd not in (0, 1, 2): + for stdfd, mode in ((0, os.O_RDONLY), (1, os.O_RDWR), (2, os.O_RDWR)): + fd = os.open('/dev/null', mode) + if fd != stdfd: + os.dup2(fd, stdfd) os.close(fd) - def _setup_stdio(self): - # #481: when stderr is a TTY due to being started via - # tty_create_child()/hybrid_tty_create_child(), and some privilege - # escalation tool like prehistoric versions of sudo exec this process - # over the top of itself, there is nothing left to keep the slave PTY - # open after we replace our stdio. Therefore if stderr is a TTY, keep - # around a permanent dup() to avoid receiving SIGHUP. + def _preserve_tty_fp(self): + """ + #481: when stderr is a TTY due to being started via tty_create_child() + or hybrid_tty_create_child(), and some privilege escalation tool like + prehistoric versions of sudo exec this process over the top of itself, + there is nothing left to keep the slave PTY open after we replace our + stdio. Therefore if stderr is a TTY, keep around a permanent dup() to + avoid receiving SIGHUP. + """ try: if os.isatty(2): - self.reserve_tty_fd = os.dup(2) - set_cloexec(self.reserve_tty_fd) + self.reserve_tty_fp = os.fdopen(os.dup(2), 'r+b', 0) + set_cloexec(self.reserve_tty_fp.fileno()) except OSError: pass + + def _setup_stdio(self): + self._preserve_tty_fp() # When sys.stdout was opened by the runtime, overwriting it will not # close FD 1. However when forking from a child that previously used # fdopen(), overwriting it /will/ close FD 1. So we must swallow the @@ -3364,8 +3936,12 @@ class ExternalContext(object): sys.stdout.close() self._nullify_stdio() - self.stdout_log = IoLogger(self.broker, 'stdout', 1) - self.stderr_log = IoLogger(self.broker, 'stderr', 2) + self.loggers = [] + for name, fd in (('stdout', 1), ('stderr', 2)): + log = IoLoggerProtocol.build_stream(name, fd) + self.broker.start_receive(log) + self.loggers.append(log) + # Reopen with line buffering. sys.stdout = os.fdopen(1, 'w', 1) @@ -3385,18 +3961,23 @@ class ExternalContext(object): self.dispatcher = Dispatcher(self) self.router.register(self.parent, self.stream) self.router._setup_logging() - self.log_handler.uncork() - sys.executable = os.environ.pop('ARGV0', sys.executable) - _v and LOG.debug('Connected to context %s; my ID is %r', - self.parent, mitogen.context_id) + _v and LOG.debug('Python version is %s', sys.version) + _v and LOG.debug('Parent is context %r (%s); my ID is %r', + self.parent.context_id, self.parent.name, + mitogen.context_id) _v and LOG.debug('pid:%r ppid:%r uid:%r/%r, gid:%r/%r host:%r', os.getpid(), os.getppid(), os.geteuid(), os.getuid(), os.getegid(), os.getgid(), socket.gethostname()) + + sys.executable = os.environ.pop('ARGV0', sys.executable) _v and LOG.debug('Recovered sys.executable: %r', sys.executable) + if self.config.get('send_ec2', True): + self.stream.transmit_side.write(b('MITO002\n')) self.broker._py24_25_compat() + self.log_handler.uncork() self.dispatcher.run() _v and LOG.debug('ExternalContext.main() normal exit') except KeyboardInterrupt: diff --git a/mitogen/debug.py b/mitogen/debug.py index 3d13347f..dbab550e 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -230,7 +230,7 @@ class ContextDebugger(object): def _handle_debug_msg(self, msg): try: method, args, kwargs = msg.unpickle() - msg.reply(getattr(cls, method)(*args, **kwargs)) + msg.reply(getattr(self, method)(*args, **kwargs)) except Exception: e = sys.exc_info()[1] msg.reply(mitogen.core.CallError(e)) diff --git a/mitogen/doas.py b/mitogen/doas.py index 1b687fb2..5b212b9b 100644 --- a/mitogen/doas.py +++ b/mitogen/doas.py @@ -29,85 +29,114 @@ # !mitogen: minify_safe import logging +import re import mitogen.core import mitogen.parent -from mitogen.core import b LOG = logging.getLogger(__name__) +password_incorrect_msg = 'doas password is incorrect' +password_required_msg = 'doas password is required' + class PasswordError(mitogen.core.StreamError): pass -class Stream(mitogen.parent.Stream): - create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) - child_is_immediate_subprocess = False - - username = 'root' +class Options(mitogen.parent.Options): + username = u'root' password = None doas_path = 'doas' - password_prompt = b('Password:') + password_prompt = u'Password:' incorrect_prompts = ( - b('doas: authentication failed'), + u'doas: authentication failed', # slicer69/doas + u'doas: Authorization failed', # openbsd/src ) - def construct(self, username=None, password=None, doas_path=None, - password_prompt=None, incorrect_prompts=None, **kwargs): - super(Stream, self).construct(**kwargs) + def __init__(self, username=None, password=None, doas_path=None, + password_prompt=None, incorrect_prompts=None, **kwargs): + super(Options, self).__init__(**kwargs) if username is not None: - self.username = username + self.username = mitogen.core.to_text(username) if password is not None: - self.password = password + self.password = mitogen.core.to_text(password) if doas_path is not None: self.doas_path = doas_path if password_prompt is not None: - self.password_prompt = password_prompt.lower() + self.password_prompt = mitogen.core.to_text(password_prompt) if incorrect_prompts is not None: - self.incorrect_prompts = map(str.lower, incorrect_prompts) + self.incorrect_prompts = [ + mitogen.core.to_text(p) + for p in incorrect_prompts + ] + + +class BootstrapProtocol(mitogen.parent.RegexProtocol): + password_sent = False + + def setup_patterns(self, conn): + prompt_pattern = re.compile( + re.escape(conn.options.password_prompt).encode('utf-8'), + re.I + ) + incorrect_prompt_pattern = re.compile( + u'|'.join( + re.escape(s) + for s in conn.options.incorrect_prompts + ).encode('utf-8'), + re.I + ) + + self.PATTERNS = [ + (incorrect_prompt_pattern, type(self)._on_incorrect_password), + ] + self.PARTIAL_PATTERNS = [ + (prompt_pattern, type(self)._on_password_prompt), + ] + + def _on_incorrect_password(self, line, match): + if self.password_sent: + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) + + def _on_password_prompt(self, line, match): + if self.stream.conn.options.password is None: + self.stream.conn._fail_connection( + PasswordError(password_required_msg) + ) + return + + if self.password_sent: + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) + return + + LOG.debug('sending password') + self.stream.transmit_side.write( + (self.stream.conn.options.password + '\n').encode('utf-8') + ) + self.password_sent = True + + +class Connection(mitogen.parent.Connection): + options_class = Options + diag_protocol_class = BootstrapProtocol + + create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) + child_is_immediate_subprocess = False def _get_name(self): - return u'doas.' + mitogen.core.to_text(self.username) + return u'doas.' + self.options.username + + def stderr_stream_factory(self): + stream = super(Connection, self).stderr_stream_factory() + stream.protocol.setup_patterns(self) + return stream def get_boot_command(self): - bits = [self.doas_path, '-u', self.username, '--'] - bits = bits + super(Stream, self).get_boot_command() - LOG.debug('doas command line: %r', bits) - return bits - - password_incorrect_msg = 'doas password is incorrect' - password_required_msg = 'doas password is required' - - def _connect_input_loop(self, it): - password_sent = False - for buf in it: - LOG.debug('%r: received %r', self, buf) - if buf.endswith(self.EC0_MARKER): - self._ec0_received() - return - if any(s in buf.lower() for s in self.incorrect_prompts): - if password_sent: - raise PasswordError(self.password_incorrect_msg) - elif self.password_prompt in buf.lower(): - if self.password is None: - raise PasswordError(self.password_required_msg) - if password_sent: - raise PasswordError(self.password_incorrect_msg) - LOG.debug('sending password') - self.diag_stream.transmit_side.write( - mitogen.core.to_text(self.password + '\n').encode('utf-8') - ) - password_sent = True - raise mitogen.core.StreamError('bootstrap failed') - - def _connect_bootstrap(self): - it = mitogen.parent.iter_read( - fds=[self.receive_side.fd, self.diag_stream.receive_side.fd], - deadline=self.connect_deadline, - ) - try: - self._connect_input_loop(it) - finally: - it.close() + bits = [self.options.doas_path, '-u', self.options.username, '--'] + return bits + super(Connection, self).get_boot_command() diff --git a/mitogen/docker.py b/mitogen/docker.py index 0c0d40e7..48848c89 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -37,45 +37,47 @@ import mitogen.parent LOG = logging.getLogger(__name__) -class Stream(mitogen.parent.Stream): - child_is_immediate_subprocess = False - +class Options(mitogen.parent.Options): container = None image = None username = None - docker_path = 'docker' - - # TODO: better way of capturing errors such as "No such container." - create_child_args = { - 'merge_stdio': True - } + docker_path = u'docker' - def construct(self, container=None, image=None, - docker_path=None, username=None, - **kwargs): + def __init__(self, container=None, image=None, docker_path=None, + username=None, **kwargs): + super(Options, self).__init__(**kwargs) assert container or image - super(Stream, self).construct(**kwargs) if container: - self.container = container + self.container = mitogen.core.to_text(container) if image: - self.image = image + self.image = mitogen.core.to_text(image) if docker_path: - self.docker_path = docker_path + self.docker_path = mitogen.core.to_text(docker_path) if username: - self.username = username + self.username = mitogen.core.to_text(username) + + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = False + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } def _get_name(self): - return u'docker.' + (self.container or self.image) + return u'docker.' + (self.options.container or self.options.image) def get_boot_command(self): args = ['--interactive'] - if self.username: - args += ['--user=' + self.username] + if self.options.username: + args += ['--user=' + self.options.username] - bits = [self.docker_path] - if self.container: - bits += ['exec'] + args + [self.container] - elif self.image: - bits += ['run'] + args + ['--rm', self.image] + bits = [self.options.docker_path] + if self.options.container: + bits += ['exec'] + args + [self.options.container] + elif self.options.image: + bits += ['run'] + args + ['--rm', self.options.image] - return bits + super(Stream, self).get_boot_command() + return bits + super(Connection, self).get_boot_command() diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index d39a710d..e62cf84a 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -117,14 +117,12 @@ SSH_GETOPTS = ( _mitogen = None -class IoPump(mitogen.core.BasicStream): +class IoPump(mitogen.core.Protocol): _output_buf = '' _closed = False - def __init__(self, broker, stdin_fd, stdout_fd): + def __init__(self, broker): self._broker = broker - self.receive_side = mitogen.core.Side(self, stdout_fd) - self.transmit_side = mitogen.core.Side(self, stdin_fd) def write(self, s): self._output_buf += s @@ -134,13 +132,13 @@ class IoPump(mitogen.core.BasicStream): self._closed = True # If local process hasn't exitted yet, ensure its write buffer is # drained before lazily triggering disconnect in on_transmit. - if self.transmit_side.fd is not None: + if self.transmit_side.fp.fileno() is not None: self._broker._start_transmit(self) - def on_shutdown(self, broker): + def on_shutdown(self, stream, broker): self.close() - def on_transmit(self, broker): + def on_transmit(self, stream, broker): written = self.transmit_side.write(self._output_buf) IOLOG.debug('%r.on_transmit() -> len %r', self, written) if written is None: @@ -153,8 +151,8 @@ class IoPump(mitogen.core.BasicStream): if self._closed: self.on_disconnect(broker) - def on_receive(self, broker): - s = self.receive_side.read() + def on_receive(self, stream, broker): + s = stream.receive_side.read() IOLOG.debug('%r.on_receive() -> len %r', self, len(s)) if s: mitogen.core.fire(self, 'receive', s) @@ -163,8 +161,8 @@ class IoPump(mitogen.core.BasicStream): def __repr__(self): return 'IoPump(%r, %r)' % ( - self.receive_side.fd, - self.transmit_side.fd, + self.receive_side.fp.fileno(), + self.transmit_side.fp.fileno(), ) @@ -173,14 +171,15 @@ class Process(object): Manages the lifetime and pipe connections of the SSH command running in the slave. """ - def __init__(self, router, stdin_fd, stdout_fd, proc=None): + def __init__(self, router, stdin, stdout, proc=None): self.router = router - self.stdin_fd = stdin_fd - self.stdout_fd = stdout_fd + self.stdin = stdin + self.stdout = stdout self.proc = proc self.control_handle = router.add_handler(self._on_control) self.stdin_handle = router.add_handler(self._on_stdin) - self.pump = IoPump(router.broker, stdin_fd, stdout_fd) + self.pump = IoPump.build_stream(router.broker) + self.pump.accept(stdin, stdout) self.stdin = None self.control = None self.wake_event = threading.Event() @@ -193,7 +192,7 @@ class Process(object): pmon.add(proc.pid, self._on_proc_exit) def __repr__(self): - return 'Process(%r, %r)' % (self.stdin_fd, self.stdout_fd) + return 'Process(%r, %r)' % (self.stdin, self.stdout) def _on_proc_exit(self, status): LOG.debug('%r._on_proc_exit(%r)', self, status) @@ -202,12 +201,12 @@ class Process(object): def _on_stdin(self, msg): if msg.is_dead: IOLOG.debug('%r._on_stdin() -> %r', self, data) - self.pump.close() + self.pump.protocol.close() return data = msg.unpickle() IOLOG.debug('%r._on_stdin() -> len %d', self, len(data)) - self.pump.write(data) + self.pump.protocol.write(data) def _on_control(self, msg): if not msg.is_dead: @@ -279,13 +278,7 @@ def _start_slave(src_id, cmdline, router): stdout=subprocess.PIPE, ) - process = Process( - router, - proc.stdin.fileno(), - proc.stdout.fileno(), - proc, - ) - + process = Process(router, proc.stdin, proc.stdout, proc) return process.control_handle, process.stdin_handle @@ -361,7 +354,9 @@ def _fakessh_main(dest_context_id, econtext): LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r', control_handle, stdin_handle) - process = Process(econtext.router, 1, 0) + process = Process(econtext.router, + stdin=os.fdopen(1, 'w+b', 0), + stdout=os.fdopen(0, 'r+b', 0)) process.start_master( stdin=mitogen.core.Sender(dest, stdin_handle), control=mitogen.core.Sender(dest, control_handle), @@ -427,7 +422,7 @@ def run(dest, router, args, deadline=None, econtext=None): stream = mitogen.core.Stream(router, context_id) stream.name = u'fakessh' - stream.accept(sock1.fileno(), sock1.fileno()) + stream.accept(sock1, sock1) router.register(fakessh, stream) # Held in socket buffer until process is booted. diff --git a/mitogen/fork.py b/mitogen/fork.py index d6685d70..4172e96f 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -28,6 +28,7 @@ # !mitogen: minify_safe +import errno import logging import os import random @@ -37,9 +38,10 @@ import traceback import mitogen.core import mitogen.parent +from mitogen.core import b -LOG = logging.getLogger('mitogen') +LOG = logging.getLogger(__name__) # Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up # interpreter state. So 2.4/2.5 interpreters start .local() contexts for @@ -71,8 +73,8 @@ def reset_logging_framework(): threads in the parent may have been using the logging package at the moment of fork. - It is not possible to solve this problem in general; see - https://github.com/dw/mitogen/issues/150 for a full discussion. + It is not possible to solve this problem in general; see :gh:issue:`150` + for a full discussion. """ logging._lock = threading.RLock() @@ -119,32 +121,53 @@ def handle_child_crash(): os._exit(1) -class Stream(mitogen.parent.Stream): - child_is_immediate_subprocess = True +def _convert_exit_status(status): + """ + Convert a :func:`os.waitpid`-style exit status to a :mod:`subprocess` style + exit status. + """ + if os.WIFEXITED(status): + return os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + elif os.WIFSTOPPED(status): + return -os.WSTOPSIG(status) + +class Process(mitogen.parent.Process): + def poll(self): + try: + pid, status = os.waitpid(self.pid, os.WNOHANG) + except OSError: + e = sys.exc_info()[1] + if e.args[0] == errno.ECHILD: + LOG.warn('%r: waitpid(%r) produced ECHILD', self, self.pid) + return + raise + + if not pid: + return + return _convert_exit_status(status) + + +class Options(mitogen.parent.Options): #: Reference to the importer, if any, recovered from the parent. importer = None #: User-supplied function for cleaning up child process state. on_fork = None - python_version_msg = ( - "The mitogen.fork method is not supported on Python versions " - "prior to 2.6, since those versions made no attempt to repair " - "critical interpreter state following a fork. Please use the " - "local() method instead." - ) - - def construct(self, old_router, max_message_size, on_fork=None, - debug=False, profiling=False, unidirectional=False, - on_start=None): + def __init__(self, old_router, max_message_size, on_fork=None, debug=False, + profiling=False, unidirectional=False, on_start=None, + name=None): if not FORK_SUPPORTED: raise Error(self.python_version_msg) # fork method only supports a tiny subset of options. - super(Stream, self).construct(max_message_size=max_message_size, - debug=debug, profiling=profiling, - unidirectional=False) + super(Options, self).__init__( + max_message_size=max_message_size, debug=debug, + profiling=profiling, unidirectional=unidirectional, name=name, + ) self.on_fork = on_fork self.on_start = on_start @@ -152,17 +175,26 @@ class Stream(mitogen.parent.Stream): if isinstance(responder, mitogen.parent.ModuleForwarder): self.importer = responder.importer + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = True + + python_version_msg = ( + "The mitogen.fork method is not supported on Python versions " + "prior to 2.6, since those versions made no attempt to repair " + "critical interpreter state following a fork. Please use the " + "local() method instead." + ) + name_prefix = u'fork' def start_child(self): parentfp, childfp = mitogen.parent.create_socketpair() - self.pid = os.fork() - if self.pid: + pid = os.fork() + if pid: childfp.close() - # Decouple the socket from the lifetime of the Python socket object. - fd = os.dup(parentfp.fileno()) - parentfp.close() - return self.pid, fd, None + return Process(pid, stdin=parentfp, stdout=parentfp) else: parentfp.close() self._wrap_child_main(childfp) @@ -173,12 +205,24 @@ class Stream(mitogen.parent.Stream): except BaseException: handle_child_crash() + def get_econtext_config(self): + config = super(Connection, self).get_econtext_config() + config['core_src_fd'] = None + config['importer'] = self.options.importer + config['send_ec2'] = False + config['setup_package'] = False + if self.options.on_start: + config['on_start'] = self.options.on_start + return config + def _child_main(self, childfp): on_fork() - if self.on_fork: - self.on_fork() + if self.options.on_fork: + self.options.on_fork() mitogen.core.set_block(childfp.fileno()) + childfp.send(b('MITO002\n')) + # Expected by the ExternalContext.main(). os.dup2(childfp.fileno(), 1) os.dup2(childfp.fileno(), 100) @@ -201,23 +245,12 @@ class Stream(mitogen.parent.Stream): if childfp.fileno() not in (0, 1, 100): childfp.close() - config = self.get_econtext_config() - config['core_src_fd'] = None - config['importer'] = self.importer - config['setup_package'] = False - if self.on_start: - config['on_start'] = self.on_start - try: try: - mitogen.core.ExternalContext(config).main() + mitogen.core.ExternalContext(self.get_econtext_config()).main() except Exception: # TODO: report exception somehow. os._exit(72) finally: # Don't trigger atexit handlers, they were copied from the parent. os._exit(0) - - def _connect_bootstrap(self): - # None required. - pass diff --git a/mitogen/jail.py b/mitogen/jail.py index 6e0ac68b..4da7eb0d 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -28,38 +28,38 @@ # !mitogen: minify_safe -import logging - import mitogen.core import mitogen.parent -LOG = logging.getLogger(__name__) +class Options(mitogen.parent.Options): + container = None + username = None + jexec_path = u'/usr/sbin/jexec' + + def __init__(self, container, jexec_path=None, username=None, **kwargs): + super(Options, self).__init__(**kwargs) + self.container = mitogen.core.to_text(container) + if username: + self.username = mitogen.core.to_text(username) + if jexec_path: + self.jexec_path = jexec_path + +class Connection(mitogen.parent.Connection): + options_class = Options -class Stream(mitogen.parent.Stream): child_is_immediate_subprocess = False create_child_args = { 'merge_stdio': True } - container = None - username = None - jexec_path = '/usr/sbin/jexec' - - def construct(self, container, jexec_path=None, username=None, **kwargs): - super(Stream, self).construct(**kwargs) - self.container = container - self.username = username - if jexec_path: - self.jexec_path = jexec_path - def _get_name(self): - return u'jail.' + self.container + return u'jail.' + self.options.container def get_boot_command(self): - bits = [self.jexec_path] - if self.username: - bits += ['-U', self.username] - bits += [self.container] - return bits + super(Stream, self).get_boot_command() + bits = [self.options.jexec_path] + if self.options.username: + bits += ['-U', self.options.username] + bits += [self.options.container] + return bits + super(Connection, self).get_boot_command() diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py index ef626e1b..374ab747 100644 --- a/mitogen/kubectl.py +++ b/mitogen/kubectl.py @@ -28,38 +28,40 @@ # !mitogen: minify_safe -import logging - import mitogen.core import mitogen.parent -LOG = logging.getLogger(__name__) - - -class Stream(mitogen.parent.Stream): - child_is_immediate_subprocess = True - +class Options(mitogen.parent.Options): pod = None kubectl_path = 'kubectl' kubectl_args = None - # TODO: better way of capturing errors such as "No such container." - create_child_args = { - 'merge_stdio': True - } - - def construct(self, pod, kubectl_path=None, kubectl_args=None, **kwargs): - super(Stream, self).construct(**kwargs) + def __init__(self, pod, kubectl_path=None, kubectl_args=None, **kwargs): + super(Options, self).__init__(**kwargs) assert pod self.pod = pod if kubectl_path: self.kubectl_path = kubectl_path self.kubectl_args = kubectl_args or [] + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = True + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + def _get_name(self): - return u'kubectl.%s%s' % (self.pod, self.kubectl_args) + return u'kubectl.%s%s' % (self.options.pod, self.options.kubectl_args) def get_boot_command(self): - bits = [self.kubectl_path] + self.kubectl_args + ['exec', '-it', self.pod] - return bits + ["--"] + super(Stream, self).get_boot_command() + bits = [ + self.options.kubectl_path + ] + self.options.kubectl_args + [ + 'exec', '-it', self.options.pod + ] + return bits + ["--"] + super(Connection, self).get_boot_command() diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 879d19a1..a86ce5f0 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -28,16 +28,24 @@ # !mitogen: minify_safe -import logging - import mitogen.core import mitogen.parent -LOG = logging.getLogger(__name__) +class Options(mitogen.parent.Options): + container = None + lxc_attach_path = 'lxc-attach' + + def __init__(self, container, lxc_attach_path=None, **kwargs): + super(Options, self).__init__(**kwargs) + self.container = container + if lxc_attach_path: + self.lxc_attach_path = lxc_attach_path + +class Connection(mitogen.parent.Connection): + options_class = Options -class Stream(mitogen.parent.Stream): child_is_immediate_subprocess = False create_child_args = { # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY, @@ -47,29 +55,20 @@ class Stream(mitogen.parent.Stream): 'merge_stdio': True } - container = None - lxc_attach_path = 'lxc-attach' - eof_error_hint = ( 'Note: many versions of LXC do not report program execution failure ' 'meaningfully. Please check the host logs (/var/log) for more ' 'information.' ) - def construct(self, container, lxc_attach_path=None, **kwargs): - super(Stream, self).construct(**kwargs) - self.container = container - if lxc_attach_path: - self.lxc_attach_path = lxc_attach_path - def _get_name(self): - return u'lxc.' + self.container + return u'lxc.' + self.options.container def get_boot_command(self): bits = [ - self.lxc_attach_path, + self.options.lxc_attach_path, '--clear-env', - '--name', self.container, + '--name', self.options.container, '--', ] - return bits + super(Stream, self).get_boot_command() + return bits + super(Connection, self).get_boot_command() diff --git a/mitogen/lxd.py b/mitogen/lxd.py index faea2561..675dddcd 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -28,16 +28,25 @@ # !mitogen: minify_safe -import logging - import mitogen.core import mitogen.parent -LOG = logging.getLogger(__name__) +class Options(mitogen.parent.Options): + container = None + lxc_path = 'lxc' + python_path = 'python' + + def __init__(self, container, lxc_path=None, **kwargs): + super(Options, self).__init__(**kwargs) + self.container = container + if lxc_path: + self.lxc_path = lxc_path + +class Connection(mitogen.parent.Connection): + options_class = Options -class Stream(mitogen.parent.Stream): child_is_immediate_subprocess = False create_child_args = { # If lxc finds any of stdin, stdout, stderr connected to a TTY, to @@ -47,31 +56,21 @@ class Stream(mitogen.parent.Stream): 'merge_stdio': True } - container = None - lxc_path = 'lxc' - python_path = 'python' - eof_error_hint = ( 'Note: many versions of LXC do not report program execution failure ' 'meaningfully. Please check the host logs (/var/log) for more ' 'information.' ) - def construct(self, container, lxc_path=None, **kwargs): - super(Stream, self).construct(**kwargs) - self.container = container - if lxc_path: - self.lxc_path = lxc_path - def _get_name(self): - return u'lxd.' + self.container + return u'lxd.' + self.options.container def get_boot_command(self): bits = [ - self.lxc_path, + self.options.lxc_path, 'exec', '--mode=noninteractive', - self.container, + self.options.container, '--', ] - return bits + super(Stream, self).get_boot_command() + return bits + super(Connection, self).get_boot_command() diff --git a/mitogen/master.py b/mitogen/master.py index 1396f4e1..f9ddf3dd 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -36,6 +36,7 @@ contexts. """ import dis +import errno import imp import inspect import itertools @@ -45,11 +46,15 @@ import pkgutil import re import string import sys -import time import threading import types import zlib +try: + import sysconfig +except ImportError: + sysconfig = None + if not hasattr(pkgutil, 'find_loader'): # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has # been kept intentionally 2.3 compatible so we can reuse it. @@ -85,22 +90,29 @@ RLOG = logging.getLogger('mitogen.ctx') def _stdlib_paths(): - """Return a set of paths from which Python imports the standard library. + """ + Return a set of paths from which Python imports the standard library. """ attr_candidates = [ 'prefix', 'real_prefix', # virtualenv: only set inside a virtual environment. 'base_prefix', # venv: always set, equal to prefix if outside. ] - prefixes = (getattr(sys, a) for a in attr_candidates if hasattr(sys, a)) + prefixes = (getattr(sys, a, None) for a in attr_candidates) version = 'python%s.%s' % sys.version_info[0:2] - return set(os.path.abspath(os.path.join(p, 'lib', version)) - for p in prefixes) + s = set(os.path.abspath(os.path.join(p, 'lib', version)) + for p in prefixes if p is not None) + + # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu + # 18.10, above is insufficient to catch the real directory. + if sysconfig is not None: + s.add(sysconfig.get_config_var('DESTLIB')) + return s def is_stdlib_name(modname): - """Return :data:`True` if `modname` appears to come from the standard - library. + """ + Return :data:`True` if `modname` appears to come from the standard library. """ if imp.is_builtin(modname) != 0: return True @@ -127,7 +139,8 @@ def is_stdlib_path(path): def get_child_modules(path): - """Return the suffixes of submodules directly neated beneath of the package + """ + Return the suffixes of submodules directly neated beneath of the package directory at `path`. :param str path: @@ -142,6 +155,41 @@ def get_child_modules(path): return [to_text(name) for _, name, _ in it] +def _looks_like_script(path): + """ + Return :data:`True` if the (possibly extensionless) file at `path` + resembles a Python script. For now we simply verify the file contains + ASCII text. + """ + try: + fp = open(path, 'rb') + except IOError: + e = sys.exc_info()[1] + if e.args[0] == errno.EISDIR: + return False + raise + + try: + sample = fp.read(512).decode('latin-1') + return not set(sample).difference(string.printable) + finally: + fp.close() + + +def _py_filename(path): + if not path: + return None + + if path[-4:] in ('.pyc', '.pyo'): + path = path.rstrip('co') + + if path.endswith('.py'): + return path + + if os.path.exists(path) and _looks_like_script(path): + return path + + def _get_core_source(): """ Master version of parent.get_core_source(). @@ -254,8 +302,10 @@ class ThreadWatcher(object): @classmethod def _reset(cls): - """If we have forked since the watch dictionaries were initialized, all - that has is garbage, so clear it.""" + """ + If we have forked since the watch dictionaries were initialized, all + that has is garbage, so clear it. + """ if os.getpid() != cls._cls_pid: cls._cls_pid = os.getpid() cls._cls_instances_by_target.clear() @@ -336,18 +386,18 @@ class LogForwarder(object): if msg.is_dead: return - logger = self._cache.get(msg.src_id) - if logger is None: - context = self._router.context_by_id(msg.src_id) - if context is None: - LOG.error('%s: dropping log from unknown context ID %d', - self, msg.src_id) - return + context = self._router.context_by_id(msg.src_id) + if context is None: + LOG.error('%s: dropping log from unknown context %d', + self, msg.src_id) + return - name = '%s.%s' % (RLOG.name, context.name) - self._cache[msg.src_id] = logger = logging.getLogger(name) + name, level_s, s = msg.data.decode('utf-8', 'replace').split('\x00', 2) - name, level_s, s = msg.data.decode('latin1').split('\x00', 2) + logger_name = '%s.[%s]' % (name, context.name) + logger = self._cache.get(logger_name) + if logger is None: + self._cache[logger_name] = logger = logging.getLogger(logger_name) # See logging.Handler.makeRecord() record = logging.LogRecord( @@ -355,7 +405,7 @@ class LogForwarder(object): level=int(level_s), pathname='(unknown file)', lineno=0, - msg=('%s: %s' % (name, s)), + msg=s, args=(), exc_info=None, ) @@ -368,55 +418,40 @@ class LogForwarder(object): return 'LogForwarder(%r)' % (self._router,) -class ModuleFinder(object): +class FinderMethod(object): """ - Given the name of a loaded module, make a best-effort attempt at finding - related modules likely needed by a child context requesting the original - module. + Interface to a method for locating a Python module or package given its + name according to the running Python interpreter. You'd think this was a + simple task, right? Naive young fellow, welcome to the real world. """ - def __init__(self): - #: Import machinery is expensive, keep :py:meth`:get_module_source` - #: results around. - self._found_cache = {} - - #: Avoid repeated dependency scanning, which is expensive. - self._related_cache = {} - def __repr__(self): - return 'ModuleFinder()' + return '%s()' % (type(self).__name__,) - def _looks_like_script(self, path): + def find(self, fullname): """ - Return :data:`True` if the (possibly extensionless) file at `path` - resembles a Python script. For now we simply verify the file contains - ASCII text. - """ - fp = open(path, 'rb') - try: - sample = fp.read(512).decode('latin-1') - return not set(sample).difference(string.printable) - finally: - fp.close() - - def _py_filename(self, path): - if not path: - return None + Accept a canonical module name as would be found in :data:`sys.modules` + and return a `(path, source, is_pkg)` tuple, where: - if path[-4:] in ('.pyc', '.pyo'): - path = path.rstrip('co') + * `path`: Unicode string containing path to source file. + * `source`: Bytestring containing source file's content. + * `is_pkg`: :data:`True` if `fullname` is a package. - if path.endswith('.py'): - return path + :returns: + :data:`None` if not found, or tuple as described above. + """ + raise NotImplementedError() - if os.path.exists(path) and self._looks_like_script(path): - return path - def _get_main_module_defective_python_3x(self, fullname): +class DefectivePython3xMainMethod(FinderMethod): + """ + Recent versions of Python 3.x introduced an incomplete notion of + importer specs, and in doing so created permanent asymmetry in the + :mod:`pkgutil` interface handling for the :mod:`__main__` module. Therefore + we must handle :mod:`__main__` specially. + """ + def find(self, fullname): """ - Recent versions of Python 3.x introduced an incomplete notion of - importer specs, and in doing so created permanent asymmetry in the - :mod:`pkgutil` interface handling for the `__main__` module. Therefore - we must handle `__main__` specially. + Find :mod:`__main__` using its :data:`__file__` attribute. """ if fullname != '__main__': return None @@ -426,7 +461,7 @@ class ModuleFinder(object): return None path = getattr(mod, '__file__', None) - if not (os.path.exists(path) and self._looks_like_script(path)): + if not (path is not None and os.path.exists(path) and _looks_like_script(path)): return None fp = open(path, 'rb') @@ -437,10 +472,15 @@ class ModuleFinder(object): return path, source, False - def _get_module_via_pkgutil(self, fullname): + +class PkgutilMethod(FinderMethod): + """ + Attempt to fetch source code via pkgutil. In an ideal world, this would + be the only required implementation of get_module(). + """ + def find(self, fullname): """ - Attempt to fetch source code via pkgutil. In an ideal world, this would - be the only required implementation of get_module(). + Find `fullname` using :func:`pkgutil.find_loader`. """ try: # Pre-'import spec' this returned None, in Python3.6 it raises @@ -458,7 +498,7 @@ class ModuleFinder(object): return try: - path = self._py_filename(loader.get_filename(fullname)) + path = _py_filename(loader.get_filename(fullname)) source = loader.get_source(fullname) is_pkg = loader.is_package(fullname) except (AttributeError, ImportError): @@ -484,22 +524,36 @@ class ModuleFinder(object): return path, source, is_pkg - def _get_module_via_sys_modules(self, fullname): + +class SysModulesMethod(FinderMethod): + """ + Attempt to fetch source code via :data:`sys.modules`. This was originally + specifically to support :mod:`__main__`, but it may catch a few more cases. + """ + def find(self, fullname): """ - Attempt to fetch source code via sys.modules. This is specifically to - support __main__, but it may catch a few more cases. + Find `fullname` using its :data:`__file__` attribute. """ module = sys.modules.get(fullname) - LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if not isinstance(module, types.ModuleType): - LOG.debug('sys.modules[%r] absent or not a regular module', - fullname) + LOG.debug('%r: sys.modules[%r] absent or not a regular module', + self, fullname) + return + + LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) + alleged_name = getattr(module, '__name__', None) + if alleged_name != fullname: + LOG.debug('sys.modules[%r].__name__ is incorrect, assuming ' + 'this is a hacky module alias and ignoring it. ' + 'Got %r, module object: %r', + fullname, alleged_name, module) return - path = self._py_filename(getattr(module, '__file__', '')) + path = _py_filename(getattr(module, '__file__', '')) if not path: return + LOG.debug('%r: sys.modules[%r]: found %s', self, fullname, path) is_pkg = hasattr(module, '__path__') try: source = inspect.getsource(module) @@ -517,44 +571,147 @@ class ModuleFinder(object): return path, source, is_pkg - def _get_module_via_parent_enumeration(self, fullname): + +class ParentEnumerationMethod(FinderMethod): + """ + Attempt to fetch source code by examining the module's (hopefully less + insane) parent package, and if no insane parents exist, simply use + :mod:`sys.path` to search for it from scratch on the filesystem using the + normal Python lookup mechanism. + + This is required for older versions of :mod:`ansible.compat.six`, + :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and + its submodule :mod:`ansible.module_utils.distro._distro`. + + When some package dynamically replaces itself in :data:`sys.modules`, but + only conditionally according to some program logic, it is possible that + children may attempt to load modules and subpackages from it that can no + longer be resolved by examining a (corrupted) parent. + + For cases like :mod:`ansible.module_utils.distro`, this must handle cases + where a package transmuted itself into a totally unrelated module during + import and vice versa, where :data:`sys.modules` is replaced with junk that + makes it impossible to discover the loaded module using the in-memory + module object or any parent package's :data:`__path__`, since they have all + been overwritten. Some men just want to watch the world burn. + """ + def _find_sane_parent(self, fullname): """ - Attempt to fetch source code by examining the module's (hopefully less - insane) parent package. Required for older versions of - ansible.compat.six and plumbum.colors. + Iteratively search :data:`sys.modules` for the least indirect parent of + `fullname` that is loaded and contains a :data:`__path__` attribute. + + :return: + `(parent_name, path, modpath)` tuple, where: + + * `modname`: canonical name of the found package, or the empty + string if none is found. + * `search_path`: :data:`__path__` attribute of the least + indirect parent found, or :data:`None` if no indirect parent + was found. + * `modpath`: list of module name components leading from `path` + to the target module. """ - if fullname not in sys.modules: - # Don't attempt this unless a module really exists in sys.modules, - # else we could return junk. - return + path = None + modpath = [] + while True: + pkgname, _, modname = str_rpartition(to_text(fullname), u'.') + modpath.insert(0, modname) + if not pkgname: + return [], None, modpath + + pkg = sys.modules.get(pkgname) + path = getattr(pkg, '__path__', None) + if pkg and path: + return pkgname.split('.'), path, modpath + + LOG.debug('%r: %r lacks __path__ attribute', self, pkgname) + fullname = pkgname + + def _found_package(self, fullname, path): + path = os.path.join(path, '__init__.py') + LOG.debug('%r: %r is PKG_DIRECTORY: %r', self, fullname, path) + return self._found_module( + fullname=fullname, + path=path, + fp=open(path, 'rb'), + is_pkg=True, + ) - pkgname, _, modname = str_rpartition(to_text(fullname), u'.') - pkg = sys.modules.get(pkgname) - if pkg is None or not hasattr(pkg, '__file__'): - return + def _found_module(self, fullname, path, fp, is_pkg=False): + try: + path = _py_filename(path) + if not path: + return + + source = fp.read() + finally: + if fp: + fp.close() + + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + return path, source, is_pkg - pkg_path = os.path.dirname(pkg.__file__) + def _find_one_component(self, modname, search_path): try: - fp, path, ext = imp.find_module(modname, [pkg_path]) - try: - path = self._py_filename(path) - if not path: - fp.close() - return + #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + return imp.find_module(modname, search_path) + except ImportError: + e = sys.exc_info()[1] + LOG.debug('%r: imp.find_module(%r, %r) -> %s', + self, modname, [search_path], e) + return None - source = fp.read() - finally: + def find(self, fullname): + """ + See implementation for a description of how this works. + """ + #if fullname not in sys.modules: + # Don't attempt this unless a module really exists in sys.modules, + # else we could return junk. + #return + + fullname = to_text(fullname) + modname, search_path, modpath = self._find_sane_parent(fullname) + while True: + tup = self._find_one_component(modpath.pop(0), search_path) + if tup is None: + return None + + fp, path, (suffix, _, kind) = tup + if modpath: + # Still more components to descent. Result must be a package if fp: fp.close() + if kind != imp.PKG_DIRECTORY: + LOG.debug('%r: %r appears to be child of non-package %r', + self, fullname, path) + return None + search_path = [path] + elif kind == imp.PKG_DIRECTORY: + return self._found_package(fullname, path) + else: + return self._found_module(fullname, path, fp) - if isinstance(source, mitogen.core.UnicodeType): - # get_source() returns "string" according to PEP-302, which was - # reinterpreted for Python 3 to mean a Unicode string. - source = source.encode('utf-8') - return path, source, False - except ImportError: - e = sys.exc_info()[1] - LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e) + +class ModuleFinder(object): + """ + Given the name of a loaded module, make a best-effort attempt at finding + related modules likely needed by a child context requesting the original + module. + """ + def __init__(self): + #: Import machinery is expensive, keep :py:meth`:get_module_source` + #: results around. + self._found_cache = {} + + #: Avoid repeated dependency scanning, which is expensive. + self._related_cache = {} + + def __repr__(self): + return 'ModuleFinder()' def add_source_override(self, fullname, path, source, is_pkg): """ @@ -576,14 +733,15 @@ class ModuleFinder(object): self._found_cache[fullname] = (path, source, is_pkg) get_module_methods = [ - _get_main_module_defective_python_3x, - _get_module_via_pkgutil, - _get_module_via_sys_modules, - _get_module_via_parent_enumeration, + DefectivePython3xMainMethod(), + PkgutilMethod(), + SysModulesMethod(), + ParentEnumerationMethod(), ] def get_module_source(self, fullname): - """Given the name of a loaded module `fullname`, attempt to find its + """ + Given the name of a loaded module `fullname`, attempt to find its source code. :returns: @@ -595,7 +753,7 @@ class ModuleFinder(object): return tup for method in self.get_module_methods: - tup = method(self, fullname) + tup = method.find(fullname) if tup: #LOG.debug('%r returned %r', method, tup) break @@ -607,9 +765,10 @@ class ModuleFinder(object): return tup def resolve_relpath(self, fullname, level): - """Given an ImportFrom AST node, guess the prefix that should be tacked - on to an alias name to produce a canonical name. `fullname` is the name - of the module in which the ImportFrom appears. + """ + Given an ImportFrom AST node, guess the prefix that should be tacked on + to an alias name to produce a canonical name. `fullname` is the name of + the module in which the ImportFrom appears. """ mod = sys.modules.get(fullname, None) if hasattr(mod, '__path__'): @@ -638,7 +797,7 @@ class ModuleFinder(object): The list is determined by retrieving the source code of `fullname`, compiling it, and examining all IMPORT_NAME ops. - :param fullname: Fully qualified name of an _already imported_ module + :param fullname: Fully qualified name of an *already imported* module for which source code can be retrieved :type fullname: str """ @@ -686,7 +845,7 @@ class ModuleFinder(object): This method is like :py:meth:`find_related_imports`, but also recursively searches any modules which are imported by `fullname`. - :param fullname: Fully qualified name of an _already imported_ module + :param fullname: Fully qualified name of an *already imported* module for which source code can be retrieved :type fullname: str """ @@ -705,6 +864,7 @@ class ModuleFinder(object): class ModuleResponder(object): def __init__(self, router): + self._log = logging.getLogger('mitogen.responder') self._router = router self._finder = ModuleFinder() self._cache = {} # fullname -> pickled @@ -733,11 +893,11 @@ class ModuleResponder(object): ) def __repr__(self): - return 'ModuleResponder(%r)' % (self._router,) + return 'ModuleResponder' def add_source_override(self, fullname, path, source, is_pkg): """ - See :meth:`ModuleFinder.add_source_override. + See :meth:`ModuleFinder.add_source_override`. """ self._finder.add_source_override(fullname, path, source, is_pkg) @@ -760,9 +920,11 @@ class ModuleResponder(object): self.blacklist.append(fullname) def neutralize_main(self, path, src): - """Given the source for the __main__ module, try to find where it - begins conditional execution based on a "if __name__ == '__main__'" - guard, and remove any code after that point.""" + """ + Given the source for the __main__ module, try to find where it begins + conditional execution based on a "if __name__ == '__main__'" guard, and + remove any code after that point. + """ match = self.MAIN_RE.search(src) if match: return src[:match.start()] @@ -770,7 +932,7 @@ class ModuleResponder(object): if b('mitogen.main(') in src: return src - LOG.error(self.main_guard_msg, path) + self._log.error(self.main_guard_msg, path) raise ImportError('refused') def _make_negative_response(self, fullname): @@ -789,8 +951,7 @@ class ModuleResponder(object): if path and is_stdlib_path(path): # Prevent loading of 2.x<->3.x stdlib modules! This costs one # RTT per hit, so a client-side solution is also required. - LOG.debug('%r: refusing to serve stdlib module %r', - self, fullname) + self._log.debug('refusing to serve stdlib module %r', fullname) tup = self._make_negative_response(fullname) self._cache[fullname] = tup return tup @@ -798,21 +959,21 @@ class ModuleResponder(object): if source is None: # TODO: make this .warning() or similar again once importer has its # own logging category. - LOG.debug('_build_tuple(%r): could not locate source', fullname) + self._log.debug('could not find source for %r', fullname) tup = self._make_negative_response(fullname) self._cache[fullname] = tup return tup if self.minify_safe_re.search(source): # If the module contains a magic marker, it's safe to minify. - t0 = time.time() + t0 = mitogen.core.now() source = mitogen.minify.minimize_source(source).encode('utf-8') - self.minify_secs += time.time() - t0 + self.minify_secs += mitogen.core.now() - t0 if is_pkg: pkg_present = get_child_modules(path) - LOG.debug('_build_tuple(%r, %r) -> %r', - path, fullname, pkg_present) + self._log.debug('%s is a package at %s with submodules %r', + fullname, path, pkg_present) else: pkg_present = None @@ -836,17 +997,17 @@ class ModuleResponder(object): return tup def _send_load_module(self, stream, fullname): - if fullname not in stream.sent_modules: + if fullname not in stream.protocol.sent_modules: tup = self._build_tuple(fullname) msg = mitogen.core.Message.pickled( tup, - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, handle=mitogen.core.LOAD_MODULE, ) - LOG.debug('%s: sending module %s (%.2f KiB)', - stream.name, fullname, len(msg.data) / 1024.0) + self._log.debug('sending %s (%.2f KiB) to %s', + fullname, len(msg.data) / 1024.0, stream.name) self._router._async_route(msg) - stream.sent_modules.add(fullname) + stream.protocol.sent_modules.add(fullname) if tup[2] is not None: self.good_load_module_count += 1 self.good_load_module_size += len(msg.data) @@ -855,23 +1016,23 @@ class ModuleResponder(object): def _send_module_load_failed(self, stream, fullname): self.bad_load_module_count += 1 - stream.send( + stream.protocol.send( mitogen.core.Message.pickled( self._make_negative_response(fullname), - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, handle=mitogen.core.LOAD_MODULE, ) ) def _send_module_and_related(self, stream, fullname): - if fullname in stream.sent_modules: + if fullname in stream.protocol.sent_modules: return try: tup = self._build_tuple(fullname) for name in tup[4]: # related parent, _, _ = str_partition(name, '.') - if parent != fullname and parent not in stream.sent_modules: + if parent != fullname and parent not in stream.protocol.sent_modules: # Parent hasn't been sent, so don't load submodule yet. continue @@ -890,25 +1051,25 @@ class ModuleResponder(object): return fullname = msg.data.decode() - LOG.debug('%s requested module %s', stream.name, fullname) + self._log.debug('%s requested module %s', stream.name, fullname) self.get_module_count += 1 - if fullname in stream.sent_modules: + if fullname in stream.protocol.sent_modules: LOG.warning('_on_get_module(): dup request for %r from %r', fullname, stream) - t0 = time.time() + t0 = mitogen.core.now() try: self._send_module_and_related(stream, fullname) finally: - self.get_module_secs += time.time() - t0 + self.get_module_secs += mitogen.core.now() - t0 def _send_forward_module(self, stream, context, fullname): - if stream.remote_id != context.context_id: - stream.send( + if stream.protocol.remote_id != context.context_id: + stream.protocol._send( mitogen.core.Message( data=b('%s\x00%s' % (context.context_id, fullname)), handle=mitogen.core.FORWARD_MODULE, - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, ) ) @@ -977,6 +1138,7 @@ class Broker(mitogen.core.Broker): on_join=self.shutdown, ) super(Broker, self).__init__() + self.timers = mitogen.parent.TimerList() def shutdown(self): super(Broker, self).shutdown() @@ -1122,6 +1284,21 @@ class Router(mitogen.parent.Router): class IdAllocator(object): + """ + Allocate IDs for new contexts constructed locally, and blocks of IDs for + children to allocate their own IDs using + :class:`mitogen.parent.ChildIdAllocator` without risk of conflict, and + without necessitating network round-trips for each new context. + + This class responds to :data:`mitogen.core.ALLOCATE_ID` messages received + from children by replying with fresh block ID allocations. + + The master's :class:`IdAllocator` instance can be accessed via + :attr:`mitogen.master.Router.id_allocator`. + """ + #: Block allocations are made in groups of 1000 by default. + BLOCK_SIZE = 1000 + def __init__(self, router): self.router = router self.next_id = 1 @@ -1134,14 +1311,12 @@ class IdAllocator(object): def __repr__(self): return 'IdAllocator(%r)' % (self.router,) - BLOCK_SIZE = 1000 - def allocate(self): """ - Arrange for a unique context ID to be allocated and associated with a - route leading to the active context. In masters, the ID is generated - directly, in children it is forwarded to the master via a - :data:`mitogen.core.ALLOCATE_ID` message. + Allocate a context ID by directly incrementing an internal counter. + + :returns: + The new context ID. """ self.lock.acquire() try: @@ -1152,6 +1327,15 @@ class IdAllocator(object): self.lock.release() def allocate_block(self): + """ + Allocate a block of IDs for use in a child context. + + This function is safe to call from any thread. + + :returns: + Tuple of the form `(id, end_id)` where `id` is the first usable ID + and `end_id` is the last usable ID. + """ self.lock.acquire() try: id_ = self.next_id diff --git a/mitogen/minify.py b/mitogen/minify.py index dc9f517c..09fdc4eb 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -44,7 +44,8 @@ else: def minimize_source(source): - """Remove comments and docstrings from Python `source`, preserving line + """ + Remove comments and docstrings from Python `source`, preserving line numbers and syntax of empty blocks. :param str source: @@ -62,7 +63,8 @@ def minimize_source(source): def strip_comments(tokens): - """Drop comment tokens from a `tokenize` stream. + """ + Drop comment tokens from a `tokenize` stream. Comments on lines 1-2 are kept, to preserve hashbang and encoding. Trailing whitespace is remove from all lines. @@ -84,7 +86,8 @@ def strip_comments(tokens): def strip_docstrings(tokens): - """Replace docstring tokens with NL tokens in a `tokenize` stream. + """ + Replace docstring tokens with NL tokens in a `tokenize` stream. Any STRING token not part of an expression is deemed a docstring. Indented docstrings are not yet recognised. @@ -119,7 +122,8 @@ def strip_docstrings(tokens): def reindent(tokens, indent=' '): - """Replace existing indentation in a token steam, with `indent`. + """ + Replace existing indentation in a token steam, with `indent`. """ old_levels = [] old_level = 0 diff --git a/mitogen/os_fork.py b/mitogen/os_fork.py index b27cfd5c..da832c65 100644 --- a/mitogen/os_fork.py +++ b/mitogen/os_fork.py @@ -35,6 +35,7 @@ Support for operating in a mixed threading/forking environment. import os import socket import sys +import threading import weakref import mitogen.core @@ -157,6 +158,7 @@ class Corker(object): held. This will not return until each thread acknowledges it has ceased execution. """ + current = threading.currentThread() s = mitogen.core.b('CORK') * ((128 // 4) * 1024) self._rsocks = [] @@ -164,12 +166,14 @@ class Corker(object): # participation of a broker in order to complete. for pool in self.pools: if not pool.closed: - for x in range(pool.size): - self._cork_one(s, pool) + for th in pool._threads: + if th != current: + self._cork_one(s, pool) for broker in self.brokers: if broker._alive: - self._cork_one(s, broker) + if broker._thread != current: + self._cork_one(s, broker) # Pause until we can detect every thread has entered write(). for rsock in self._rsocks: diff --git a/mitogen/parent.py b/mitogen/parent.py index 3d02bc43..1c3e1874 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -38,9 +38,11 @@ import codecs import errno import fcntl import getpass +import heapq import inspect import logging import os +import re import signal import socket import struct @@ -49,7 +51,6 @@ import sys import termios import textwrap import threading -import time import zlib # Absolute imports for <2.5. @@ -63,9 +64,22 @@ except ImportError: import mitogen.core from mitogen.core import b from mitogen.core import bytes_partition -from mitogen.core import LOG from mitogen.core import IOLOG + +LOG = logging.getLogger(__name__) + +# #410: we must avoid the use of socketpairs if SELinux is enabled. +try: + fp = open('/sys/fs/selinux/enforce', 'rb') + try: + SELINUX_ENABLED = bool(int(fp.read())) + finally: + fp.close() +except IOError: + SELINUX_ENABLED = False + + try: next except NameError: @@ -89,6 +103,10 @@ try: except ValueError: SC_OPEN_MAX = 1024 +BROKER_SHUTDOWN_MSG = ( + 'Connection cancelled because the associated Broker began to shut down.' +) + OPENPTY_MSG = ( "Failed to create a PTY: %s. It is likely the maximum number of PTYs has " "been reached. Consider increasing the 'kern.tty.ptmx_max' sysctl on OS " @@ -136,9 +154,12 @@ SIGNAL_BY_NUM = dict( if name.startswith('SIG') and not name.startswith('SIG_') ) +_core_source_lock = threading.Lock() +_core_source_partial = None + def get_log_level(): - return (LOG.level or logging.getLogger().level or logging.INFO) + return (LOG.getEffectiveLevel() or logging.INFO) def get_sys_executable(): @@ -157,10 +178,6 @@ def get_sys_executable(): return '/usr/bin/python' -_core_source_lock = threading.Lock() -_core_source_partial = None - - def _get_core_source(): """ In non-masters, simply fetch the cached mitogen.core source code via the @@ -208,27 +225,33 @@ def is_immediate_child(msg, stream): Handler policy that requires messages to arrive only from immediately connected children. """ - return msg.src_id == stream.remote_id + return msg.src_id == stream.protocol.remote_id def flags(names): - """Return the result of ORing a set of (space separated) :py:mod:`termios` - module constants together.""" + """ + Return the result of ORing a set of (space separated) :py:mod:`termios` + module constants together. + """ return sum(getattr(termios, name, 0) for name in names.split()) def cfmakeraw(tflags): - """Given a list returned by :py:func:`termios.tcgetattr`, return a list + """ + Given a list returned by :py:func:`termios.tcgetattr`, return a list modified in a manner similar to the `cfmakeraw()` C library function, but - additionally disabling local echo.""" - # BSD: https://github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162 - # Linux: https://github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20 + additionally disabling local echo. + """ + # BSD: github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162 + # Linux: github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20 iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags - iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ISTRIP INLCR ICRNL IXON IGNPAR') + iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ' + 'ISTRIP INLCR ICRNL IXON IGNPAR') iflag &= ~flags('IGNBRK BRKINT PARMRK') oflag &= ~flags('OPOST') - lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG IEXTEN NOFLSH TOSTOP PENDIN') + lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG ' + 'IEXTEN NOFLSH TOSTOP PENDIN') cflag &= ~flags('CSIZE PARENB') cflag |= flags('CS8 CREAD') return [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] @@ -245,128 +268,141 @@ def disable_echo(fd): termios.tcsetattr(fd, flags, new) -def close_nonstandard_fds(): - for fd in xrange(3, SC_OPEN_MAX): - try: - os.close(fd) - except OSError: - pass - - def create_socketpair(size=None): """ - Create a :func:`socket.socketpair` to use for use as a child process's UNIX - stdio channels. As socket pairs are bidirectional, they are economical on - file descriptor usage as the same descriptor can be used for ``stdin`` and + Create a :func:`socket.socketpair` for use as a child's UNIX stdio + channels. As socketpairs are bidirectional, they are economical on file + descriptor usage as one descriptor can be used for ``stdin`` and ``stdout``. As they are sockets their buffers are tunable, allowing large - buffers to be configured in order to improve throughput for file transfers - and reduce :class:`mitogen.core.Broker` IO loop iterations. + buffers to improve file transfer throughput and reduce IO loop iterations. """ + if size is None: + size = mitogen.core.CHUNK_SIZE + parentfp, childfp = socket.socketpair() - parentfp.setsockopt(socket.SOL_SOCKET, - socket.SO_SNDBUF, - size or mitogen.core.CHUNK_SIZE) - childfp.setsockopt(socket.SOL_SOCKET, - socket.SO_RCVBUF, - size or mitogen.core.CHUNK_SIZE) + for fp in parentfp, childfp: + fp.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, size) + return parentfp, childfp -def detach_popen(**kwargs): +def create_best_pipe(escalates_privilege=False): """ - Use :class:`subprocess.Popen` to construct a child process, then hack the - Popen so that it forgets the child it created, allowing it to survive a - call to Popen.__del__. + By default we prefer to communicate with children over a UNIX socket, as a + single file descriptor can represent bidirectional communication, and a + cross-platform API exists to align buffer sizes with the needs of the + library. - If the child process is not detached, there is a race between it exitting - and __del__ being called. If it exits before __del__ runs, then __del__'s - call to :func:`os.waitpid` will capture the one and only exit event - delivered to this process, causing later 'legitimate' calls to fail with - ECHILD. + SELinux prevents us setting up a privileged process to inherit an AF_UNIX + socket, a facility explicitly designed as a better replacement for pipes, + because at some point in the mid 90s it might have been commonly possible + for AF_INET sockets to end up undesirably connected to a privileged + process, so let's make up arbitrary rules breaking all sockets instead. - :param list close_on_error: - Array of integer file descriptors to close on exception. + If SELinux is detected, fall back to using pipes. + + :param bool escalates_privilege: + If :data:`True`, the target program may escalate privileges, causing + SELinux to disconnect AF_UNIX sockets, so avoid those. :returns: - Process ID of the new child. + `(parent_rfp, child_wfp, child_rfp, parent_wfp)` + """ + if (not escalates_privilege) or (not SELINUX_ENABLED): + parentfp, childfp = create_socketpair() + return parentfp, childfp, childfp, parentfp + + parent_rfp, child_wfp = mitogen.core.pipe() + try: + child_rfp, parent_wfp = mitogen.core.pipe() + return parent_rfp, child_wfp, child_rfp, parent_wfp + except: + parent_rfp.close() + child_wfp.close() + raise + + +def popen(**kwargs): + """ + Wrap :class:`subprocess.Popen` to ensure any global :data:`_preexec_hook` + is invoked in the child. """ - # This allows Popen() to be used for e.g. graceful post-fork error - # handling, without tying the surrounding code into managing a Popen - # object, which isn't possible for at least :mod:`mitogen.fork`. This - # should be replaced by a swappable helper class in a future version. real_preexec_fn = kwargs.pop('preexec_fn', None) def preexec_fn(): if _preexec_hook: _preexec_hook() if real_preexec_fn: real_preexec_fn() - proc = subprocess.Popen(preexec_fn=preexec_fn, **kwargs) - proc._child_created = False - return proc.pid + return subprocess.Popen(preexec_fn=preexec_fn, **kwargs) -def create_child(args, merge_stdio=False, stderr_pipe=False, preexec_fn=None): +def create_child(args, merge_stdio=False, stderr_pipe=False, + escalates_privilege=False, preexec_fn=None): """ Create a child process whose stdin/stdout is connected to a socket. - :param args: - Argument vector for execv() call. + :param list args: + Program argument vector. :param bool merge_stdio: If :data:`True`, arrange for `stderr` to be connected to the `stdout` socketpair, rather than inherited from the parent process. This may be - necessary to ensure that not TTY is connected to any stdio handle, for + necessary to ensure that no TTY is connected to any stdio handle, for instance when using LXC. :param bool stderr_pipe: If :data:`True` and `merge_stdio` is :data:`False`, arrange for `stderr` to be connected to a separate pipe, to allow any ongoing debug - logs generated by e.g. SSH to be outpu as the session progresses, + logs generated by e.g. SSH to be output as the session progresses, without interfering with `stdout`. + :param bool escalates_privilege: + If :data:`True`, the target program may escalate privileges, causing + SELinux to disconnect AF_UNIX sockets, so avoid those. + :param function preexec_fn: + If not :data:`None`, a function to run within the post-fork child + before executing the target program. :returns: - `(pid, socket_obj, :data:`None` or pipe_fd)` + :class:`Process` instance. """ - parentfp, childfp = create_socketpair() - # When running under a monkey patches-enabled gevent, the socket module - # yields file descriptors who already have O_NONBLOCK, which is - # persisted across fork, totally breaking Python. Therefore, drop - # O_NONBLOCK from Python's future stdin fd. - mitogen.core.set_block(childfp.fileno()) + parent_rfp, child_wfp, child_rfp, parent_wfp = create_best_pipe( + escalates_privilege=escalates_privilege + ) + stderr = None stderr_r = None - extra = {} if merge_stdio: - extra = {'stderr': childfp} + stderr = child_wfp elif stderr_pipe: - stderr_r, stderr_w = os.pipe() - mitogen.core.set_cloexec(stderr_r) - mitogen.core.set_cloexec(stderr_w) - extra = {'stderr': stderr_w} + stderr_r, stderr = mitogen.core.pipe() + mitogen.core.set_cloexec(stderr_r.fileno()) try: - pid = detach_popen( + proc = popen( args=args, - stdin=childfp, - stdout=childfp, + stdin=child_rfp, + stdout=child_wfp, + stderr=stderr, close_fds=True, preexec_fn=preexec_fn, - **extra ) - except Exception: - childfp.close() - parentfp.close() + except: + child_rfp.close() + child_wfp.close() + parent_rfp.close() + parent_wfp.close() if stderr_pipe: - os.close(stderr_r) - os.close(stderr_w) + stderr.close() + stderr_r.close() raise + child_rfp.close() + child_wfp.close() if stderr_pipe: - os.close(stderr_w) - childfp.close() - # Decouple the socket from the lifetime of the Python socket object. - fd = os.dup(parentfp.fileno()) - parentfp.close() + stderr.close() - LOG.debug('create_child() child %d fd %d, parent %d, cmd: %s', - pid, fd, os.getpid(), Argv(args)) - return pid, fd, stderr_r + return PopenProcess( + proc=proc, + stdin=parent_wfp, + stdout=parent_rfp, + stderr=stderr_r, + ) def _acquire_controlling_tty(): @@ -431,15 +467,22 @@ def openpty(): :raises mitogen.core.StreamError: Creating a PTY failed. :returns: - See :func`os.openpty`. + `(master_fp, slave_fp)` file-like objects. """ try: - return os.openpty() + master_fd, slave_fd = os.openpty() except OSError: e = sys.exc_info()[1] - if IS_LINUX and e.args[0] == errno.EPERM: - return _linux_broken_devpts_openpty() - raise mitogen.core.StreamError(OPENPTY_MSG, e) + if not (IS_LINUX and e.args[0] == errno.EPERM): + raise mitogen.core.StreamError(OPENPTY_MSG, e) + master_fd, slave_fd = _linux_broken_devpts_openpty() + + master_fp = os.fdopen(master_fd, 'r+b', 0) + slave_fp = os.fdopen(slave_fd, 'r+b', 0) + disable_echo(master_fd) + disable_echo(slave_fd) + mitogen.core.set_block(slave_fd) + return master_fp, slave_fp def tty_create_child(args): @@ -451,130 +494,187 @@ def tty_create_child(args): slave end. :param list args: - :py:func:`os.execl` argument list. - + Program argument vector. :returns: - `(pid, tty_fd, None)` + :class:`Process` instance. """ - master_fd, slave_fd = openpty() + master_fp, slave_fp = openpty() try: - mitogen.core.set_block(slave_fd) - disable_echo(master_fd) - disable_echo(slave_fd) - - pid = detach_popen( + proc = popen( args=args, - stdin=slave_fd, - stdout=slave_fd, - stderr=slave_fd, + stdin=slave_fp, + stdout=slave_fp, + stderr=slave_fp, preexec_fn=_acquire_controlling_tty, close_fds=True, ) - except Exception: - os.close(master_fd) - os.close(slave_fd) + except: + master_fp.close() + slave_fp.close() raise - os.close(slave_fd) - LOG.debug('tty_create_child() child %d fd %d, parent %d, cmd: %s', - pid, master_fd, os.getpid(), Argv(args)) - return pid, master_fd, None + slave_fp.close() + return PopenProcess( + proc=proc, + stdin=master_fp, + stdout=master_fp, + ) -def hybrid_tty_create_child(args): +def hybrid_tty_create_child(args, escalates_privilege=False): """ Like :func:`tty_create_child`, except attach stdin/stdout to a socketpair like :func:`create_child`, but leave stderr and the controlling TTY attached to a TTY. - :param list args: - :py:func:`os.execl` argument list. + This permits high throughput communication with programs that are reached + via some program that requires a TTY for password input, like many + configurations of sudo. The UNIX TTY layer tends to have tiny (no more than + 14KiB) buffers, forcing many IO loop iterations when transferring bulk + data, causing significant performance loss. + :param bool escalates_privilege: + If :data:`True`, the target program may escalate privileges, causing + SELinux to disconnect AF_UNIX sockets, so avoid those. + :param list args: + Program argument vector. :returns: - `(pid, socketpair_fd, tty_fd)` + :class:`Process` instance. """ - master_fd, slave_fd = openpty() - + master_fp, slave_fp = openpty() try: - disable_echo(master_fd) - disable_echo(slave_fd) - mitogen.core.set_block(slave_fd) - - parentfp, childfp = create_socketpair() + parent_rfp, child_wfp, child_rfp, parent_wfp = create_best_pipe( + escalates_privilege=escalates_privilege, + ) try: - mitogen.core.set_block(childfp) - pid = detach_popen( + mitogen.core.set_block(child_rfp) + mitogen.core.set_block(child_wfp) + proc = popen( args=args, - stdin=childfp, - stdout=childfp, - stderr=slave_fd, + stdin=child_rfp, + stdout=child_wfp, + stderr=slave_fp, preexec_fn=_acquire_controlling_tty, close_fds=True, ) - except Exception: - parentfp.close() - childfp.close() + except: + parent_rfp.close() + child_wfp.close() + parent_wfp.close() + child_rfp.close() raise - except Exception: - os.close(master_fd) - os.close(slave_fd) + except: + master_fp.close() + slave_fp.close() raise - os.close(slave_fd) - childfp.close() - # Decouple the socket from the lifetime of the Python socket object. - stdio_fd = os.dup(parentfp.fileno()) - parentfp.close() - - LOG.debug('hybrid_tty_create_child() pid=%d stdio=%d, tty=%d, cmd: %s', - pid, stdio_fd, master_fd, Argv(args)) - return pid, stdio_fd, master_fd - - -def write_all(fd, s, deadline=None): - """Arrange for all of bytestring `s` to be written to the file descriptor - `fd`. - - :param int fd: - File descriptor to write to. - :param bytes s: - Bytestring to write to file descriptor. - :param float deadline: - If not :data:`None`, absolute UNIX timestamp after which timeout should - occur. - - :raises mitogen.core.TimeoutError: - Bytestring could not be written entirely before deadline was exceeded. - :raises mitogen.parent.EofError: - Stream indicated EOF, suggesting the child process has exitted. - :raises mitogen.core.StreamError: - File descriptor was disconnected before write could complete. + slave_fp.close() + child_rfp.close() + child_wfp.close() + return PopenProcess( + proc=proc, + stdin=parent_wfp, + stdout=parent_rfp, + stderr=master_fp, + ) + + +class Timer(object): + """ + Represents a future event. """ - timeout = None - written = 0 - poller = PREFERRED_POLLER() - poller.start_transmit(fd) + #: Set to :data:`False` if :meth:`cancel` has been called, or immediately + #: prior to being executed by :meth:`TimerList.expire`. + active = True - try: - while written < len(s): - if deadline is not None: - timeout = max(0, deadline - time.time()) - if timeout == 0: - raise mitogen.core.TimeoutError('write timed out') - - if mitogen.core.PY3: - window = memoryview(s)[written:] - else: - window = buffer(s, written) + def __init__(self, when, func): + self.when = when + self.func = func - for fd in poller.poll(timeout): - n, disconnected = mitogen.core.io_op(os.write, fd, window) - if disconnected: - raise EofError('EOF on stream during write') + def __repr__(self): + return 'Timer(%r, %r)' % (self.when, self.func) - written += n - finally: - poller.close() + def __eq__(self, other): + return self.when == other.when + + def __lt__(self, other): + return self.when < other.when + + def __le__(self, other): + return self.when <= other.when + + def cancel(self): + """ + Cancel this event. If it has not yet executed, it will not execute + during any subsequent :meth:`TimerList.expire` call. + """ + self.active = False + + +class TimerList(object): + """ + Efficiently manage a list of cancellable future events relative to wall + clock time. An instance of this class is installed as + :attr:`mitogen.master.Broker.timers` by default, and as + :attr:`mitogen.core.Broker.timers` in children after a call to + :func:`mitogen.parent.upgrade_router`. + + You can use :class:`TimerList` to cause the broker to wake at arbitrary + future moments, useful for implementing timeouts and polling in an + asynchronous context. + + :class:`TimerList` methods can only be called from asynchronous context, + for example via :meth:`mitogen.core.Broker.defer`. + + The broker automatically adjusts its sleep delay according to the installed + timer list, and arranges for timers to expire via automatic calls to + :meth:`expire`. The main user interface to :class:`TimerList` is + :meth:`schedule`. + """ + _now = mitogen.core.now + + def __init__(self): + self._lst = [] + + def get_timeout(self): + """ + Return the floating point seconds until the next event is due. + + :returns: + Floating point delay, or 0.0, or :data:`None` if no events are + scheduled. + """ + while self._lst and not self._lst[0].active: + heapq.heappop(self._lst) + if self._lst: + return max(0, self._lst[0].when - self._now()) + + def schedule(self, when, func): + """ + Schedule a future event. + + :param float when: + UNIX time in seconds when event should occur. + :param callable func: + Callable to invoke on expiry. + :returns: + A :class:`Timer` instance, exposing :meth:`Timer.cancel`, which may + be used to cancel the future invocation. + """ + timer = Timer(when, func) + heapq.heappush(self._lst, timer) + return timer + + def expire(self): + """ + Invoke callbacks for any events in the past. + """ + now = self._now() + while self._lst and self._lst[0].when <= now: + timer = heapq.heappop(self._lst) + if timer.active: + timer.active = False + timer.func() class PartialZlib(object): @@ -614,103 +714,6 @@ class PartialZlib(object): return out + compressor.flush() -class IteratingRead(object): - def __init__(self, fds, deadline=None): - self.deadline = deadline - self.timeout = None - self.poller = PREFERRED_POLLER() - for fd in fds: - self.poller.start_receive(fd) - - self.bits = [] - self.timeout = None - - def close(self): - self.poller.close() - - def __iter__(self): - return self - - def next(self): - while self.poller.readers: - if self.deadline is not None: - self.timeout = max(0, self.deadline - time.time()) - if self.timeout == 0: - break - - for fd in self.poller.poll(self.timeout): - s, disconnected = mitogen.core.io_op(os.read, fd, 4096) - if disconnected or not s: - LOG.debug('iter_read(%r) -> disconnected: %s', - fd, disconnected) - self.poller.stop_receive(fd) - else: - IOLOG.debug('iter_read(%r) -> %r', fd, s) - self.bits.append(s) - return s - - if not self.poller.readers: - raise EofError(u'EOF on stream; last 300 bytes received: %r' % - (b('').join(self.bits)[-300:].decode('latin1'),)) - - raise mitogen.core.TimeoutError('read timed out') - - __next__ = next - - -def iter_read(fds, deadline=None): - """Return a generator that arranges for up to 4096-byte chunks to be read - at a time from the file descriptor `fd` until the generator is destroyed. - - :param int fd: - File descriptor to read from. - :param float deadline: - If not :data:`None`, an absolute UNIX timestamp after which timeout - should occur. - - :raises mitogen.core.TimeoutError: - Attempt to read beyond deadline. - :raises mitogen.parent.EofError: - All streams indicated EOF, suggesting the child process has exitted. - :raises mitogen.core.StreamError: - Attempt to read past end of file. - """ - return IteratingRead(fds=fds, deadline=deadline) - - -def discard_until(fd, s, deadline): - """Read chunks from `fd` until one is encountered that ends with `s`. This - is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and - mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to - appear, indicating the first stage is ready to receive the compressed - :mod:`mitogen.core` source. - - :param int fd: - File descriptor to read from. - :param bytes s: - Marker string to discard until encountered. - :param float deadline: - Absolute UNIX timestamp after which timeout should occur. - - :raises mitogen.core.TimeoutError: - Attempt to read beyond deadline. - :raises mitogen.parent.EofError: - All streams indicated EOF, suggesting the child process has exitted. - :raises mitogen.core.StreamError: - Attempt to read past end of file. - """ - it = iter_read([fd], deadline) - try: - for buf in it: - if IOLOG.level == logging.DEBUG: - for line in buf.splitlines(): - IOLOG.debug('discard_until: discarding %r', line) - if buf.endswith(s): - return - finally: - it.close() # ensure Poller.close() is called. - - def _upgrade_broker(broker): """ Extract the poller state from Broker and replace it with the industrial @@ -719,25 +722,28 @@ def _upgrade_broker(broker): # This function is deadly! The act of calling start_receive() generates log # messages which must be silenced as the upgrade progresses, otherwise the # poller state will change as it is copied, resulting in write fds that are - # lost. (Due to LogHandler->Router->Stream->Broker->Poller, where Stream - # only calls start_transmit() when transitioning from empty to non-empty - # buffer. If the start_transmit() is lost, writes from the child hang - # permanently). + # lost. (Due to LogHandler->Router->Stream->Protocol->Broker->Poller, where + # Stream only calls start_transmit() when transitioning from empty to + # non-empty buffer. If the start_transmit() is lost, writes from the child + # hang permanently). root = logging.getLogger() old_level = root.level root.setLevel(logging.CRITICAL) + try: + old = broker.poller + new = PREFERRED_POLLER() + for fd, data in old.readers: + new.start_receive(fd, data) + for fd, data in old.writers: + new.start_transmit(fd, data) + + old.close() + broker.poller = new + finally: + root.setLevel(old_level) - old = broker.poller - new = PREFERRED_POLLER() - for fd, data in old.readers: - new.start_receive(fd, data) - for fd, data in old.writers: - new.start_transmit(fd, data) - - old.close() - broker.poller = new - root.setLevel(old_level) - LOG.debug('replaced %r with %r (new: %d readers, %d writers; ' + broker.timers = TimerList() + LOG.debug('upgraded %r with %r (new: %d readers, %d writers; ' 'old: %d readers, %d writers)', old, new, len(new.readers), len(new.writers), len(old.readers), len(old.writers)) @@ -754,7 +760,7 @@ def upgrade_router(econtext): ) -def stream_by_method_name(name): +def get_connection_class(name): """ Given the name of a Mitogen connection method, import its implementation module and return its Stream subclass. @@ -762,14 +768,14 @@ def stream_by_method_name(name): if name == u'local': name = u'parent' module = mitogen.core.import_module(u'mitogen.' + name) - return module.Stream + return module.Connection @mitogen.core.takes_econtext def _proxy_connect(name, method_name, kwargs, econtext): """ Implements the target portion of Router._proxy_connect() by upgrading the - local context to a parent if it was not already, then calling back into + local process to a parent if it was not already, then calling back into Router._connect() using the arguments passed to the parent's Router.connect(). @@ -783,7 +789,7 @@ def _proxy_connect(name, method_name, kwargs, econtext): try: context = econtext.router._connect( - klass=stream_by_method_name(method_name), + klass=get_connection_class(method_name), name=name, **kwargs ) @@ -804,30 +810,32 @@ def _proxy_connect(name, method_name, kwargs, econtext): } -def wstatus_to_str(status): +def returncode_to_str(n): """ Parse and format a :func:`os.waitpid` exit status. """ - if os.WIFEXITED(status): - return 'exited with return code %d' % (os.WEXITSTATUS(status),) - if os.WIFSIGNALED(status): - n = os.WTERMSIG(status) - return 'exited due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) - if os.WIFSTOPPED(status): - n = os.WSTOPSIG(status) - return 'stopped due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) - return 'unknown wait status (%d)' % (status,) + if n < 0: + return 'exited due to signal %d (%s)' % (-n, SIGNAL_BY_NUM.get(-n)) + return 'exited with return code %d' % (n,) class EofError(mitogen.core.StreamError): """ - Raised by :func:`iter_read` and :func:`write_all` when EOF is detected by - the child process. + Raised by :class:`Connection` when an empty read is detected from the + remote process before bootstrap completes. """ # inherits from StreamError to maintain compatibility. pass +class CancelledError(mitogen.core.StreamError): + """ + Raised by :class:`Connection` when :meth:`mitogen.core.Broker.shutdown` is + called before bootstrap completes. + """ + pass + + class Argv(object): """ Wrapper to defer argv formatting when debug logging is disabled. @@ -893,8 +901,9 @@ class CallSpec(object): class PollPoller(mitogen.core.Poller): """ - Poller based on the POSIX poll(2) interface. Not available on some versions - of OS X, otherwise it is the preferred poller for small FD counts. + Poller based on the POSIX :linux:man2:`poll` interface. Not available on + some versions of OS X, otherwise it is the preferred poller for small FD + counts, as there is no setup/teardown/configuration system call overhead. """ SUPPORTED = hasattr(select, 'poll') _repr = 'PollPoller()' @@ -940,7 +949,7 @@ class PollPoller(mitogen.core.Poller): class KqueuePoller(mitogen.core.Poller): """ - Poller based on the FreeBSD/Darwin kqueue(2) interface. + Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface. """ SUPPORTED = hasattr(select, 'kqueue') _repr = 'KqueuePoller()' @@ -1018,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux epoll(2) interface. + Poller based on the Linux :linux:man2:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' @@ -1096,90 +1105,256 @@ for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: if _klass.SUPPORTED: PREFERRED_POLLER = _klass -# For apps that start threads dynamically, it's possible Latch will also get -# very high-numbered wait fds when there are many connections, and so select() -# becomes useless there too. So swap in our favourite poller. +# For processes that start many threads or connections, it's possible Latch +# will also get high-numbered FDs, and so select() becomes useless there too. +# So swap in our favourite poller. if PollPoller.SUPPORTED: mitogen.core.Latch.poller_class = PollPoller else: mitogen.core.Latch.poller_class = PREFERRED_POLLER -class DiagLogStream(mitogen.core.BasicStream): +class LineLoggingProtocolMixin(object): + def __init__(self, **kwargs): + super(LineLoggingProtocolMixin, self).__init__(**kwargs) + self.logged_lines = [] + self.logged_partial = None + + def on_line_received(self, line): + self.logged_partial = None + self.logged_lines.append((mitogen.core.now(), line)) + self.logged_lines[:] = self.logged_lines[-100:] + return super(LineLoggingProtocolMixin, self).on_line_received(line) + + def on_partial_line_received(self, line): + self.logged_partial = line + return super(LineLoggingProtocolMixin, self).on_partial_line_received(line) + + def on_disconnect(self, broker): + if self.logged_partial: + self.logged_lines.append((mitogen.core.now(), self.logged_partial)) + self.logged_partial = None + super(LineLoggingProtocolMixin, self).on_disconnect(broker) + + +def get_history(streams): + history = [] + for stream in streams: + if stream: + history.extend(getattr(stream.protocol, 'logged_lines', [])) + history.sort() + + s = b('\n').join(h[1] for h in history) + return mitogen.core.to_text(s) + + +class RegexProtocol(LineLoggingProtocolMixin, mitogen.core.DelimitedProtocol): """ - For "hybrid TTY/socketpair" mode, after a connection has been setup, a - spare TTY file descriptor will exist that cannot be closed, and to which - SSH or sudo may continue writing log messages. + Implement a delimited protocol where messages matching a set of regular + expressions are dispatched to individual handler methods. Input is + dispatches using :attr:`PATTERNS` and :attr:`PARTIAL_PATTERNS`, before + falling back to :meth:`on_unrecognized_line_received` and + :meth:`on_unrecognized_partial_line_received`. + """ + #: A sequence of 2-tuples of the form `(compiled pattern, method)` for + #: patterns that should be matched against complete (delimited) messages, + #: i.e. full lines. + PATTERNS = [] + + #: Like :attr:`PATTERNS`, but patterns that are matched against incomplete + #: lines. + PARTIAL_PATTERNS = [] + + def on_line_received(self, line): + super(RegexProtocol, self).on_line_received(line) + for pattern, func in self.PATTERNS: + match = pattern.search(line) + if match is not None: + return func(self, line, match) + + return self.on_unrecognized_line_received(line) + + def on_unrecognized_line_received(self, line): + LOG.debug('%s: (unrecognized): %s', + self.stream.name, line.decode('utf-8', 'replace')) - The descriptor cannot be closed since the UNIX TTY layer will send a - termination signal to any processes whose controlling TTY is the TTY that - has been closed. + def on_partial_line_received(self, line): + super(RegexProtocol, self).on_partial_line_received(line) + LOG.debug('%s: (partial): %s', + self.stream.name, line.decode('utf-8', 'replace')) + for pattern, func in self.PARTIAL_PATTERNS: + match = pattern.search(line) + if match is not None: + return func(self, line, match) - DiagLogStream takes over this descriptor and creates corresponding log - messages for anything written to it. + return self.on_unrecognized_partial_line_received(line) + + def on_unrecognized_partial_line_received(self, line): + LOG.debug('%s: (unrecognized partial): %s', + self.stream.name, line.decode('utf-8', 'replace')) + + +class BootstrapProtocol(RegexProtocol): """ + Respond to stdout of a child during bootstrap. Wait for :attr:`EC0_MARKER` + to be written by the first stage to indicate it can receive the bootstrap, + then await :attr:`EC1_MARKER` to indicate success, and + :class:`MitogenProtocol` can be enabled. + """ + #: Sentinel value emitted by the first stage to indicate it is ready to + #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have + #: length of at least `max(len('password'), len('debug1:'))` + EC0_MARKER = b('MITO000') + EC1_MARKER = b('MITO001') + EC2_MARKER = b('MITO002') - def __init__(self, fd, stream): - self.receive_side = mitogen.core.Side(self, fd) - self.transmit_side = self.receive_side - self.stream = stream - self.buf = '' + def __init__(self, broker): + super(BootstrapProtocol, self).__init__() + self._writer = mitogen.core.BufferedWriter(broker, self) - def __repr__(self): - return "mitogen.parent.DiagLogStream(fd=%r, '%s')" % ( - self.receive_side.fd, - self.stream.name, - ) + def on_transmit(self, broker): + self._writer.on_transmit(broker) + + def _on_ec0_received(self, line, match): + LOG.debug('%r: first stage started succcessfully', self) + self._writer.write(self.stream.conn.get_preamble()) + + def _on_ec1_received(self, line, match): + LOG.debug('%r: first stage received mitogen.core source', self) - def on_receive(self, broker): + def _on_ec2_received(self, line, match): + LOG.debug('%r: new child booted successfully', self) + self.stream.conn._complete_connection() + return False + + def on_unrecognized_line_received(self, line): + LOG.debug('%s: stdout: %s', self.stream.name, + line.decode('utf-8', 'replace')) + + PATTERNS = [ + (re.compile(EC0_MARKER), _on_ec0_received), + (re.compile(EC1_MARKER), _on_ec1_received), + (re.compile(EC2_MARKER), _on_ec2_received), + ] + + +class LogProtocol(LineLoggingProtocolMixin, mitogen.core.DelimitedProtocol): + """ + For "hybrid TTY/socketpair" mode, after connection setup a spare TTY master + FD exists that cannot be closed, and to which SSH or sudo may continue + writing log messages. + + The descriptor cannot be closed since the UNIX TTY layer sends SIGHUP to + processes whose controlling TTY is the slave whose master side was closed. + LogProtocol takes over this FD and creates log messages for anything + written to it. + """ + def on_line_received(self, line): """ - This handler is only called after the stream is registered with the IO - loop, the descriptor is manually read/written by _connect_bootstrap() - prior to that. + Read a line, decode it as UTF-8, and log it. """ - buf = self.receive_side.read() - if not buf: - return self.on_disconnect(broker) - - self.buf += buf.decode('utf-8', 'replace') - while u'\n' in self.buf: - lines = self.buf.split('\n') - self.buf = lines[-1] - for line in lines[:-1]: - LOG.debug('%s: %s', self.stream.name, line.rstrip()) + super(LogProtocol, self).on_line_received(line) + LOG.info(u'%s: %s', self.stream.name, line.decode('utf-8', 'replace')) -class Stream(mitogen.core.Stream): +class MitogenProtocol(mitogen.core.MitogenProtocol): """ - Base for streams capable of starting new slaves. + Extend core.MitogenProtocol to cause SHUTDOWN to be sent to the child + during graceful shutdown. """ + def on_shutdown(self, broker): + """ + Respond to the broker's request for the stream to shut down by sending + SHUTDOWN to the child. + """ + LOG.debug('%r: requesting child shutdown', self) + self._send( + mitogen.core.Message( + src_id=mitogen.context_id, + dst_id=self.remote_id, + handle=mitogen.core.SHUTDOWN, + ) + ) + + +class Options(object): + name = None + #: The path to the remote Python interpreter. python_path = get_sys_executable() #: Maximum time to wait for a connection attempt. connect_timeout = 30.0 - #: Derived from :py:attr:`connect_timeout`; absolute floating point - #: UNIX timestamp after which the connection attempt should be abandoned. - connect_deadline = None - #: True to cause context to write verbose /tmp/mitogen.<pid>.log. debug = False #: True to cause context to write /tmp/mitogen.stats.<pid>.<thread>.log. profiling = False - #: Set to the child's PID by connect(). - pid = None + #: True if unidirectional routing is enabled in the new child. + unidirectional = False #: Passed via Router wrapper methods, must eventually be passed to #: ExternalContext.main(). max_message_size = None - #: If :attr:`create_child` supplied a diag_fd, references the corresponding - #: :class:`DiagLogStream`, allowing it to be disconnected when this stream - #: is disconnected. Set to :data:`None` if no `diag_fd` was present. - diag_stream = None + #: Remote name. + remote_name = None + + #: Derived from :py:attr:`connect_timeout`; absolute floating point + #: UNIX timestamp after which the connection attempt should be abandoned. + connect_deadline = None + + def __init__(self, max_message_size, name=None, remote_name=None, + python_path=None, debug=False, connect_timeout=None, + profiling=False, unidirectional=False, old_router=None): + self.name = name + self.max_message_size = max_message_size + if python_path: + self.python_path = python_path + if connect_timeout: + self.connect_timeout = connect_timeout + if remote_name is None: + remote_name = get_default_remote_name() + if '/' in remote_name or '\\' in remote_name: + raise ValueError('remote_name= cannot contain slashes') + if remote_name: + self.remote_name = mitogen.core.to_text(remote_name) + self.debug = debug + self.profiling = profiling + self.unidirectional = unidirectional + self.max_message_size = max_message_size + self.connect_deadline = mitogen.core.now() + self.connect_timeout + + +class Connection(object): + """ + Manage the lifetime of a set of :class:`Streams <Stream>` connecting to a + remote Python interpreter, including bootstrap, disconnection, and external + tool integration. + + Base for streams capable of starting children. + """ + options_class = Options + + #: The protocol attached to stdio of the child. + stream_protocol_class = BootstrapProtocol + + #: The protocol attached to stderr of the child. + diag_protocol_class = LogProtocol + + #: :class:`Process` + proc = None + + #: :class:`mitogen.core.Stream` with sides connected to stdin/stdout. + stdio_stream = None + + #: If `proc.stderr` is set, referencing either a plain pipe or the + #: controlling TTY, this references the corresponding + #: :class:`LogProtocol`'s stream, allowing it to be disconnected when this + #: stream is disconnected. + stderr_stream = None #: Function with the semantics of :func:`create_child` used to create the #: child process. @@ -1201,93 +1376,30 @@ class Stream(mitogen.core.Stream): #: Prefix given to default names generated by :meth:`connect`. name_prefix = u'local' - _reaped = False + #: :class:`Timer` that runs :meth:`_on_timer_expired` when connection + #: timeout occurs. + _timer = None - def __init__(self, *args, **kwargs): - super(Stream, self).__init__(*args, **kwargs) - self.sent_modules = set(['mitogen', 'mitogen.core']) - - def construct(self, max_message_size, remote_name=None, python_path=None, - debug=False, connect_timeout=None, profiling=False, - unidirectional=False, old_router=None, **kwargs): - """Get the named context running on the local machine, creating it if - it does not exist.""" - super(Stream, self).construct(**kwargs) - self.max_message_size = max_message_size - if python_path: - self.python_path = python_path - if connect_timeout: - self.connect_timeout = connect_timeout - if remote_name is None: - remote_name = get_default_remote_name() - if '/' in remote_name or '\\' in remote_name: - raise ValueError('remote_name= cannot contain slashes') - self.remote_name = remote_name - self.debug = debug - self.profiling = profiling - self.unidirectional = unidirectional - self.max_message_size = max_message_size - self.connect_deadline = time.time() + self.connect_timeout + #: When disconnection completes, instance of :class:`Reaper` used to wait + #: on the exit status of the subprocess. + _reaper = None - def on_shutdown(self, broker): - """Request the slave gracefully shut itself down.""" - LOG.debug('%r closing CALL_FUNCTION channel', self) - self._send( - mitogen.core.Message( - src_id=mitogen.context_id, - dst_id=self.remote_id, - handle=mitogen.core.SHUTDOWN, - ) - ) - - def _reap_child(self): - """ - Reap the child process during disconnection. - """ - if self.detached and self.child_is_immediate_subprocess: - LOG.debug('%r: immediate child is detached, won\'t reap it', self) - return + #: On failure, the exception object that should be propagated back to the + #: user. + exception = None - if self.profiling: - LOG.info('%r: wont kill child because profiling=True', self) - return - - if self._reaped: - # on_disconnect() may be invoked more than once, for example, if - # there is still a pending message to be sent after the first - # on_disconnect() call. - return - - try: - pid, status = os.waitpid(self.pid, os.WNOHANG) - except OSError: - e = sys.exc_info()[1] - if e.args[0] == errno.ECHILD: - LOG.warn('%r: waitpid(%r) produced ECHILD', self, self.pid) - return - raise - - self._reaped = True - if pid: - LOG.debug('%r: PID %d %s', self, pid, wstatus_to_str(status)) - return + #: Extra text appended to :class:`EofError` if that exception is raised on + #: a failed connection attempt. May be used in subclasses to hint at common + #: problems with a particular connection method. + eof_error_hint = None - if not self._router.profiling: - # For processes like sudo we cannot actually send sudo a signal, - # because it is setuid, so this is best-effort only. - LOG.debug('%r: child process still alive, sending SIGTERM', self) - try: - os.kill(self.pid, signal.SIGTERM) - except OSError: - e = sys.exc_info()[1] - if e.args[0] != errno.EPERM: - raise + def __init__(self, options, router): + #: :class:`Options` + self.options = options + self._router = router - def on_disconnect(self, broker): - super(Stream, self).on_disconnect(broker) - if self.diag_stream is not None: - self.diag_stream.on_disconnect(broker) - self._reap_child() + def __repr__(self): + return 'Connection(%r)' % (self.stdio_stream,) # Minimised, gzipped, base64'd and passed to 'python -c'. It forks, dups # file descriptor 0 as 100, creates a pipe, then execs a new interpreter @@ -1346,15 +1458,15 @@ class Stream(mitogen.core.Stream): This allows emulation of existing tools where the Python invocation may be set to e.g. `['/usr/bin/env', 'python']`. """ - if isinstance(self.python_path, list): - return self.python_path - return [self.python_path] + if isinstance(self.options.python_path, list): + return self.options.python_path + return [self.options.python_path] def get_boot_command(self): source = inspect.getsource(self._first_stage) source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:])) source = source.replace(' ', '\t') - source = source.replace('CONTEXT_NAME', self.remote_name) + source = source.replace('CONTEXT_NAME', self.options.remote_name) preamble_compressed = self.get_preamble() source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) @@ -1372,19 +1484,19 @@ class Stream(mitogen.core.Stream): ] def get_econtext_config(self): - assert self.max_message_size is not None + assert self.options.max_message_size is not None parent_ids = mitogen.parent_ids[:] parent_ids.insert(0, mitogen.context_id) return { 'parent_ids': parent_ids, - 'context_id': self.remote_id, - 'debug': self.debug, - 'profiling': self.profiling, - 'unidirectional': self.unidirectional, + 'context_id': self.context.context_id, + 'debug': self.options.debug, + 'profiling': self.options.profiling, + 'unidirectional': self.options.unidirectional, 'log_level': get_log_level(), 'whitelist': self._router.get_module_whitelist(), 'blacklist': self._router.get_module_blacklist(), - 'max_message_size': self.max_message_size, + 'max_message_size': self.options.max_message_size, 'version': mitogen.__version__, } @@ -1396,93 +1508,232 @@ class Stream(mitogen.core.Stream): partial = get_core_source_partial() return partial.append(suffix.encode('utf-8')) + def _get_name(self): + """ + Called by :meth:`connect` after :attr:`pid` is known. Subclasses can + override it to specify a default stream name, or set + :attr:`name_prefix` to generate a default format. + """ + return u'%s.%s' % (self.name_prefix, self.proc.pid) + def start_child(self): args = self.get_boot_command() + LOG.debug('command line for %r: %s', self, Argv(args)) try: - return self.create_child(args, **self.create_child_args) + return self.create_child(args=args, **self.create_child_args) except OSError: e = sys.exc_info()[1] msg = 'Child start failed: %s. Command was: %s' % (e, Argv(args)) raise mitogen.core.StreamError(msg) - eof_error_hint = None - def _adorn_eof_error(self, e): """ - Used by subclasses to provide additional information in the case of a - failed connection. + Subclasses may provide additional information in the case of a failed + connection. """ if self.eof_error_hint: e.args = ('%s\n\n%s' % (e.args[0], self.eof_error_hint),) - def _get_name(self): + def _complete_connection(self): + self._timer.cancel() + if not self.exception: + mitogen.core.unlisten(self._router.broker, 'shutdown', + self._on_broker_shutdown) + self._router.register(self.context, self.stdio_stream) + self.stdio_stream.set_protocol( + MitogenProtocol( + router=self._router, + remote_id=self.context.context_id, + ) + ) + self._router.route_monitor.notice_stream(self.stdio_stream) + self.latch.put() + + def _fail_connection(self, exc): """ - Called by :meth:`connect` after :attr:`pid` is known. Subclasses can - override it to specify a default stream name, or set - :attr:`name_prefix` to generate a default format. + Fail the connection attempt. + """ + LOG.debug('failing connection %s due to %r', + self.stdio_stream and self.stdio_stream.name, exc) + if self.exception is None: + self._adorn_eof_error(exc) + self.exception = exc + mitogen.core.unlisten(self._router.broker, 'shutdown', + self._on_broker_shutdown) + for stream in self.stdio_stream, self.stderr_stream: + if stream and not stream.receive_side.closed: + stream.on_disconnect(self._router.broker) + self._complete_connection() + + eof_error_msg = 'EOF on stream; last 100 lines received:\n' + + def on_stdio_disconnect(self): """ - return u'%s.%s' % (self.name_prefix, self.pid) + Handle stdio stream disconnection by failing the Connection if the + stderr stream has already been closed. Otherwise, wait for it to close + (or timeout), to allow buffered diagnostic logs to be consumed. + + It is normal that when a subprocess aborts, stdio has nothing buffered + when it is closed, thus signalling readability, causing an empty read + (interpreted as indicating disconnection) on the next loop iteration, + even if its stderr pipe has lots of diagnostic logs still buffered in + the kernel. Therefore we must wait for both pipes to indicate they are + empty before triggering connection failure. + """ + stderr = self.stderr_stream + if stderr is None or stderr.receive_side.closed: + self._on_streams_disconnected() - def connect(self): - LOG.debug('%r.connect()', self) - self.pid, fd, diag_fd = self.start_child() - self.name = self._get_name() - self.receive_side = mitogen.core.Side(self, fd) - self.transmit_side = mitogen.core.Side(self, os.dup(fd)) - if diag_fd is not None: - self.diag_stream = DiagLogStream(diag_fd, self) - else: - self.diag_stream = None + def on_stderr_disconnect(self): + """ + Inverse of :func:`on_stdio_disconnect`. + """ + if self.stdio_stream.receive_side.closed: + self._on_streams_disconnected() + + def _on_streams_disconnected(self): + """ + When disconnection has been detected for both streams, cancel the + connection timer, mark the connection failed, and reap the child + process. Do nothing if the timer has already been cancelled, indicating + some existing failure has already been noticed. + """ + if self._timer.active: + self._timer.cancel() + self._fail_connection(EofError( + self.eof_error_msg + get_history( + [self.stdio_stream, self.stderr_stream] + ) + )) + + if self._reaper: + return + + self._reaper = Reaper( + broker=self._router.broker, + proc=self.proc, + kill=not ( + (self.detached and self.child_is_immediate_subprocess) or + # Avoid killing so child has chance to write cProfile data + self._router.profiling + ), + # Don't delay shutdown waiting for a detached child, since the + # detached child may expect to live indefinitely after its parent + # exited. + wait_on_shutdown=(not self.detached), + ) + self._reaper.reap() + + def _on_broker_shutdown(self): + """ + Respond to broker.shutdown() being called by failing the connection + attempt. + """ + self._fail_connection(CancelledError(BROKER_SHUTDOWN_MSG)) - LOG.debug('%r.connect(): pid:%r stdin:%r, stdout:%r, diag:%r', - self, self.pid, self.receive_side.fd, self.transmit_side.fd, - self.diag_stream and self.diag_stream.receive_side.fd) + def stream_factory(self): + return self.stream_protocol_class.build_stream( + broker=self._router.broker, + ) + + def stderr_stream_factory(self): + return self.diag_protocol_class.build_stream() + + def _setup_stdio_stream(self): + stream = self.stream_factory() + stream.conn = self + stream.name = self.options.name or self._get_name() + stream.accept(self.proc.stdout, self.proc.stdin) + + mitogen.core.listen(stream, 'disconnect', self.on_stdio_disconnect) + self._router.broker.start_receive(stream) + return stream + + def _setup_stderr_stream(self): + stream = self.stderr_stream_factory() + stream.conn = self + stream.name = self.options.name or self._get_name() + stream.accept(self.proc.stderr, self.proc.stderr) + + mitogen.core.listen(stream, 'disconnect', self.on_stderr_disconnect) + self._router.broker.start_receive(stream) + return stream + + def _on_timer_expired(self): + self._fail_connection( + mitogen.core.TimeoutError( + 'Failed to setup connection after %.2f seconds', + self.options.connect_timeout, + ) + ) + + def _async_connect(self): + LOG.debug('creating connection to context %d using %s', + self.context.context_id, self.__class__.__module__) + mitogen.core.listen(self._router.broker, 'shutdown', + self._on_broker_shutdown) + self._timer = self._router.broker.timers.schedule( + when=self.options.connect_deadline, + func=self._on_timer_expired, + ) try: - self._connect_bootstrap() - except EofError: - self.on_disconnect(self._router.broker) - e = sys.exc_info()[1] - self._adorn_eof_error(e) - raise + self.proc = self.start_child() except Exception: - self.on_disconnect(self._router.broker) - self._reap_child() - raise + self._fail_connection(sys.exc_info()[1]) + return - #: Sentinel value emitted by the first stage to indicate it is ready to - #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have - #: length of at least `max(len('password'), len('debug1:'))` - EC0_MARKER = mitogen.core.b('MITO000\n') - EC1_MARKER = mitogen.core.b('MITO001\n') + LOG.debug('child for %r started: pid:%r stdin:%r stdout:%r stderr:%r', + self, self.proc.pid, + self.proc.stdin.fileno(), + self.proc.stdout.fileno(), + self.proc.stderr and self.proc.stderr.fileno()) - def _ec0_received(self): - LOG.debug('%r._ec0_received()', self) - write_all(self.transmit_side.fd, self.get_preamble()) - discard_until(self.receive_side.fd, self.EC1_MARKER, - self.connect_deadline) - if self.diag_stream: - self._router.broker.start_receive(self.diag_stream) + self.stdio_stream = self._setup_stdio_stream() + if self.context.name is None: + self.context.name = self.stdio_stream.name + self.proc.name = self.stdio_stream.name + if self.proc.stderr: + self.stderr_stream = self._setup_stderr_stream() - def _connect_bootstrap(self): - discard_until(self.receive_side.fd, self.EC0_MARKER, - self.connect_deadline) - self._ec0_received() + def connect(self, context): + self.context = context + self.latch = mitogen.core.Latch() + self._router.broker.defer(self._async_connect) + self.latch.get() + if self.exception: + raise self.exception class ChildIdAllocator(object): + """ + Allocate new context IDs from a block of unique context IDs allocated by + the master process. + """ def __init__(self, router): self.router = router self.lock = threading.Lock() self.it = iter(xrange(0)) def allocate(self): + """ + Allocate an ID, requesting a fresh block from the master if the + existing block is exhausted. + + :returns: + The new context ID. + + .. warning:: + + This method is not safe to call from the :class:`Broker` thread, as + it may block on IO of its own. + """ self.lock.acquire() try: for id_ in self.it: return id_ - master = mitogen.core.Context(self.router, 0) + master = self.router.context_by_id(0) start, end = master.send_await( mitogen.core.Message(dst_id=0, handle=mitogen.core.ALLOCATE_ID) ) @@ -1570,7 +1821,7 @@ class CallChain(object): socket.gethostname(), os.getpid(), thread.get_ident(), - int(1e6 * time.time()), + int(1e6 * mitogen.core.now()), ) def __repr__(self): @@ -1643,7 +1894,9 @@ class CallChain(object): pipelining is disabled, the exception will be logged to the target context's logging framework. """ - LOG.debug('%r.call_no_reply(): %r', self, CallSpec(fn, args, kwargs)) + LOG.debug('starting no-reply function call to %r: %r', + self.context.name or self.context.context_id, + CallSpec(fn, args, kwargs)) self.context.send(self.make_msg(fn, *args, **kwargs)) def call_async(self, fn, *args, **kwargs): @@ -1699,7 +1952,9 @@ class CallChain(object): contexts and consumed as they complete using :class:`mitogen.select.Select`. """ - LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs)) + LOG.debug('starting function call to %s: %r', + self.context.name or self.context.context_id, + CallSpec(fn, args, kwargs)) return self.context.send_async(self.make_msg(fn, *args, **kwargs)) def call(self, fn, *args, **kwargs): @@ -1739,9 +1994,11 @@ class Context(mitogen.core.Context): return not (self == other) def __eq__(self, other): - return (isinstance(other, mitogen.core.Context) and - (other.context_id == self.context_id) and - (other.router == self.router)) + return ( + isinstance(other, mitogen.core.Context) and + (other.context_id == self.context_id) and + (other.router == self.router) + ) def __hash__(self): return hash((self.router, self.context_id)) @@ -1819,15 +2076,16 @@ class RouteMonitor(object): RouteMonitor lives entirely on the broker thread, so its data requires no locking. - :param Router router: + :param mitogen.master.Router router: Router to install handlers on. - :param Context parent: + :param mitogen.core.Context parent: :data:`None` in the master process, or reference to the parent context we should propagate route updates towards. """ def __init__(self, router, parent=None): self.router = router self.parent = parent + self._log = logging.getLogger('mitogen.route_monitor') #: Mapping of Stream instance to integer context IDs reachable via the #: stream; used to cleanup routes during disconnection. self._routes_by_stream = {} @@ -1869,11 +2127,11 @@ class RouteMonitor(object): data = str(target_id) if name: data = '%s:%s' % (target_id, name) - stream.send( + stream.protocol.send( mitogen.core.Message( handle=handle, data=data.encode('utf-8'), - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, ) ) @@ -1907,20 +2165,20 @@ class RouteMonitor(object): ID of the connecting or disconnecting context. """ for stream in self.router.get_streams(): - if target_id in stream.egress_ids and ( + if target_id in stream.protocol.egress_ids and ( (self.parent is None) or - (self.parent.context_id != stream.remote_id) + (self.parent.context_id != stream.protocol.remote_id) ): self._send_one(stream, mitogen.core.DEL_ROUTE, target_id, None) def notice_stream(self, stream): """ When this parent is responsible for a new directly connected child - stream, we're also responsible for broadcasting DEL_ROUTE upstream - if/when that child disconnects. + stream, we're also responsible for broadcasting + :data:`mitogen.core.DEL_ROUTE` upstream when that child disconnects. """ - self._routes_by_stream[stream] = set([stream.remote_id]) - self._propagate_up(mitogen.core.ADD_ROUTE, stream.remote_id, + self._routes_by_stream[stream] = set([stream.protocol.remote_id]) + self._propagate_up(mitogen.core.ADD_ROUTE, stream.protocol.remote_id, stream.name) mitogen.core.listen( obj=stream, @@ -1948,8 +2206,8 @@ class RouteMonitor(object): if routes is None: return - LOG.debug('%r: %r is gone; propagating DEL_ROUTE for %r', - self, stream, routes) + self._log.debug('stream %s is gone; propagating DEL_ROUTE for %r', + stream.name, routes) for target_id in routes: self.router.del_route(target_id) self._propagate_up(mitogen.core.DEL_ROUTE, target_id) @@ -1974,13 +2232,13 @@ class RouteMonitor(object): self.router.context_by_id(target_id).name = target_name stream = self.router.stream_by_id(msg.auth_id) current = self.router.stream_by_id(target_id) - if current and current.remote_id != mitogen.parent_id: - LOG.error('Cannot add duplicate route to %r via %r, ' - 'already have existing route via %r', - target_id, stream, current) + if current and current.protocol.remote_id != mitogen.parent_id: + self._log.error('Cannot add duplicate route to %r via %r, ' + 'already have existing route via %r', + target_id, stream, current) return - LOG.debug('Adding route to %d via %r', target_id, stream) + self._log.debug('Adding route to %d via %r', target_id, stream) self._routes_by_stream[stream].add(target_id) self.router.add_route(target_id, stream) self._propagate_up(mitogen.core.ADD_ROUTE, target_id, target_name) @@ -2002,22 +2260,22 @@ class RouteMonitor(object): stream = self.router.stream_by_id(msg.auth_id) if registered_stream != stream: - LOG.error('%r: received DEL_ROUTE for %d from %r, expected %r', - self, target_id, stream, registered_stream) + self._log.error('received DEL_ROUTE for %d from %r, expected %r', + target_id, stream, registered_stream) return context = self.router.context_by_id(target_id, create=False) if context: - LOG.debug('%r: firing local disconnect for %r', self, context) + self._log.debug('firing local disconnect signal for %r', context) mitogen.core.fire(context, 'disconnect') - LOG.debug('%r: deleting route to %d via %r', self, target_id, stream) + self._log.debug('deleting route to %d via %r', target_id, stream) routes = self._routes_by_stream.get(stream) if routes: routes.discard(target_id) self.router.del_route(target_id) - if stream.remote_id != mitogen.parent_id: + if stream.protocol.remote_id != mitogen.parent_id: self._propagate_up(mitogen.core.DEL_ROUTE, target_id) self._propagate_down(mitogen.core.DEL_ROUTE, target_id) @@ -2033,7 +2291,7 @@ class Router(mitogen.core.Router): route_monitor = None def upgrade(self, importer, parent): - LOG.debug('%r.upgrade()', self) + LOG.debug('upgrading %r with capabilities to start new children', self) self.id_allocator = ChildIdAllocator(router=self) self.responder = ModuleForwarder( router=self, @@ -2051,16 +2309,17 @@ class Router(mitogen.core.Router): if msg.is_dead: return stream = self.stream_by_id(msg.src_id) - if stream.remote_id != msg.src_id or stream.detached: + if stream.protocol.remote_id != msg.src_id or stream.conn.detached: LOG.warning('bad DETACHING received on %r: %r', stream, msg) return LOG.debug('%r: marking as detached', stream) - stream.detached = True + stream.conn.detached = True msg.reply(None) def get_streams(self): """ - Return a snapshot of all streams in existence at time of call. + Return an atomic snapshot of all streams in existence at time of call. + This is safe to call from any thread. """ self._write_lock.acquire() try: @@ -2068,17 +2327,42 @@ class Router(mitogen.core.Router): finally: self._write_lock.release() + def disconnect(self, context): + """ + Disconnect a context and forget its stream, assuming the context is + directly connected. + """ + stream = self.stream_by_id(context) + if stream is None or stream.protocol.remote_id != context.context_id: + return + + l = mitogen.core.Latch() + mitogen.core.listen(stream, 'disconnect', l.put) + def disconnect(): + LOG.debug('Starting disconnect of %r', stream) + stream.on_disconnect(self.broker) + self.broker.defer(disconnect) + l.get() + def add_route(self, target_id, stream): """ - Arrange for messages whose `dst_id` is `target_id` to be forwarded on - the directly connected stream for `via_id`. This method is called - automatically in response to :data:`mitogen.core.ADD_ROUTE` messages, - but remains public while the design has not yet settled, and situations - may arise where routing is not fully automatic. + Arrange for messages whose `dst_id` is `target_id` to be forwarded on a + directly connected :class:`Stream`. Safe to call from any thread. + + This is called automatically by :class:`RouteMonitor` in response to + :data:`mitogen.core.ADD_ROUTE` messages, but remains public while the + design has not yet settled, and situations may arise where routing is + not fully automatic. + + :param int target_id: + Target context ID to add a route for. + :param mitogen.core.Stream stream: + Stream over which messages to the target should be routed. """ - LOG.debug('%r.add_route(%r, %r)', self, target_id, stream) + LOG.debug('%r: adding route to context %r via %r', + self, target_id, stream) assert isinstance(target_id, int) - assert isinstance(stream, Stream) + assert isinstance(stream, mitogen.core.Stream) self._write_lock.acquire() try: @@ -2087,7 +2371,20 @@ class Router(mitogen.core.Router): self._write_lock.release() def del_route(self, target_id): - LOG.debug('%r.del_route(%r)', self, target_id) + """ + Delete any route that exists for `target_id`. It is not an error to + delete a route that does not currently exist. Safe to call from any + thread. + + This is called automatically by :class:`RouteMonitor` in response to + :data:`mitogen.core.DEL_ROUTE` messages, but remains public while the + design has not yet settled, and situations may arise where routing is + not fully automatic. + + :param int target_id: + Target context ID to delete route for. + """ + LOG.debug('%r: deleting route to %r', self, target_id) # DEL_ROUTE may be sent by a parent if it knows this context sent # messages to a peer that has now disconnected, to let us raise # 'disconnect' event on the appropriate Context instance. In that case, @@ -2114,35 +2411,36 @@ class Router(mitogen.core.Router): connection_timeout_msg = u"Connection timed out." - def _connect(self, klass, name=None, **kwargs): + def _connect(self, klass, **kwargs): context_id = self.allocate_id() context = self.context_class(self, context_id) + context.name = kwargs.get('name') + kwargs['old_router'] = self kwargs['max_message_size'] = self.max_message_size - stream = klass(self, context_id, **kwargs) - if name is not None: - stream.name = name + conn = klass(klass.options_class(**kwargs), self) try: - stream.connect() + conn.connect(context=context) except mitogen.core.TimeoutError: raise mitogen.core.StreamError(self.connection_timeout_msg) - context.name = stream.name - self.route_monitor.notice_stream(stream) - self.register(context, stream) + return context def connect(self, method_name, name=None, **kwargs): - klass = stream_by_method_name(method_name) + if name: + name = mitogen.core.to_text(name) + + klass = get_connection_class(method_name) kwargs.setdefault(u'debug', self.debug) kwargs.setdefault(u'profiling', self.profiling) kwargs.setdefault(u'unidirectional', self.unidirectional) + kwargs.setdefault(u'name', name) via = kwargs.pop(u'via', None) if via is not None: - return self.proxy_connect(via, method_name, name=name, - **mitogen.core.Kwargs(kwargs)) - return self._connect(klass, name=name, - **mitogen.core.Kwargs(kwargs)) + return self.proxy_connect(via, method_name, + **mitogen.core.Kwargs(kwargs)) + return self._connect(klass, **mitogen.core.Kwargs(kwargs)) def proxy_connect(self, via_context, method_name, name=None, **kwargs): resp = via_context.call(_proxy_connect, @@ -2163,6 +2461,9 @@ class Router(mitogen.core.Router): self._write_lock.release() return context + def buildah(self, **kwargs): + return self.connect(u'buildah', **kwargs) + def doas(self, **kwargs): return self.connect(u'doas', **kwargs) @@ -2200,49 +2501,187 @@ class Router(mitogen.core.Router): return self.connect(u'ssh', **kwargs) -class ProcessMonitor(object): - """ - Install a :data:`signal.SIGCHLD` handler that generates callbacks when a - specific child process has exitted. This class is obsolete, do not use. - """ - def __init__(self): - # pid -> callback() - self.callback_by_pid = {} - signal.signal(signal.SIGCHLD, self._on_sigchld) +class Reaper(object): + """ + Asynchronous logic for reaping :class:`Process` objects. This is necessary + to prevent uncontrolled buildup of zombie processes in long-lived parents + that will eventually reach an OS limit, preventing creation of new threads + and processes, and to log the exit status of the child in the case of an + error. + + To avoid modifying process-global state such as with + :func:`signal.set_wakeup_fd` or installing a :data:`signal.SIGCHLD` handler + that might interfere with the user's ability to use those facilities, + Reaper polls for exit with backoff using timers installed on an associated + :class:`Broker`. + + :param mitogen.core.Broker broker: + The :class:`Broker` on which to install timers + :param mitogen.parent.Process proc: + The process to reap. + :param bool kill: + If :data:`True`, send ``SIGTERM`` and ``SIGKILL`` to the process. + :param bool wait_on_shutdown: + If :data:`True`, delay :class:`Broker` shutdown if child has not yet + exited. If :data:`False` simply forget the child. + """ + #: :class:`Timer` that invokes :meth:`reap` after some polling delay. + _timer = None + + def __init__(self, broker, proc, kill, wait_on_shutdown): + self.broker = broker + self.proc = proc + self.kill = kill + self.wait_on_shutdown = wait_on_shutdown + self._tries = 0 + + def _signal_child(self, signum): + # For processes like sudo we cannot actually send sudo a signal, + # because it is setuid, so this is best-effort only. + LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum]) + try: + os.kill(self.proc.pid, signum) + except OSError: + e = sys.exc_info()[1] + if e.args[0] != errno.EPERM: + raise + + def _calc_delay(self, count): + """ + Calculate a poll delay given `count` attempts have already been made. + These constants have no principle, they just produce rapid but still + relatively conservative retries. + """ + delay = 0.05 + for _ in xrange(count): + delay *= 1.72 + return delay + + def _on_broker_shutdown(self): + """ + Respond to :class:`Broker` shutdown by cancelling the reap timer if + :attr:`Router.await_children_at_shutdown` is disabled. Otherwise + shutdown is delayed for up to :attr:`Broker.shutdown_timeout` for + subprocesses may have no intention of exiting any time soon. + """ + if not self.wait_on_shutdown: + self._timer.cancel() + + def _install_timer(self, delay): + new = self._timer is None + self._timer = self.broker.timers.schedule( + when=mitogen.core.now() + delay, + func=self.reap, + ) + if new: + mitogen.core.listen(self.broker, 'shutdown', + self._on_broker_shutdown) - def _on_sigchld(self, _signum, _frame): - for pid, callback in self.callback_by_pid.items(): - pid, status = os.waitpid(pid, os.WNOHANG) - if pid: - callback(status) - del self.callback_by_pid[pid] + def _remove_timer(self): + if self._timer and self._timer.active: + self._timer.cancel() + mitogen.core.unlisten(self.broker, 'shutdown', + self._on_broker_shutdown) - def add(self, pid, callback): + def reap(self): """ - Add a callback function to be notified of the exit status of a process. + Reap the child process during disconnection. + """ + status = self.proc.poll() + if status is not None: + LOG.debug('%r: %s', self.proc, returncode_to_str(status)) + mitogen.core.fire(self.proc, 'exit') + self._remove_timer() + return - :param int pid: - Process ID to be notified of. + self._tries += 1 + if self._tries > 20: + LOG.warning('%r: child will not exit, giving up', self) + self._remove_timer() + return + + delay = self._calc_delay(self._tries - 1) + LOG.debug('%r still running after IO disconnect, recheck in %.03fs', + self.proc, delay) + self._install_timer(delay) - :param callback: - Function invoked as `callback(status)`, where `status` is the raw - exit status of the child process. + if not self.kill: + pass + elif self._tries == 2: + self._signal_child(signal.SIGTERM) + elif self._tries == 6: # roughly 4 seconds + self._signal_child(signal.SIGKILL) + + +class Process(object): + """ + Process objects provide a uniform interface to the :mod:`subprocess` and + :mod:`mitogen.fork`. This class is extended by :class:`PopenProcess` and + :class:`mitogen.fork.Process`. + + :param int pid: + The process ID. + :param file stdin: + File object attached to standard input. + :param file stdout: + File object attached to standard output. + :param file stderr: + File object attached to standard error, or :data:`None`. + """ + #: Name of the process used in logs. Set to the stream/context name by + #: :class:`Connection`. + name = None + + def __init__(self, pid, stdin, stdout, stderr=None): + #: The process ID. + self.pid = pid + #: File object attached to standard input. + self.stdin = stdin + #: File object attached to standard output. + self.stdout = stdout + #: File object attached to standard error. + self.stderr = stderr + + def __repr__(self): + return '%s %s pid %d' % ( + type(self).__name__, + self.name, + self.pid, + ) + + def poll(self): """ - self.callback_by_pid[pid] = callback + Fetch the child process exit status, or :data:`None` if it is still + running. This should be overridden by subclasses. - _instance = None + :returns: + Exit status in the style of the :attr:`subprocess.Popen.returncode` + attribute, i.e. with signals represented by a negative integer. + """ + raise NotImplementedError() - @classmethod - def instance(cls): - if cls._instance is None: - cls._instance = cls() - return cls._instance + +class PopenProcess(Process): + """ + :class:`Process` subclass wrapping a :class:`subprocess.Popen` object. + + :param subprocess.Popen proc: + The subprocess. + """ + def __init__(self, proc, stdin, stdout, stderr=None): + super(PopenProcess, self).__init__(proc.pid, stdin, stdout, stderr) + #: The subprocess. + self.proc = proc + + def poll(self): + return self.proc.poll() class ModuleForwarder(object): """ - Respond to GET_MODULE requests in a slave by forwarding the request to our - parent context, or satisfying the request from our local Importer cache. + Respond to :data:`mitogen.core.GET_MODULE` requests in a child by + forwarding the request to our parent context, or satisfying the request + from our local Importer cache. """ def __init__(self, router, parent_context, importer): self.router = router @@ -2262,7 +2701,7 @@ class ModuleForwarder(object): ) def __repr__(self): - return 'ModuleForwarder(%r)' % (self.router,) + return 'ModuleForwarder' def _on_forward_module(self, msg): if msg.is_dead: @@ -2272,38 +2711,38 @@ class ModuleForwarder(object): fullname = mitogen.core.to_text(fullname) context_id = int(context_id_s) stream = self.router.stream_by_id(context_id) - if stream.remote_id == mitogen.parent_id: + if stream.protocol.remote_id == mitogen.parent_id: LOG.error('%r: dropping FORWARD_MODULE(%d, %r): no route to child', self, context_id, fullname) return - if fullname in stream.sent_modules: + if fullname in stream.protocol.sent_modules: return LOG.debug('%r._on_forward_module() sending %r to %r via %r', - self, fullname, context_id, stream.remote_id) + self, fullname, context_id, stream.protocol.remote_id) self._send_module_and_related(stream, fullname) - if stream.remote_id != context_id: - stream._send( + if stream.protocol.remote_id != context_id: + stream.protocol._send( mitogen.core.Message( data=msg.data, handle=mitogen.core.FORWARD_MODULE, - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, ) ) def _on_get_module(self, msg): - LOG.debug('%r._on_get_module(%r)', self, msg) if msg.is_dead: return fullname = msg.data.decode('utf-8') + LOG.debug('%r: %s requested by context %d', self, fullname, msg.src_id) callback = lambda: self._on_cache_callback(msg, fullname) self.importer._request_module(fullname, callback) def _on_cache_callback(self, msg, fullname): - LOG.debug('%r._on_get_module(): sending %r', self, fullname) stream = self.router.stream_by_id(msg.src_id) + LOG.debug('%r: sending %s to %r', self, fullname, stream) self._send_module_and_related(stream, fullname) def _send_module_and_related(self, stream, fullname): @@ -2313,18 +2752,18 @@ class ModuleForwarder(object): if rtup: self._send_one_module(stream, rtup) else: - LOG.debug('%r._send_module_and_related(%r): absent: %r', - self, fullname, related) + LOG.debug('%r: %s not in cache (for %s)', + self, related, fullname) self._send_one_module(stream, tup) def _send_one_module(self, stream, tup): - if tup[0] not in stream.sent_modules: - stream.sent_modules.add(tup[0]) + if tup[0] not in stream.protocol.sent_modules: + stream.protocol.sent_modules.add(tup[0]) self.router._async_route( mitogen.core.Message.pickled( tup, - dst_id=stream.remote_id, + dst_id=stream.protocol.remote_id, handle=mitogen.core.LOAD_MODULE, ) ) diff --git a/mitogen/profiler.py b/mitogen/profiler.py index 74bbdb23..bbf6086a 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -28,7 +28,8 @@ # !mitogen: minify_safe -"""mitogen.profiler +""" +mitogen.profiler Record and report cProfile statistics from a run. Creates one aggregated output file, one aggregate containing only workers, and one for the top-level process. @@ -56,28 +57,25 @@ Example: from __future__ import print_function import os import pstats -import cProfile import shutil import subprocess import sys import tempfile import time -import mitogen.core - def try_merge(stats, path): try: stats.add(path) return True except Exception as e: - print('Failed. Race? Will retry. %s' % (e,)) + print('%s failed. Will retry. %s' % (path, e)) return False def merge_stats(outpath, inpaths): first, rest = inpaths[0], inpaths[1:] - for x in range(5): + for x in range(1): try: stats = pstats.Stats(first) except EOFError: @@ -152,7 +150,7 @@ def do_stat(tmpdir, sort, *args): def main(): if len(sys.argv) < 2 or sys.argv[1] not in ('record', 'report', 'stat'): - sys.stderr.write(__doc__) + sys.stderr.write(__doc__.lstrip()) sys.exit(1) func = globals()['do_' + sys.argv[1]] diff --git a/mitogen/select.py b/mitogen/select.py index 51aebc22..3875042b 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -57,9 +57,7 @@ class Select(object): If `oneshot` is :data:`True`, then remove each receiver as it yields a result; since :meth:`__iter__` terminates once the final receiver is - removed, this makes it convenient to respond to calls made in parallel: - - .. code-block:: python + removed, this makes it convenient to respond to calls made in parallel:: total = 0 recvs = [c.call_async(long_running_operation) for c in contexts] @@ -98,7 +96,7 @@ class Select(object): for msg in mitogen.select.Select(selects): print(msg.unpickle()) - :class:`Select` may be used to mix inter-thread and inter-process IO: + :class:`Select` may be used to mix inter-thread and inter-process IO:: latch = mitogen.core.Latch() start_thread(latch) @@ -124,9 +122,10 @@ class Select(object): @classmethod def all(cls, receivers): """ - Take an iterable of receivers and retrieve a :class:`Message` from - each, returning the result of calling `msg.unpickle()` on each in turn. - Results are returned in the order they arrived. + Take an iterable of receivers and retrieve a :class:`Message + <mitogen.core.Message>` from each, returning the result of calling + :meth:`Message.unpickle() <mitogen.core.Message.unpickle>` on each in + turn. Results are returned in the order they arrived. This is sugar for handling batch :meth:`Context.call_async <mitogen.parent.Context.call_async>` invocations: @@ -226,8 +225,15 @@ class Select(object): raise Error(self.owned_msg) recv.notify = self._put - # Avoid race by polling once after installation. - if not recv.empty(): + # After installing the notify function, _put() will potentially begin + # receiving calls from other threads immediately, but not for items + # they already had buffered. For those we call _put(), possibly + # duplicating the effect of other _put() being made concurrently, such + # that the Select ends up with more items in its buffer than exist in + # the underlying receivers. We handle the possibility of receivers + # marked notified yet empty inside Select.get(), so this should be + # robust. + for _ in range(recv.size()): self._put(recv) not_present_msg = 'Instance is not a member of this Select' @@ -261,18 +267,26 @@ class Select(object): self.remove(recv) self._latch.close() - def empty(self): + def size(self): + """ + Return the number of items currently buffered. + + As with :class:`Queue.Queue`, `0` may be returned even though a + subsequent call to :meth:`get` will succeed, since a message may be + posted at any moment between :meth:`size` and :meth:`get`. + + As with :class:`Queue.Queue`, `>0` may be returned even though a + subsequent call to :meth:`get` will block, since another waiting thread + may be woken at any moment between :meth:`size` and :meth:`get`. """ - Return :data:`True` if calling :meth:`get` would block. + return sum(recv.size() for recv in self._receivers) - As with :class:`Queue.Queue`, :data:`True` may be returned even though - a subsequent call to :meth:`get` will succeed, since a message may be - posted at any moment between :meth:`empty` and :meth:`get`. + def empty(self): + """ + Return `size() == 0`. - :meth:`empty` may return :data:`False` even when :meth:`get` would - block if another thread has drained a receiver added to this select. - This can be avoided by only consuming each receiver from a single - thread. + .. deprecated:: 0.2.8 + Use :meth:`size` instead. """ return self._latch.empty() @@ -329,5 +343,6 @@ class Select(object): # A receiver may have been queued with no result if another # thread drained it before we woke up, or because another # thread drained it between add() calling recv.empty() and - # self._put(). In this case just sleep again. + # self._put(), or because Select.add() caused duplicate _put() + # calls. In this case simply retry. continue diff --git a/mitogen/service.py b/mitogen/service.py index 302e81ab..6bd64eb0 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -29,6 +29,7 @@ # !mitogen: minify_safe import grp +import logging import os import os.path import pprint @@ -36,12 +37,10 @@ import pwd import stat import sys import threading -import time import mitogen.core import mitogen.select from mitogen.core import b -from mitogen.core import LOG from mitogen.core import str_rpartition try: @@ -54,7 +53,8 @@ except NameError: return True -DEFAULT_POOL_SIZE = 16 +LOG = logging.getLogger(__name__) + _pool = None _pool_pid = None #: Serialize pool construction. @@ -77,19 +77,51 @@ else: def get_or_create_pool(size=None, router=None): global _pool global _pool_pid - _pool_lock.acquire() - try: - if _pool_pid != os.getpid(): - _pool = Pool(router, [], size=size or DEFAULT_POOL_SIZE, - overwrite=True) - # In case of Broker shutdown crash, Pool can cause 'zombie' - # processes. - mitogen.core.listen(router.broker, 'shutdown', - lambda: _pool.stop(join=False)) - _pool_pid = os.getpid() - return _pool - finally: - _pool_lock.release() + + my_pid = os.getpid() + if _pool is None or _pool.closed or my_pid != _pool_pid: + # Avoid acquiring heavily contended lock if possible. + _pool_lock.acquire() + try: + if _pool_pid != my_pid: + _pool = Pool( + router, + services=[], + size=size or 2, + overwrite=True, + recv=mitogen.core.Dispatcher._service_recv, + ) + # In case of Broker shutdown crash, Pool can cause 'zombie' + # processes. + mitogen.core.listen(router.broker, 'shutdown', + lambda: _pool.stop(join=True)) + _pool_pid = os.getpid() + finally: + _pool_lock.release() + + return _pool + + +def get_thread_name(): + return threading.currentThread().getName() + + +def call(service_name, method_name, call_context=None, **kwargs): + """ + Call a service registered with this pool, using the calling thread as a + host. + """ + if isinstance(service_name, mitogen.core.BytesType): + service_name = service_name.encode('utf-8') + elif not isinstance(service_name, mitogen.core.UnicodeType): + service_name = service_name.name() # Service.name() + + if call_context: + return call_context.call_service(service_name, method_name, **kwargs) + else: + pool = get_or_create_pool() + invoker = pool.get_invoker(service_name, msg=None) + return getattr(invoker.service, method_name)(**kwargs) def validate_arg_spec(spec, args): @@ -239,12 +271,13 @@ class Invoker(object): if not policies: raise mitogen.core.CallError('Method has no policies set.') - if not all(p.is_authorized(self.service, msg) for p in policies): - raise mitogen.core.CallError( - self.unauthorized_msg, - method_name, - self.service.name() - ) + if msg is not None: + if not all(p.is_authorized(self.service, msg) for p in policies): + raise mitogen.core.CallError( + self.unauthorized_msg, + method_name, + self.service.name() + ) required = getattr(method, 'mitogen_service__arg_spec', {}) validate_arg_spec(required, kwargs) @@ -264,7 +297,7 @@ class Invoker(object): except Exception: if no_reply: LOG.exception('While calling no-reply method %s.%s', - type(self.service).__name__, + self.service.name(), func_name(method)) else: raise @@ -445,13 +478,19 @@ class Pool(object): program's configuration or its input data. :param mitogen.core.Router router: - Router to listen for ``CALL_SERVICE`` messages on. + :class:`mitogen.core.Router` to listen for + :data:`mitogen.core.CALL_SERVICE` messages. :param list services: Initial list of services to register. + :param mitogen.core.Receiver recv: + :data:`mitogen.core.CALL_SERVICE` receiver to reuse. This is used by + :func:`get_or_create_pool` to hand off a queue of messages from the + Dispatcher stub handler while avoiding a race. """ activator_class = Activator - def __init__(self, router, services=(), size=1, overwrite=False): + def __init__(self, router, services=(), size=1, overwrite=False, + recv=None): self.router = router self._activator = self.activator_class() self._ipc_latch = mitogen.core.Latch() @@ -472,12 +511,22 @@ class Pool(object): } self._invoker_by_name = {} + if recv is not None: + # When inheriting from mitogen.core.Dispatcher, we must remove its + # stub notification function before adding it to our Select. We + # always overwrite this receiver since the standard service.Pool + # handler policy differs from the one inherited from + # core.Dispatcher. + recv.notify = None + self._select.add(recv) + self._func_by_source[recv] = self._on_service_call + for service in services: self.add(service) self._py_24_25_compat() self._threads = [] for x in range(size): - name = 'mitogen.service.Pool.%x.worker-%d' % (id(self), x,) + name = 'mitogen.Pool.%04x.%d' % (id(self) & 0xffff, x,) thread = threading.Thread( name=name, target=mitogen.core._profile_hook, @@ -485,7 +534,6 @@ class Pool(object): ) thread.start() self._threads.append(thread) - LOG.debug('%r: initialized', self) def _py_24_25_compat(self): @@ -524,15 +572,18 @@ class Pool(object): invoker.service.on_shutdown() def get_invoker(self, name, msg): - self._lock.acquire() - try: - invoker = self._invoker_by_name.get(name) - if not invoker: - service = self._activator.activate(self, name, msg) - invoker = service.invoker_class(service=service) - self._invoker_by_name[name] = invoker - finally: - self._lock.release() + invoker = self._invoker_by_name.get(name) + if invoker is None: + # Avoid acquiring lock if possible. + self._lock.acquire() + try: + invoker = self._invoker_by_name.get(name) + if not invoker: + service = self._activator.activate(self, name, msg) + invoker = service.invoker_class(service=service) + self._invoker_by_name[name] = invoker + finally: + self._lock.release() return invoker @@ -582,9 +633,12 @@ class Pool(object): while not self.closed: try: event = self._select.get_event() - except (mitogen.core.ChannelError, mitogen.core.LatchError): - e = sys.exc_info()[1] - LOG.debug('%r: channel or latch closed, exitting: %s', self, e) + except mitogen.core.LatchError: + LOG.debug('thread %s exiting gracefully', get_thread_name()) + return + except mitogen.core.ChannelError: + LOG.debug('thread %s exiting with error: %s', + get_thread_name(), sys.exc_info()[1]) return func = self._func_by_source[event.source] @@ -597,16 +651,14 @@ class Pool(object): try: self._worker_run() except Exception: - th = threading.currentThread() - LOG.exception('%r: worker %r crashed', self, th.getName()) + LOG.exception('%r: worker %r crashed', self, get_thread_name()) raise def __repr__(self): - th = threading.currentThread() - return 'mitogen.service.Pool(%#x, size=%d, th=%r)' % ( - id(self), + return 'Pool(%04x, size=%d, th=%r)' % ( + id(self) & 0xffff, len(self._threads), - th.getName(), + get_thread_name(), ) @@ -625,7 +677,7 @@ class PushFileService(Service): """ Push-based file service. Files are delivered and cached in RAM, sent recursively from parent to child. A child that requests a file via - :meth:`get` will block until it has ben delivered by a parent. + :meth:`get` will block until it has been delivered by a parent. This service will eventually be merged into FileService. """ @@ -658,10 +710,12 @@ class PushFileService(Service): def _forward(self, context, path): stream = self.router.stream_by_id(context.context_id) - child = mitogen.core.Context(self.router, stream.remote_id) + child = self.router.context_by_id(stream.protocol.remote_id) sent = self._sent_by_stream.setdefault(stream, set()) if path in sent: if child.context_id != context.context_id: + LOG.debug('requesting %s forward small file to %s: %s', + child, context, path) child.call_service_async( service_name=self.name(), method_name='forward', @@ -669,6 +723,8 @@ class PushFileService(Service): context=context ).close() else: + LOG.debug('requesting %s cache and forward small file to %s: %s', + child, context, path) child.call_service_async( service_name=self.name(), method_name='store_and_forward', @@ -691,7 +747,7 @@ class PushFileService(Service): """ for path in paths: self.propagate_to(context, mitogen.core.to_text(path)) - self.router.responder.forward_modules(context, modules) + #self.router.responder.forward_modules(context, modules) TODO @expose(policy=AllowParents()) @arg_spec({ @@ -699,8 +755,8 @@ class PushFileService(Service): 'path': mitogen.core.FsPathTypes, }) def propagate_to(self, context, path): - LOG.debug('%r.propagate_to(%r, %r)', self, context, path) if path not in self._cache: + LOG.debug('caching small file %s', path) fp = open(path, 'rb') try: self._cache[path] = mitogen.core.Blob(fp.read()) @@ -718,7 +774,7 @@ class PushFileService(Service): def store_and_forward(self, path, data, context): LOG.debug('%r.store_and_forward(%r, %r, %r) %r', self, path, data, context, - threading.currentThread().getName()) + get_thread_name()) self._lock.acquire() try: self._cache[path] = data @@ -891,7 +947,7 @@ class FileService(Service): # The IO loop pumps 128KiB chunks. An ideal message is a multiple of this, # odd-sized messages waste one tiny write() per message on the trailer. # Therefore subtract 10 bytes pickle overhead + 24 bytes header. - IO_SIZE = mitogen.core.CHUNK_SIZE - (mitogen.core.Stream.HEADER_LEN + ( + IO_SIZE = mitogen.core.CHUNK_SIZE - (mitogen.core.Message.HEADER_LEN + ( len( mitogen.core.Message.pickled( mitogen.core.Blob(b(' ') * mitogen.core.CHUNK_SIZE) @@ -965,7 +1021,11 @@ class FileService(Service): :raises Error: Unregistered path, or Sender did not match requestee context. """ - if path not in self._paths and not self._prefix_is_authorized(path): + if ( + (path not in self._paths) and + (not self._prefix_is_authorized(path)) and + (not mitogen.core._has_parent_authority(msg.auth_id)) + ): msg.reply(mitogen.core.CallError( Error(self.unregistered_msg % (path,)) )) @@ -1047,7 +1107,7 @@ class FileService(Service): :meth:`fetch`. """ LOG.debug('get_file(): fetching %r from %r', path, context) - t0 = time.time() + t0 = mitogen.core.now() recv = mitogen.core.Receiver(router=context.router) metadata = context.call_service( service_name=cls.name(), @@ -1081,5 +1141,6 @@ class FileService(Service): path, metadata['size'], received_bytes) LOG.debug('target.get_file(): fetched %d bytes of %r from %r in %dms', - metadata['size'], path, context, 1000 * (time.time() - t0)) + metadata['size'], path, context, + 1000 * (mitogen.core.now() - t0)) return ok, metadata diff --git a/mitogen/setns.py b/mitogen/setns.py index b1d69783..46a50301 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -116,9 +116,15 @@ def get_machinectl_pid(path, name): raise Error("could not find PID from machinectl output.\n%s", output) -class Stream(mitogen.parent.Stream): - child_is_immediate_subprocess = False +GET_LEADER_BY_KIND = { + 'docker': ('docker_path', get_docker_pid), + 'lxc': ('lxc_info_path', get_lxc_pid), + 'lxd': ('lxc_path', get_lxd_pid), + 'machinectl': ('machinectl_path', get_machinectl_pid), +} + +class Options(mitogen.parent.Options): container = None username = 'root' kind = None @@ -128,24 +134,17 @@ class Stream(mitogen.parent.Stream): lxc_info_path = 'lxc-info' machinectl_path = 'machinectl' - GET_LEADER_BY_KIND = { - 'docker': ('docker_path', get_docker_pid), - 'lxc': ('lxc_info_path', get_lxc_pid), - 'lxd': ('lxc_path', get_lxd_pid), - 'machinectl': ('machinectl_path', get_machinectl_pid), - } - - def construct(self, container, kind, username=None, docker_path=None, - lxc_path=None, lxc_info_path=None, machinectl_path=None, - **kwargs): - super(Stream, self).construct(**kwargs) - if kind not in self.GET_LEADER_BY_KIND: + def __init__(self, container, kind, username=None, docker_path=None, + lxc_path=None, lxc_info_path=None, machinectl_path=None, + **kwargs): + super(Options, self).__init__(**kwargs) + if kind not in GET_LEADER_BY_KIND: raise Error('unsupported container kind: %r', kind) - self.container = container + self.container = mitogen.core.to_text(container) self.kind = kind if username: - self.username = username + self.username = mitogen.core.to_text(username) if docker_path: self.docker_path = docker_path if lxc_path: @@ -155,6 +154,11 @@ class Stream(mitogen.parent.Stream): if machinectl_path: self.machinectl_path = machinectl_path + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = False + # Order matters. https://github.com/karelzak/util-linux/commit/854d0fe/ NS_ORDER = ('ipc', 'uts', 'net', 'pid', 'mnt', 'user') @@ -189,15 +193,15 @@ class Stream(mitogen.parent.Stream): try: os.setgroups([grent.gr_gid for grent in grp.getgrall() - if self.username in grent.gr_mem]) - pwent = pwd.getpwnam(self.username) + if self.options.username in grent.gr_mem]) + pwent = pwd.getpwnam(self.options.username) os.setreuid(pwent.pw_uid, pwent.pw_uid) # shadow-4.4/libmisc/setupenv.c. Not done: MAIL, PATH os.environ.update({ 'HOME': pwent.pw_dir, 'SHELL': pwent.pw_shell or '/bin/sh', - 'LOGNAME': self.username, - 'USER': self.username, + 'LOGNAME': self.options.username, + 'USER': self.options.username, }) if ((os.path.exists(pwent.pw_dir) and os.access(pwent.pw_dir, os.X_OK))): @@ -217,7 +221,7 @@ class Stream(mitogen.parent.Stream): # namespaces, meaning starting new threads in the exec'd program will # fail. The solution is forking, so inject a /bin/sh call to achieve # this. - argv = super(Stream, self).get_boot_command() + argv = super(Connection, self).get_boot_command() # bash will exec() if a single command was specified and the shell has # nothing left to do, so "; exit $?" gives bash a reason to live. return ['/bin/sh', '-c', '%s; exit $?' % (mitogen.parent.Argv(argv),)] @@ -226,13 +230,12 @@ class Stream(mitogen.parent.Stream): return mitogen.parent.create_child(args, preexec_fn=self.preexec_fn) def _get_name(self): - return u'setns.' + self.container + return u'setns.' + self.options.container - def connect(self): - self.name = self._get_name() - attr, func = self.GET_LEADER_BY_KIND[self.kind] - tool_path = getattr(self, attr) - self.leader_pid = func(tool_path, self.container) + def connect(self, **kwargs): + attr, func = GET_LEADER_BY_KIND[self.options.kind] + tool_path = getattr(self.options, attr) + self.leader_pid = func(tool_path, self.options.container) LOG.debug('Leader PID for %s container %r: %d', - self.kind, self.container, self.leader_pid) - super(Stream, self).connect() + self.options.kind, self.options.container, self.leader_pid) + return super(Connection, self).connect(**kwargs) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 11b74c1b..b276dd28 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -29,7 +29,7 @@ # !mitogen: minify_safe """ -Functionality to allow establishing new slave contexts over an SSH connection. +Construct new children via the OpenSSH client. """ import logging @@ -42,7 +42,6 @@ except ImportError: import mitogen.parent from mitogen.core import b -from mitogen.core import bytes_partition try: any @@ -50,84 +49,124 @@ except NameError: from mitogen.core import any -LOG = logging.getLogger('mitogen') +LOG = logging.getLogger(__name__) + +auth_incorrect_msg = 'SSH authentication is incorrect' +password_incorrect_msg = 'SSH password is incorrect' +password_required_msg = 'SSH password was requested, but none specified' +hostkey_config_msg = ( + 'SSH requested permission to accept unknown host key, but ' + 'check_host_keys=ignore. This is likely due to ssh_args= ' + 'conflicting with check_host_keys=. Please correct your ' + 'configuration.' +) +hostkey_failed_msg = ( + 'Host key checking is enabled, and SSH reported an unrecognized or ' + 'mismatching host key.' +) # sshpass uses 'assword' because it doesn't lowercase the input. -PASSWORD_PROMPT = b('password') -HOSTKEY_REQ_PROMPT = b('are you sure you want to continue connecting (yes/no)?') -HOSTKEY_FAIL = b('host key verification failed.') +PASSWORD_PROMPT_PATTERN = re.compile( + b('password'), + re.I +) + +HOSTKEY_REQ_PATTERN = re.compile( + b(r'are you sure you want to continue connecting \(yes/no\)\?'), + re.I +) + +HOSTKEY_FAIL_PATTERN = re.compile( + b(r'host key verification failed\.'), + re.I +) # [user@host: ] permission denied -PERMDENIED_RE = re.compile( - ('(?:[^@]+@[^:]+: )?' # Absent in OpenSSH <7.5 - 'Permission denied').encode(), +# issue #271: work around conflict with user shell reporting 'permission +# denied' e.g. during chdir($HOME) by only matching it at the start of the +# line. +PERMDENIED_PATTERN = re.compile( + b('^(?:[^@]+@[^:]+: )?' # Absent in OpenSSH <7.5 + 'Permission denied'), re.I ) +DEBUG_PATTERN = re.compile(b('^debug[123]:')) + -DEBUG_PREFIXES = (b('debug1:'), b('debug2:'), b('debug3:')) +class PasswordError(mitogen.core.StreamError): + pass -def filter_debug(stream, it): - """ - Read line chunks from it, either yielding them directly, or building up and - logging individual lines if they look like SSH debug output. +class HostKeyError(mitogen.core.StreamError): + pass - This contains the mess of dealing with both line-oriented input, and partial - lines such as the password prompt. - Yields `(line, partial)` tuples, where `line` is the line, `partial` is - :data:`True` if no terminating newline character was present and no more - data exists in the read buffer. Consuming code can use this to unreliably - detect the presence of an interactive prompt. +class SetupProtocol(mitogen.parent.RegexProtocol): + """ + This protocol is attached to stderr of the SSH client. It responds to + various interactive prompts as required. """ - # The `partial` test is unreliable, but is only problematic when verbosity - # is enabled: it's possible for a combination of SSH banner, password - # prompt, verbose output, timing and OS buffering specifics to create a - # situation where an otherwise newline-terminated line appears to not be - # terminated, due to a partial read(). If something is broken when - # ssh_debug_level>0, this is the first place to look. - state = 'start_of_line' - buf = b('') - for chunk in it: - buf += chunk - while buf: - if state == 'start_of_line': - if len(buf) < 8: - # short read near buffer limit, block awaiting at least 8 - # bytes so we can discern a debug line, or the minimum - # interesting token from above or the bootstrap - # ('password', 'MITO000\n'). - break - elif any(buf.startswith(p) for p in DEBUG_PREFIXES): - state = 'in_debug' - else: - state = 'in_plain' - elif state == 'in_debug': - if b('\n') not in buf: - break - line, _, buf = bytes_partition(buf, b('\n')) - LOG.debug('%s: %s', stream.name, - mitogen.core.to_text(line.rstrip())) - state = 'start_of_line' - elif state == 'in_plain': - line, nl, buf = bytes_partition(buf, b('\n')) - yield line + nl, not (nl or buf) - if nl: - state = 'start_of_line' + password_sent = False + def _on_host_key_request(self, line, match): + if self.stream.conn.options.check_host_keys == 'accept': + LOG.debug('%s: accepting host key', self.stream.name) + self.stream.transmit_side.write(b('yes\n')) + return -class PasswordError(mitogen.core.StreamError): - pass + # _host_key_prompt() should never be reached with ignore or enforce + # mode, SSH should have handled that. User's ssh_args= is conflicting + # with ours. + self.stream.conn._fail_connection(HostKeyError(hostkey_config_msg)) + + def _on_host_key_failed(self, line, match): + self.stream.conn._fail_connection(HostKeyError(hostkey_failed_msg)) + + def _on_permission_denied(self, line, match): + if self.stream.conn.options.password is not None and \ + self.password_sent: + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) + elif PASSWORD_PROMPT_PATTERN.search(line) and \ + self.stream.conn.options.password is None: + # Permission denied (password,pubkey) + self.stream.conn._fail_connection( + PasswordError(password_required_msg) + ) + else: + self.stream.conn._fail_connection( + PasswordError(auth_incorrect_msg) + ) + def _on_password_prompt(self, line, match): + LOG.debug('%s: (password prompt): %s', self.stream.name, line) + if self.stream.conn.options.password is None: + self.stream.conn._fail(PasswordError(password_required_msg)) -class HostKeyError(mitogen.core.StreamError): - pass + self.stream.transmit_side.write( + (self.stream.conn.options.password + '\n').encode('utf-8') + ) + self.password_sent = True + def _on_debug_line(self, line, match): + text = mitogen.core.to_text(line.rstrip()) + LOG.debug('%s: %s', self.stream.name, text) + + PATTERNS = [ + (DEBUG_PATTERN, _on_debug_line), + (HOSTKEY_FAIL_PATTERN, _on_host_key_failed), + (PERMDENIED_PATTERN, _on_permission_denied), + ] + + PARTIAL_PATTERNS = [ + (PASSWORD_PROMPT_PATTERN, _on_password_prompt), + (HOSTKEY_REQ_PATTERN, _on_host_key_request), + ] -class Stream(mitogen.parent.Stream): - child_is_immediate_subprocess = False +class Options(mitogen.parent.Options): #: Default to whatever is available as 'python' on the remote machine, #: overriding sys.executable use. python_path = 'python' @@ -141,19 +180,19 @@ class Stream(mitogen.parent.Stream): hostname = None username = None port = None - identity_file = None password = None ssh_args = None check_host_keys_msg = 'check_host_keys= must be set to accept, enforce or ignore' - def construct(self, hostname, username=None, ssh_path=None, port=None, - check_host_keys='enforce', password=None, identity_file=None, - compression=True, ssh_args=None, keepalive_enabled=True, - keepalive_count=3, keepalive_interval=15, - identities_only=True, ssh_debug_level=None, **kwargs): - super(Stream, self).construct(**kwargs) + def __init__(self, hostname, username=None, ssh_path=None, port=None, + check_host_keys='enforce', password=None, identity_file=None, + compression=True, ssh_args=None, keepalive_enabled=True, + keepalive_count=3, keepalive_interval=15, + identities_only=True, ssh_debug_level=None, **kwargs): + super(Options, self).__init__(**kwargs) + if check_host_keys not in ('accept', 'enforce', 'ignore'): raise ValueError(self.check_host_keys_msg) @@ -175,143 +214,81 @@ class Stream(mitogen.parent.Stream): if ssh_debug_level: self.ssh_debug_level = ssh_debug_level - self._init_create_child() + +class Connection(mitogen.parent.Connection): + options_class = Options + diag_protocol_class = SetupProtocol + + child_is_immediate_subprocess = False + + def _get_name(self): + s = u'ssh.' + mitogen.core.to_text(self.options.hostname) + if self.options.port and self.options.port != 22: + s += u':%s' % (self.options.port,) + return s def _requires_pty(self): """ - Return :data:`True` if the configuration requires a PTY to be - allocated. This is only true if we must interactively accept host keys, - or type a password. + Return :data:`True` if a PTY to is required for this configuration, + because it must interactively accept host keys or type a password. """ - return (self.check_host_keys == 'accept' or - self.password is not None) + return ( + self.options.check_host_keys == 'accept' or + self.options.password is not None + ) - def _init_create_child(self): + def create_child(self, **kwargs): """ - Initialize the base class :attr:`create_child` and - :attr:`create_child_args` according to whether we need a PTY or not. + Avoid PTY use when possible to avoid a scaling limitation. """ if self._requires_pty(): - self.create_child = mitogen.parent.hybrid_tty_create_child + return mitogen.parent.hybrid_tty_create_child(**kwargs) else: - self.create_child = mitogen.parent.create_child - self.create_child_args = { - 'stderr_pipe': True, - } + return mitogen.parent.create_child(stderr_pipe=True, **kwargs) def get_boot_command(self): - bits = [self.ssh_path] - if self.ssh_debug_level: - bits += ['-' + ('v' * min(3, self.ssh_debug_level))] + bits = [self.options.ssh_path] + if self.options.ssh_debug_level: + bits += ['-' + ('v' * min(3, self.options.ssh_debug_level))] else: # issue #307: suppress any login banner, as it may contain the # password prompt, and there is no robust way to tell the # difference. bits += ['-o', 'LogLevel ERROR'] - if self.username: - bits += ['-l', self.username] - if self.port is not None: - bits += ['-p', str(self.port)] - if self.identities_only and (self.identity_file or self.password): + if self.options.username: + bits += ['-l', self.options.username] + if self.options.port is not None: + bits += ['-p', str(self.options.port)] + if self.options.identities_only and (self.options.identity_file or + self.options.password): bits += ['-o', 'IdentitiesOnly yes'] - if self.identity_file: - bits += ['-i', self.identity_file] - if self.compression: + if self.options.identity_file: + bits += ['-i', self.options.identity_file] + if self.options.compression: bits += ['-o', 'Compression yes'] - if self.keepalive_enabled: + if self.options.keepalive_enabled: bits += [ - '-o', 'ServerAliveInterval %s' % (self.keepalive_interval,), - '-o', 'ServerAliveCountMax %s' % (self.keepalive_count,), + '-o', 'ServerAliveInterval %s' % ( + self.options.keepalive_interval, + ), + '-o', 'ServerAliveCountMax %s' % ( + self.options.keepalive_count, + ), ] if not self._requires_pty(): bits += ['-o', 'BatchMode yes'] - if self.check_host_keys == 'enforce': + if self.options.check_host_keys == 'enforce': bits += ['-o', 'StrictHostKeyChecking yes'] - if self.check_host_keys == 'accept': + if self.options.check_host_keys == 'accept': bits += ['-o', 'StrictHostKeyChecking ask'] - elif self.check_host_keys == 'ignore': + elif self.options.check_host_keys == 'ignore': bits += [ '-o', 'StrictHostKeyChecking no', '-o', 'UserKnownHostsFile /dev/null', '-o', 'GlobalKnownHostsFile /dev/null', ] - if self.ssh_args: - bits += self.ssh_args - bits.append(self.hostname) - base = super(Stream, self).get_boot_command() + if self.options.ssh_args: + bits += self.options.ssh_args + bits.append(self.options.hostname) + base = super(Connection, self).get_boot_command() return bits + [shlex_quote(s).strip() for s in base] - - def _get_name(self): - s = u'ssh.' + mitogen.core.to_text(self.hostname) - if self.port: - s += u':%s' % (self.port,) - return s - - auth_incorrect_msg = 'SSH authentication is incorrect' - password_incorrect_msg = 'SSH password is incorrect' - password_required_msg = 'SSH password was requested, but none specified' - hostkey_config_msg = ( - 'SSH requested permission to accept unknown host key, but ' - 'check_host_keys=ignore. This is likely due to ssh_args= ' - 'conflicting with check_host_keys=. Please correct your ' - 'configuration.' - ) - hostkey_failed_msg = ( - 'Host key checking is enabled, and SSH reported an unrecognized or ' - 'mismatching host key.' - ) - - def _host_key_prompt(self): - if self.check_host_keys == 'accept': - LOG.debug('%s: accepting host key', self.name) - self.diag_stream.transmit_side.write(b('yes\n')) - return - - # _host_key_prompt() should never be reached with ignore or enforce - # mode, SSH should have handled that. User's ssh_args= is conflicting - # with ours. - raise HostKeyError(self.hostkey_config_msg) - - def _connect_input_loop(self, it): - password_sent = False - for buf, partial in filter_debug(self, it): - LOG.debug('%s: stdout: %s', self.name, buf.rstrip()) - if buf.endswith(self.EC0_MARKER): - self._ec0_received() - return - elif HOSTKEY_REQ_PROMPT in buf.lower(): - self._host_key_prompt() - elif HOSTKEY_FAIL in buf.lower(): - raise HostKeyError(self.hostkey_failed_msg) - elif PERMDENIED_RE.match(buf): - # issue #271: work around conflict with user shell reporting - # 'permission denied' e.g. during chdir($HOME) by only matching - # it at the start of the line. - if self.password is not None and password_sent: - raise PasswordError(self.password_incorrect_msg) - elif PASSWORD_PROMPT in buf and self.password is None: - # Permission denied (password,pubkey) - raise PasswordError(self.password_required_msg) - else: - raise PasswordError(self.auth_incorrect_msg) - elif partial and PASSWORD_PROMPT in buf.lower(): - if self.password is None: - raise PasswordError(self.password_required_msg) - LOG.debug('%s: sending password', self.name) - self.diag_stream.transmit_side.write( - (self.password + '\n').encode() - ) - password_sent = True - - raise mitogen.core.StreamError('bootstrap failed') - - def _connect_bootstrap(self): - fds = [self.receive_side.fd] - if self.diag_stream is not None: - fds.append(self.diag_stream.receive_side.fd) - - it = mitogen.parent.iter_read(fds=fds, deadline=self.connect_deadline) - try: - self._connect_input_loop(it) - finally: - it.close() diff --git a/mitogen/su.py b/mitogen/su.py index 5ff9e177..080c9782 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -29,10 +29,10 @@ # !mitogen: minify_safe import logging +import re import mitogen.core import mitogen.parent -from mitogen.core import b try: any @@ -42,87 +42,119 @@ except NameError: LOG = logging.getLogger(__name__) +password_incorrect_msg = 'su password is incorrect' +password_required_msg = 'su password is required' + class PasswordError(mitogen.core.StreamError): pass -class Stream(mitogen.parent.Stream): - # TODO: BSD su cannot handle stdin being a socketpair, but it does let the - # child inherit fds from the parent. So we can still pass a socketpair in - # for hybrid_tty_create_child(), there just needs to be either a shell - # snippet or bootstrap support for fixing things up afterwards. - create_child = staticmethod(mitogen.parent.tty_create_child) - child_is_immediate_subprocess = False +class SetupBootstrapProtocol(mitogen.parent.BootstrapProtocol): + password_sent = False + + def setup_patterns(self, conn): + """ + su options cause the regexes used to vary. This is a mess, requires + reworking. + """ + incorrect_pattern = re.compile( + mitogen.core.b('|').join( + re.escape(s.encode('utf-8')) + for s in conn.options.incorrect_prompts + ), + re.I + ) + prompt_pattern = re.compile( + re.escape( + conn.options.password_prompt.encode('utf-8') + ), + re.I + ) + + self.PATTERNS = mitogen.parent.BootstrapProtocol.PATTERNS + [ + (incorrect_pattern, type(self)._on_password_incorrect), + ] + self.PARTIAL_PATTERNS = mitogen.parent.BootstrapProtocol.PARTIAL_PATTERNS + [ + (prompt_pattern, type(self)._on_password_prompt), + ] + + def _on_password_prompt(self, line, match): + LOG.debug('%r: (password prompt): %r', + self.stream.name, line.decode('utf-8', 'replace')) + + if self.stream.conn.options.password is None: + self.stream.conn._fail_connection( + PasswordError(password_required_msg) + ) + return + + if self.password_sent: + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) + return + + self.stream.transmit_side.write( + (self.stream.conn.options.password + '\n').encode('utf-8') + ) + self.password_sent = True + + def _on_password_incorrect(self, line, match): + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) - #: Once connected, points to the corresponding DiagLogStream, allowing it to - #: be disconnected at the same time this stream is being torn down. - username = 'root' +class Options(mitogen.parent.Options): + username = u'root' password = None su_path = 'su' - password_prompt = b('password:') + password_prompt = u'password:' incorrect_prompts = ( - b('su: sorry'), # BSD - b('su: authentication failure'), # Linux - b('su: incorrect password'), # CentOS 6 - b('authentication is denied'), # AIX + u'su: sorry', # BSD + u'su: authentication failure', # Linux + u'su: incorrect password', # CentOS 6 + u'authentication is denied', # AIX ) - def construct(self, username=None, password=None, su_path=None, - password_prompt=None, incorrect_prompts=None, **kwargs): - super(Stream, self).construct(**kwargs) + def __init__(self, username=None, password=None, su_path=None, + password_prompt=None, incorrect_prompts=None, **kwargs): + super(Options, self).__init__(**kwargs) if username is not None: - self.username = username + self.username = mitogen.core.to_text(username) if password is not None: - self.password = password + self.password = mitogen.core.to_text(password) if su_path is not None: self.su_path = su_path if password_prompt is not None: - self.password_prompt = password_prompt.lower() + self.password_prompt = password_prompt if incorrect_prompts is not None: - self.incorrect_prompts = map(str.lower, incorrect_prompts) + self.incorrect_prompts = [ + mitogen.core.to_text(p) + for p in incorrect_prompts + ] + + +class Connection(mitogen.parent.Connection): + options_class = Options + stream_protocol_class = SetupBootstrapProtocol + + # TODO: BSD su cannot handle stdin being a socketpair, but it does let the + # child inherit fds from the parent. So we can still pass a socketpair in + # for hybrid_tty_create_child(), there just needs to be either a shell + # snippet or bootstrap support for fixing things up afterwards. + create_child = staticmethod(mitogen.parent.tty_create_child) + child_is_immediate_subprocess = False def _get_name(self): - return u'su.' + mitogen.core.to_text(self.username) + return u'su.' + self.options.username + + def stream_factory(self): + stream = super(Connection, self).stream_factory() + stream.protocol.setup_patterns(self) + return stream def get_boot_command(self): - argv = mitogen.parent.Argv(super(Stream, self).get_boot_command()) - return [self.su_path, self.username, '-c', str(argv)] - - password_incorrect_msg = 'su password is incorrect' - password_required_msg = 'su password is required' - - def _connect_input_loop(self, it): - password_sent = False - - for buf in it: - LOG.debug('%r: received %r', self, buf) - if buf.endswith(self.EC0_MARKER): - self._ec0_received() - return - if any(s in buf.lower() for s in self.incorrect_prompts): - if password_sent: - raise PasswordError(self.password_incorrect_msg) - elif self.password_prompt in buf.lower(): - if self.password is None: - raise PasswordError(self.password_required_msg) - if password_sent: - raise PasswordError(self.password_incorrect_msg) - LOG.debug('sending password') - self.transmit_side.write( - mitogen.core.to_text(self.password + '\n').encode('utf-8') - ) - password_sent = True - - raise mitogen.core.StreamError('bootstrap failed') - - def _connect_bootstrap(self): - it = mitogen.parent.iter_read( - fds=[self.receive_side.fd], - deadline=self.connect_deadline, - ) - try: - self._connect_input_loop(it) - finally: - it.close() + argv = mitogen.parent.Argv(super(Connection, self).get_boot_command()) + return [self.options.su_path, self.options.username, '-c', str(argv)] diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 868d4d76..ea07d0c1 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -35,11 +35,13 @@ import re import mitogen.core import mitogen.parent -from mitogen.core import b LOG = logging.getLogger(__name__) +password_incorrect_msg = 'sudo password is incorrect' +password_required_msg = 'sudo password is required' + # These are base64-encoded UTF-8 as our existing minifier/module server # struggles with Unicode Python source in some (forgotten) circumstances. PASSWORD_PROMPTS = [ @@ -99,14 +101,13 @@ PASSWORD_PROMPTS = [ PASSWORD_PROMPT_RE = re.compile( - u'|'.join( - base64.b64decode(s).decode('utf-8') + mitogen.core.b('|').join( + base64.b64decode(s) for s in PASSWORD_PROMPTS - ) + ), + re.I ) - -PASSWORD_PROMPT = b('password') SUDO_OPTIONS = [ #(False, 'bool', '--askpass', '-A') #(False, 'str', '--auth-type', '-a') @@ -181,10 +182,7 @@ def option(default, *args): return default -class Stream(mitogen.parent.Stream): - create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) - child_is_immediate_subprocess = False - +class Options(mitogen.parent.Options): sudo_path = 'sudo' username = 'root' password = None @@ -195,15 +193,16 @@ class Stream(mitogen.parent.Stream): selinux_role = None selinux_type = None - def construct(self, username=None, sudo_path=None, password=None, - preserve_env=None, set_home=None, sudo_args=None, - login=None, selinux_role=None, selinux_type=None, **kwargs): - super(Stream, self).construct(**kwargs) + def __init__(self, username=None, sudo_path=None, password=None, + preserve_env=None, set_home=None, sudo_args=None, + login=None, selinux_role=None, selinux_type=None, **kwargs): + super(Options, self).__init__(**kwargs) opts = parse_sudo_flags(sudo_args or []) self.username = option(self.username, username, opts.user) self.sudo_path = option(self.sudo_path, sudo_path) - self.password = password or None + if password: + self.password = mitogen.core.to_text(password) self.preserve_env = option(self.preserve_env, preserve_env, opts.preserve_env) self.set_home = option(self.set_home, set_home, opts.set_home) @@ -211,67 +210,62 @@ class Stream(mitogen.parent.Stream): self.selinux_role = option(self.selinux_role, selinux_role, opts.role) self.selinux_type = option(self.selinux_type, selinux_type, opts.type) + +class SetupProtocol(mitogen.parent.RegexProtocol): + password_sent = False + + def _on_password_prompt(self, line, match): + LOG.debug('%s: (password prompt): %s', + self.stream.name, line.decode('utf-8', 'replace')) + + if self.stream.conn.options.password is None: + self.stream.conn._fail_connection( + PasswordError(password_required_msg) + ) + return + + if self.password_sent: + self.stream.conn._fail_connection( + PasswordError(password_incorrect_msg) + ) + return + + self.stream.transmit_side.write( + (self.stream.conn.options.password + '\n').encode('utf-8') + ) + self.password_sent = True + + PARTIAL_PATTERNS = [ + (PASSWORD_PROMPT_RE, _on_password_prompt), + ] + + +class Connection(mitogen.parent.Connection): + diag_protocol_class = SetupProtocol + options_class = Options + create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) + create_child_args = { + 'escalates_privilege': True, + } + child_is_immediate_subprocess = False + def _get_name(self): - return u'sudo.' + mitogen.core.to_text(self.username) + return u'sudo.' + mitogen.core.to_text(self.options.username) def get_boot_command(self): # Note: sudo did not introduce long-format option processing until July # 2013, so even though we parse long-format options, supply short-form # to the sudo command. - bits = [self.sudo_path, '-u', self.username] - if self.preserve_env: + bits = [self.options.sudo_path, '-u', self.options.username] + if self.options.preserve_env: bits += ['-E'] - if self.set_home: + if self.options.set_home: bits += ['-H'] - if self.login: + if self.options.login: bits += ['-i'] - if self.selinux_role: - bits += ['-r', self.selinux_role] - if self.selinux_type: - bits += ['-t', self.selinux_type] - - bits = bits + ['--'] + super(Stream, self).get_boot_command() - LOG.debug('sudo command line: %r', bits) - return bits - - password_incorrect_msg = 'sudo password is incorrect' - password_required_msg = 'sudo password is required' - - def _connect_input_loop(self, it): - password_sent = False - - for buf in it: - LOG.debug('%s: received %r', self.name, buf) - if buf.endswith(self.EC0_MARKER): - self._ec0_received() - return - - match = PASSWORD_PROMPT_RE.search(buf.decode('utf-8').lower()) - if match is not None: - LOG.debug('%s: matched password prompt %r', - self.name, match.group(0)) - if self.password is None: - raise PasswordError(self.password_required_msg) - if password_sent: - raise PasswordError(self.password_incorrect_msg) - self.diag_stream.transmit_side.write( - (mitogen.core.to_text(self.password) + '\n').encode('utf-8') - ) - password_sent = True - - raise mitogen.core.StreamError('bootstrap failed') - - def _connect_bootstrap(self): - fds = [self.receive_side.fd] - if self.diag_stream is not None: - fds.append(self.diag_stream.receive_side.fd) - - it = mitogen.parent.iter_read( - fds=fds, - deadline=self.connect_deadline, - ) + if self.options.selinux_role: + bits += ['-r', self.options.selinux_role] + if self.options.selinux_type: + bits += ['-t', self.options.selinux_type] - try: - self._connect_input_loop(it) - finally: - it.close() + return bits + ['--'] + super(Connection, self).get_boot_command() diff --git a/mitogen/unix.py b/mitogen/unix.py index 66141eec..1af1c0ec 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -36,6 +36,7 @@ have the same privilege (auth_id) as the current process. """ import errno +import logging import os import socket import struct @@ -45,7 +46,24 @@ import tempfile import mitogen.core import mitogen.master -from mitogen.core import LOG + +LOG = logging.getLogger(__name__) + + +class Error(mitogen.core.Error): + """ + Base for errors raised by :mod:`mitogen.unix`. + """ + pass + + +class ConnectError(Error): + """ + Raised when :func:`mitogen.unix.connect` fails to connect to the listening + socket. + """ + #: UNIX error number reported by underlying exception. + errno = None def is_path_dead(path): @@ -65,9 +83,38 @@ def make_socket_path(): return tempfile.mktemp(prefix='mitogen_unix_', suffix='.sock') -class Listener(mitogen.core.BasicStream): +class ListenerStream(mitogen.core.Stream): + def on_receive(self, broker): + sock, _ = self.receive_side.fp.accept() + try: + self.protocol.on_accept_client(sock) + except: + sock.close() + raise + + +class Listener(mitogen.core.Protocol): + stream_class = ListenerStream keep_alive = True + @classmethod + def build_stream(cls, router, path=None, backlog=100): + if not path: + path = make_socket_path() + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + if os.path.exists(path) and is_path_dead(path): + LOG.debug('%r: deleting stale %r', cls.__name__, path) + os.unlink(path) + + sock.bind(path) + os.chmod(path, int('0600', 8)) + sock.listen(backlog) + + stream = super(Listener, cls).build_stream(router, path) + stream.accept(sock, sock) + router.broker.start_receive(stream) + return stream + def __repr__(self): return '%s.%s(%r)' % ( __name__, @@ -75,20 +122,9 @@ class Listener(mitogen.core.BasicStream): self.path, ) - def __init__(self, router, path=None, backlog=100): + def __init__(self, router, path): self._router = router - self.path = path or make_socket_path() - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - - if os.path.exists(self.path) and is_path_dead(self.path): - LOG.debug('%r: deleting stale %r', self, self.path) - os.unlink(self.path) - - self._sock.bind(self.path) - os.chmod(self.path, int('0600', 8)) - self._sock.listen(backlog) - self.receive_side = mitogen.core.Side(self, self._sock.fileno()) - router.broker.start_receive(self) + self.path = path def _unlink_socket(self): try: @@ -100,69 +136,91 @@ class Listener(mitogen.core.BasicStream): raise def on_shutdown(self, broker): - broker.stop_receive(self) + broker.stop_receive(self.stream) self._unlink_socket() - self._sock.close() - self.receive_side.closed = True + self.stream.receive_side.close() - def _accept_client(self, sock): + def on_accept_client(self, sock): sock.setblocking(True) try: pid, = struct.unpack('>L', sock.recv(4)) except (struct.error, socket.error): - LOG.error('%r: failed to read remote identity: %s', - self, sys.exc_info()[1]) + LOG.error('listener: failed to read remote identity: %s', + sys.exc_info()[1]) return context_id = self._router.id_allocator.allocate() - context = mitogen.parent.Context(self._router, context_id) - stream = mitogen.core.Stream(self._router, context_id) - stream.name = u'unix_client.%d' % (pid,) - stream.auth_id = mitogen.context_id - stream.is_privileged = True - try: sock.send(struct.pack('>LLL', context_id, mitogen.context_id, os.getpid())) except socket.error: - LOG.error('%r: failed to assign identity to PID %d: %s', - self, pid, sys.exc_info()[1]) + LOG.error('listener: failed to assign identity to PID %d: %s', + pid, sys.exc_info()[1]) return - LOG.debug('%r: accepted %r', self, stream) - stream.accept(sock.fileno(), sock.fileno()) + context = mitogen.parent.Context(self._router, context_id) + stream = mitogen.core.MitogenProtocol.build_stream( + router=self._router, + remote_id=context_id, + auth_id=mitogen.context_id, + ) + stream.name = u'unix_client.%d' % (pid,) + stream.accept(sock, sock) + LOG.debug('listener: accepted connection from PID %d: %s', + pid, stream.name) self._router.register(context, stream) - def on_receive(self, broker): - sock, _ = self._sock.accept() - try: - self._accept_client(sock) - finally: - sock.close() +def _connect(path, broker, sock): + try: + # ENOENT, ECONNREFUSED + sock.connect(path) + + # ECONNRESET + sock.send(struct.pack('>L', os.getpid())) + mitogen.context_id, remote_id, pid = struct.unpack('>LLL', sock.recv(12)) + except socket.error: + e = sys.exc_info()[1] + ce = ConnectError('could not connect to %s: %s', path, e.args[1]) + ce.errno = e.args[0] + raise ce -def connect(path, broker=None): - LOG.debug('unix.connect(path=%r)', path) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(path) - sock.send(struct.pack('>L', os.getpid())) - mitogen.context_id, remote_id, pid = struct.unpack('>LLL', sock.recv(12)) mitogen.parent_id = remote_id mitogen.parent_ids = [remote_id] - LOG.debug('unix.connect(): local ID is %r, remote is %r', + LOG.debug('client: local ID is %r, remote is %r', mitogen.context_id, remote_id) router = mitogen.master.Router(broker=broker) - stream = mitogen.core.Stream(router, remote_id) - stream.accept(sock.fileno(), sock.fileno()) + stream = mitogen.core.MitogenProtocol.build_stream(router, remote_id) + stream.accept(sock, sock) stream.name = u'unix_listener.%d' % (pid,) + mitogen.core.listen(stream, 'disconnect', _cleanup) + mitogen.core.listen(router.broker, 'shutdown', + lambda: router.disconnect_stream(stream)) + context = mitogen.parent.Context(router, remote_id) router.register(context, stream) + return router, context - mitogen.core.listen(router.broker, 'shutdown', - lambda: router.disconnect_stream(stream)) - sock.close() - return router, context +def connect(path, broker=None): + LOG.debug('client: connecting to %s', path) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + return _connect(path, broker, sock) + except: + sock.close() + raise + + +def _cleanup(): + """ + Reset mitogen.context_id and friends when our connection to the parent is + lost. Per comments on #91, these globals need to move to the Router so + fix-ups like this become unnecessary. + """ + mitogen.context_id = 0 + mitogen.parent_id = None + mitogen.parent_ids = [] diff --git a/mitogen/utils.py b/mitogen/utils.py index 94a171fb..b1347d02 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -39,7 +39,6 @@ import mitogen.master import mitogen.parent -LOG = logging.getLogger('mitogen') iteritems = getattr(dict, 'iteritems', dict.items) if mitogen.core.PY3: diff --git a/preamble_size.py b/preamble_size.py index f5f1adc1..f0d1e804 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -19,15 +19,19 @@ import mitogen.sudo router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) -stream = mitogen.ssh.Stream(router, 0, max_message_size=0, hostname='foo') +options = mitogen.ssh.Options(max_message_size=0, hostname='foo') +conn = mitogen.ssh.Connection(options, router) +conn.context = context -print('SSH command size: %s' % (len(' '.join(stream.get_boot_command())),)) -print('Preamble size: %s (%.2fKiB)' % ( - len(stream.get_preamble()), - len(stream.get_preamble()) / 1024.0, +print('SSH command size: %s' % (len(' '.join(conn.get_boot_command())),)) +print('Bootstrap (mitogen.core) size: %s (%.2fKiB)' % ( + len(conn.get_preamble()), + len(conn.get_preamble()) / 1024.0, )) +print('') + if '--dump' in sys.argv: - print(zlib.decompress(stream.get_preamble())) + print(zlib.decompress(conn.get_preamble())) exit() @@ -55,7 +59,7 @@ for mod in ( original_size = len(original) minimized = mitogen.minify.minimize_source(original) minimized_size = len(minimized) - compressed = zlib.compress(minimized, 9) + compressed = zlib.compress(minimized.encode(), 9) compressed_size = len(compressed) print( '%-25s' diff --git a/scripts/affin.sh b/scripts/affin.sh new file mode 100755 index 00000000..34c03d8b --- /dev/null +++ b/scripts/affin.sh @@ -0,0 +1,4 @@ +# show process affinities for running ansible-playbook +who="$1" +[ ! "$who" ] && who=ansible-playbook +for i in $(pgrep -f "$who") ; do taskset -c -p $i ; done|cut -d: -f2|sort -n |uniq -c diff --git a/scripts/debug-helpers.sh b/scripts/debug-helpers.sh new file mode 100644 index 00000000..7011c18c --- /dev/null +++ b/scripts/debug-helpers.sh @@ -0,0 +1,39 @@ +# +# Bash helpers for debugging. +# + +# Tell Ansible to write PID files for the mux and top-level process to CWD. +export MITOGEN_SAVE_PIDS=1 + + +# strace -ff -p $(muxpid) +muxpid() { + cat .ansible-mux.pid +} + +# gdb -p $(anspid) +anspid() { + cat .ansible-controller.pid +} + +# perf top -git $(muxtids) +# perf top -git $(muxtids) +muxtids() { + ls /proc/$(muxpid)/task | tr \\n , +} + +# perf top -git $(anstids) +anstids() { + ls /proc/$(anspid)/task | tr \\n , +} + +# ttrace $(muxpid) [.. options ..] +# strace only threads of PID, not children +ttrace() { + local pid=$1; shift; + local s="" + for i in $(ls /proc/$pid/task) ; do + s="-p $i $s" + done + strace $s "$@" +} diff --git a/scripts/release-notes.py b/scripts/release-notes.py new file mode 100644 index 00000000..08b60c0c --- /dev/null +++ b/scripts/release-notes.py @@ -0,0 +1,47 @@ +# coding=UTF-8 + +# Generate the fragment used to make email release announcements +# usage: release-notes.py 0.2.6 + +import sys +import urllib +import lxml.html + +import subprocess + + +response = urllib.urlopen('https://mitogen.networkgenomics.com/changelog.html') +tree = lxml.html.parse(response) + +prefix = 'v' + sys.argv[1].replace('.', '-') + +for elem in tree.getroot().cssselect('div.section[id]'): + if elem.attrib['id'].startswith(prefix): + break +else: + print('cant find') + + + +for child in tree.getroot().cssselect('body > *'): + child.getparent().remove(child) + +body, = tree.getroot().cssselect('body') +body.append(elem) + +proc = subprocess.Popen( + args=['w3m', '-T', 'text/html', '-dump', '-cols', '72'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, +) + +stdout, _ = proc.communicate(input=(lxml.html.tostring(tree))) +stdout = stdout.decode('UTF-8') +stdout = stdout.translate({ + ord(u'¶'): None, + ord(u'•'): ord(u'*'), + ord(u'’'): ord(u"'"), + ord(u'“'): ord(u'"'), + ord(u'”'): ord(u'"'), +}) +print(stdout) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index bec749f7..0135736b 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -13,6 +13,9 @@ retry_files_enabled = False display_args_to_stdout = True forks = 100 +# We use lots of deprecated functionality to support older versions. +deprecation_warnings = False + # issue #434; hosts/delegate_to; integration/delegate_to remote_user = ansible-cfg-remote-user diff --git a/tests/ansible/bench/_includes.yml b/tests/ansible/bench/_includes.yml new file mode 100644 index 00000000..6501db21 --- /dev/null +++ b/tests/ansible/bench/_includes.yml @@ -0,0 +1 @@ +- meta: end_play diff --git a/tests/ansible/bench/includes.yml b/tests/ansible/bench/includes.yml new file mode 100644 index 00000000..4f50113a --- /dev/null +++ b/tests/ansible/bench/includes.yml @@ -0,0 +1,4 @@ +- hosts: test-targets + tasks: + - include_tasks: _includes.yml + with_sequence: start=1 end=1000 diff --git a/tests/ansible/gcloud/.gitignore b/tests/ansible/gcloud/.gitignore new file mode 100644 index 00000000..ea1f477c --- /dev/null +++ b/tests/ansible/gcloud/.gitignore @@ -0,0 +1,2 @@ +terraform.tfstate* +.terraform diff --git a/tests/ansible/gcloud/Makefile b/tests/ansible/gcloud/Makefile new file mode 100644 index 00000000..c5413cd6 --- /dev/null +++ b/tests/ansible/gcloud/Makefile @@ -0,0 +1,3 @@ + +default: + terraform fmt diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index 3c7f9ea0..1128a9b1 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -1,19 +1,89 @@ -- hosts: controller - vars: - git_username: '{{ lookup("pipe", "git config --global user.name") }}' - git_email: '{{ lookup("pipe", "git config --global user.email") }}' - +- hosts: all + become: true tasks: + - apt: name={{item}} state=installed + with_items: + - openvpn + - tcpdump + - python-pip + - python-virtualenv + - strace + - libldap2-dev + - linux-perf + - libsasl2-dev + - build-essential + - git + - rsync + + - file: + path: /etc/openvpn + state: directory + + - copy: + dest: /etc/openvpn/secret + mode: '0600' + content: | + -----BEGIN OpenVPN Static key V1----- + f94005e4206828e281eb397aefd69b37 + ebe6cd39057d5641c5d8dd539cd07651 + 557d94d0077852bd8f92b68bef927169 + c5f0e42ac962a2cbbed35e107ffa0e71 + 1a2607c6bcd919ec5846917b20eb6684 + c7505152815d6ed7b4420714777a3d4a + 8edb27ca81971cba7a1e88fe3936e13b + 85e9be6706a30cd1334836ed0f08e899 + 78942329a330392dff42e4570731ac24 + 9330358aaa6828c07ecb41fb9c498a89 + 1e0435c5a45bfed390cd2104073634ef + b00f9fae1d3c49ef5de51854103edac9 + 5ff39c9dfc66ae270510b2ffa74d87d2 + 9d4b3844b1e1473237bc6dc78fb03e2e + 643ce58e667a532efceec7177367fb37 + a16379a51e0a8c8e3ec00a59952b79d4 + -----END OpenVPN Static key V1----- + + - copy: + dest: /etc/openvpn/k3.conf + content: | + remote k3.botanicus.net + dev tun + ifconfig 10.18.0.1 10.18.0.2 + secret secret + + - shell: systemctl enable openvpn@k3.service + - shell: systemctl start openvpn@k3.service + - lineinfile: line: "{{item}}" path: /etc/sysctl.conf register: sysctl_conf - become: true with_items: - "net.ipv4.ip_forward=1" - "kernel.perf_event_paranoid=-1" + - shell: /sbin/sysctl -p + when: sysctl_conf.changed + + - copy: + dest: /etc/rc.local + mode: "0744" + content: | + #!/bin/bash + iptables -t nat -F; + iptables -t nat -X; + iptables -t nat -A POSTROUTING -j MASQUERADE; + + - shell: systemctl daemon-reload + - shell: systemctl enable rc-local + - shell: systemctl start rc-local + + +- hosts: all + vars: + git_username: '{{ lookup("pipe", "git config --global user.name") }}' + git_email: '{{ lookup("pipe", "git config --global user.email") }}' + tasks: - copy: src: ~/.ssh/id_gitlab dest: ~/.ssh/id_gitlab @@ -23,38 +93,6 @@ dest: ~/.ssh/config src: ssh_config.j2 - - lineinfile: - line: "{{item}}" - path: /etc/sysctl.conf - become: true - with_items: - - net.ipv4.ip_forward=1 - - kernel.perf_event_paranoid=-1 - register: sysctl_conf - - - shell: /sbin/sysctl -p - when: sysctl_conf.changed - become: true - - - shell: | - iptables -t nat -F; - iptables -t nat -X; - iptables -t nat -A POSTROUTING -j MASQUERADE; - become: true - - - apt: name={{item}} state=installed - become: true - with_items: - - python-pip - - python-virtualenv - - strace - - libldap2-dev - - linux-perf - - libsasl2-dev - - build-essential - - git - - rsync - - shell: "rsync -a ~/.ssh {{inventory_hostname}}:" connection: local @@ -119,4 +157,3 @@ path: ~/prj/ansible/inventory/gcloud.py state: link src: ~/mitogen/tests/ansible/lib/inventory/gcloud.py - diff --git a/tests/ansible/gcloud/gce.yml b/tests/ansible/gcloud/gce.yml deleted file mode 100644 index e3f64c23..00000000 --- a/tests/ansible/gcloud/gce.yml +++ /dev/null @@ -1,11 +0,0 @@ - -- hosts: localhost - tasks: - - command: date +%Y%m%d-%H%M%S - register: out - - - set_fact: - instance_name: "controller-{{out.stdout}}" - - - command: > - gcloud compute instances create {{instance_name}} --can-ip-forward --machine-type=n1-standard-8 --preemptible --scopes=compute-ro --image-project=debian-cloud --image-family=debian-9 diff --git a/tests/ansible/gcloud/mitogen-load-testing.tf b/tests/ansible/gcloud/mitogen-load-testing.tf new file mode 100644 index 00000000..9bab03d4 --- /dev/null +++ b/tests/ansible/gcloud/mitogen-load-testing.tf @@ -0,0 +1,149 @@ +variable "node-count" { + default = 0 +} + +variable "preemptible" { + default = true +} + +variable "big" { + default = false +} + +provider "google" { + project = "mitogen-load-testing" + region = "europe-west1" + zone = "europe-west1-d" +} + +resource "google_compute_instance" "controller" { + name = "ansible-controller" + machine_type = "${var.big ? "n1-highcpu-32" : "custom-1-1024"}" + + allow_stopping_for_update = true + can_ip_forward = true + + boot_disk { + initialize_params { + image = "debian-cloud/debian-9" + } + } + + scheduling { + preemptible = true + automatic_restart = false + } + + network_interface { + subnetwork = "${google_compute_subnetwork.loadtest-subnet.self_link}" + access_config = {} + } + + provisioner "local-exec" { + command = <<-EOF + ip=${google_compute_instance.controller.network_interface.0.access_config.0.nat_ip}; + ssh-keygen -R $ip; + ssh-keyscan $ip >> ~/.ssh/known_hosts; + sed -ri -e "s/.*CONTROLLER_IP_HERE.*/ Hostname $ip/" ~/.ssh/config; + ansible-playbook -i $ip, controller.yml + EOF + } +} + +resource "google_compute_network" "loadtest" { + name = "loadtest" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "loadtest-subnet" { + name = "loadtest-subnet" + ip_cidr_range = "10.19.0.0/16" + network = "${google_compute_network.loadtest.id}" +} + +resource "google_compute_firewall" "allow-all-in" { + name = "allow-all-in" + network = "${google_compute_network.loadtest.name}" + direction = "INGRESS" + + allow { + protocol = "all" + } +} + +resource "google_compute_firewall" "allow-all-out" { + name = "allow-all-out" + network = "${google_compute_network.loadtest.name}" + direction = "EGRESS" + + allow { + protocol = "all" + } +} + +resource "google_compute_route" "route-nodes-via-controller" { + name = "route-nodes-via-controller" + dest_range = "0.0.0.0/0" + network = "${google_compute_network.loadtest.name}" + next_hop_instance = "${google_compute_instance.controller.self_link}" + next_hop_instance_zone = "${google_compute_instance.controller.zone}" + priority = 800 + tags = ["node"] +} + +resource "google_compute_instance_template" "node" { + name = "node" + tags = ["node"] + machine_type = "custom-1-1024" + + scheduling { + preemptible = "${var.preemptible}" + automatic_restart = false + } + + disk { + source_image = "debian-cloud/debian-9" + auto_delete = true + boot = true + } + + network_interface { + subnetwork = "${google_compute_subnetwork.loadtest-subnet.self_link}" + } +} + +# +# Compute Engine tops out at 1000 VMs per group +# + +resource "google_compute_instance_group_manager" "nodes-a" { + name = "nodes-a" + + base_instance_name = "node" + instance_template = "${google_compute_instance_template.node.self_link}" + target_size = "${var.node-count / 4}" +} + +resource "google_compute_instance_group_manager" "nodes-b" { + name = "nodes-b" + + base_instance_name = "node" + instance_template = "${google_compute_instance_template.node.self_link}" + target_size = "${var.node-count / 4}" +} + +resource "google_compute_instance_group_manager" "nodes-c" { + name = "nodes-c" + + base_instance_name = "node" + instance_template = "${google_compute_instance_template.node.self_link}" + target_size = "${var.node-count / 4}" +} + +resource "google_compute_instance_group_manager" "nodes-d" { + name = "nodes-d" + + base_instance_name = "node" + instance_template = "${google_compute_instance_template.node.self_link}" + target_size = "${var.node-count / 4}" +} diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index 25f86d6d..3e81ce6a 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -7,6 +7,10 @@ ansible_user: mitogen__has_sudo_pubkey ansible_become_pass: has_sudo_pubkey_password ansible_ssh_private_key_file: /tmp/synchronize-action-key + + # https://github.com/ansible/ansible/issues/56629 + ansible_ssh_pass: '' + ansible_password: '' tasks: # must copy git file to set proper file mode. - copy: diff --git a/tests/ansible/integration/async/runner_one_job.yml b/tests/ansible/integration/async/runner_one_job.yml index ca798a7f..871d672f 100644 --- a/tests/ansible/integration/async/runner_one_job.yml +++ b/tests/ansible/integration/async/runner_one_job.yml @@ -9,9 +9,10 @@ # Verify output of a single async job. - name: start 2 second op + # Sleep after writing; see https://github.com/ansible/ansible/issues/51393 shell: | + echo alldone; sleep 1; - echo alldone async: 1000 poll: 0 register: job1 @@ -37,7 +38,12 @@ - result1.ansible_job_id == job1.ansible_job_id - result1.attempts <= 100000 - result1.changed == True - - result1.cmd == "sleep 1;\n echo alldone" + # ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44 + - | + (ansible_version.full >= '2.8' and + result1.cmd == "echo alldone;\nsleep 1;\n") or + (ansible_version.full < '2.8' and + result1.cmd == "echo alldone;\n sleep 1;") - result1.delta|length == 14 - result1.start|length == 26 - result1.finished == 1 diff --git a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml index 6d87fe6c..dcfa186f 100644 --- a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml +++ b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml @@ -20,5 +20,6 @@ - job1.failed == True - | job1.msg == "async task did not complete within the requested time" or + job1.msg == "async task did not complete within the requested time - 1s" or job1.msg == "Job reached maximum time limit of 1 seconds." diff --git a/tests/ansible/integration/become/sudo_flags_failure.yml b/tests/ansible/integration/become/sudo_flags_failure.yml index 52404019..39fbb4b8 100644 --- a/tests/ansible/integration/become/sudo_flags_failure.yml +++ b/tests/ansible/integration/become/sudo_flags_failure.yml @@ -17,5 +17,6 @@ - out.failed - | ('sudo: no such option: --derps' in out.msg) or + ("sudo: invalid option -- '-'" in out.module_stderr) or ("sudo: unrecognized option `--derps'" in out.module_stderr) or ("sudo: unrecognized option '--derps'" in out.module_stderr) diff --git a/tests/ansible/integration/become/sudo_password.yml b/tests/ansible/integration/become/sudo_password.yml index 128d8aee..f377fead 100644 --- a/tests/ansible/integration/become/sudo_password.yml +++ b/tests/ansible/integration/become/sudo_password.yml @@ -16,6 +16,7 @@ that: | out.failed and ( ('password is required' in out.msg) or + ('Missing sudo password' in out.msg) or ('password is required' in out.module_stderr) ) diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index d7af7f81..bfde1265 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -37,6 +37,8 @@ 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, 'python_path': ["/usr/bin/python"], @@ -65,6 +67,8 @@ 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, 'python_path': ["/usr/bin/python"], diff --git a/tests/ansible/integration/connection_delegation/local_action.yml b/tests/ansible/integration/connection_delegation/local_action.yml index 91fb9739..05fc3db9 100644 --- a/tests/ansible/integration/connection_delegation/local_action.yml +++ b/tests/ansible/integration/connection_delegation/local_action.yml @@ -27,7 +27,7 @@ 'remote_name': null, 'password': null, 'username': 'root', - 'sudo_path': null, + 'sudo_path': 'sudo', 'sudo_args': ['-H', '-S', '-n'], }, 'method': 'sudo', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 50029569..ed298599 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -71,6 +71,8 @@ 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -112,6 +114,8 @@ 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -164,6 +168,8 @@ 'hostname': 'cd-normal-normal', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -205,6 +211,8 @@ 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -233,6 +241,8 @@ 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -285,6 +295,8 @@ 'hostname': 'cd-newuser-normal-normal', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], @@ -327,6 +339,8 @@ 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, + 'keepalive_interval': 30, + 'keepalive_count': 10, 'password': null, 'port': null, "python_path": ["/usr/bin/python"], diff --git a/tests/ansible/integration/context_service/disconnect_cleanup.yml b/tests/ansible/integration/context_service/disconnect_cleanup.yml index 575358f6..3275b596 100644 --- a/tests/ansible/integration/context_service/disconnect_cleanup.yml +++ b/tests/ansible/integration/context_service/disconnect_cleanup.yml @@ -24,7 +24,7 @@ - mitogen_action_script: script: | self._connection._connect() - result['dump'] = self._connection.parent.call_service( + result['dump'] = self._connection.get_binding().get_service_context().call_service( service_name='ansible_mitogen.services.ContextService', method_name='dump' ) @@ -39,7 +39,7 @@ - mitogen_action_script: script: | self._connection._connect() - result['dump'] = self._connection.parent.call_service( + result['dump'] = self._connection.get_binding().get_service_context().call_service( service_name='ansible_mitogen.services.ContextService', method_name='dump' ) diff --git a/tests/ansible/integration/context_service/remote_name.yml b/tests/ansible/integration/context_service/remote_name.yml index 827abaee..d7116ec1 100644 --- a/tests/ansible/integration/context_service/remote_name.yml +++ b/tests/ansible/integration/context_service/remote_name.yml @@ -7,6 +7,10 @@ - meta: end_play when: not is_mitogen + # Too much hassle to make this work for OSX + - meta: end_play + when: ansible_system != 'Linux' + - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out - debug: var=out diff --git a/tests/ansible/integration/runner/crashy_new_style_module.yml b/tests/ansible/integration/runner/crashy_new_style_module.yml index 40ee7f88..a29493be 100644 --- a/tests/ansible/integration/runner/crashy_new_style_module.yml +++ b/tests/ansible/integration/runner/crashy_new_style_module.yml @@ -12,7 +12,14 @@ that: - not out.changed - out.rc == 1 - - out.msg == "MODULE FAILURE" + # ansible/62d8c8fde6a76d9c567ded381e9b34dad69afcd6 + - | + (ansible_version.full < '2.7' and out.msg == "MODULE FAILURE") or + (ansible_version.full >= '2.7' and + out.msg == ( + "MODULE FAILURE\n" + + "See stdout/stderr for the exact error" + )) - out.module_stdout == "" - "'Traceback (most recent call last)' in out.module_stderr" - "\"NameError: name 'kaboom' is not defined\" in out.module_stderr" diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml index d8a1af0c..8e215bf3 100644 --- a/tests/ansible/integration/runner/custom_binary_single_null.yml +++ b/tests/ansible/integration/runner/custom_binary_single_null.yml @@ -17,8 +17,8 @@ - "out.results[0].msg.startswith('MODULE FAILURE')" - "out.results[0].module_stdout.startswith('/bin/sh: ')" - | - out.results[0].module_stdout.endswith('/custom_binary_single_null: cannot execute binary file\r\n') or - out.results[0].module_stdout.endswith('/custom_binary_single_null: Exec format error\r\n') + out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or + out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n') # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 9f7d08ba..77f2cb5c 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -5,13 +5,13 @@ tasks: - custom_python_new_style_missing_interpreter: foo: true - with_sequence: start=1 end={{end|default(1)}} + with_sequence: start=0 end={{end|default(1)}} register: out - assert: - that: | - (not out.changed) and - (not out.results[0].changed) and - out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and - out.results[0].msg == 'Here is my input' - + that: + - "not out.changed" + - "not out.results[0].changed" + # Random breaking interface change since 2.7.x + #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" + - "out.results[0].msg == 'Here is my input'" diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index d86bff4a..0d29d0ac 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -4,16 +4,16 @@ tasks: - custom_python_new_style_module: foo: true - with_sequence: start=1 end={{end|default(1)}} + with_sequence: start=0 end={{end|default(1)}} register: out - assert: - that: | - (not out.changed) and - (not out.results[0].changed) and - out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and - out.results[0].msg == 'Here is my input' - + that: + - "not out.changed" + - "not out.results[0].changed" + # Random breaking interface change since 2.7.x + #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" + - "out.results[0].msg == 'Here is my input'" # Verify sys.argv is not Unicode. - custom_python_detect_environment: diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index ba53d1e0..867a8c17 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -13,6 +13,7 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index 7a2cd81c..1dbe2a48 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxc + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index 86f4b185..7839a35f 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxd + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 40d4f4b0..5387744e 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -10,7 +10,8 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_doas - ansible_become_exe: stub-doas.py + ansible_python_interpreter: python # avoid Travis virtualenv breakage + ansible_doas_exe: stub-doas.py ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index b7ca3d26..e78afebc 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo + ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index c57a8c5c..efef3761 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -18,6 +18,7 @@ -i localhost, -c setns -e mitogen_kind=lxc + -e ansible_python_interpreter=python -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py -m shell -a "echo hi" diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index 7db47661..adee0b14 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -18,6 +18,7 @@ -i localhost, -c setns -e mitogen_kind=lxd + -e ansible_python_interpreter=python -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py -m shell -a "echo hi" diff --git a/tests/ansible/lib/action/mitogen_shutdown_all.py b/tests/ansible/lib/action/mitogen_shutdown_all.py index c28d9d4b..59191450 100644 --- a/tests/ansible/lib/action/mitogen_shutdown_all.py +++ b/tests/ansible/lib/action/mitogen_shutdown_all.py @@ -23,9 +23,10 @@ class ActionModule(ActionBase): } self._connection._connect() + binding = self._connection.get_binding() return { 'changed': True, - 'result': self._connection.parent.call_service( + 'result': binding.get_service_context().call_service( service_name='ansible_mitogen.services.ContextService', method_name='shutdown_all', ) diff --git a/tests/ansible/lib/callback/fork_histogram.py b/tests/ansible/lib/callback/fork_histogram.py index 9ce50e13..15260cb5 100644 --- a/tests/ansible/lib/callback/fork_histogram.py +++ b/tests/ansible/lib/callback/fork_histogram.py @@ -10,7 +10,11 @@ import sys import time import ansible.plugins.callback -import hdrh.histogram + +try: + import hdrh.histogram +except ImportError: + hdrh = None def get_fault_count(who=resource.RUSAGE_CHILDREN): @@ -25,9 +29,9 @@ class CallbackModule(ansible.plugins.callback.CallbackBase): if self.hist is not None: return - self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3) - self.fork_latency_sum_usec = 0.0 - if 'FORK_HISTOGRAM' in os.environ: + if hdrh and 'FORK_HISTOGRAM' in os.environ: + self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3) + self.fork_latency_sum_usec = 0.0 self.install() def install(self): @@ -54,7 +58,7 @@ class CallbackModule(ansible.plugins.callback.CallbackBase): self.hist.record_value(latency_usec) def playbook_on_stats(self, stats): - if 'FORK_HISTOGRAM' not in os.environ: + if hdrh is None or 'FORK_HISTOGRAM' not in os.environ: return self_faults = get_fault_count(resource.RUSAGE_SELF) - self.faults_at_start diff --git a/tests/ansible/lib/inventory/gcloud.py b/tests/ansible/lib/inventory/gcloud.py index 73e083f4..9920a008 100755 --- a/tests/ansible/lib/inventory/gcloud.py +++ b/tests/ansible/lib/inventory/gcloud.py @@ -14,14 +14,14 @@ import googleapiclient.discovery def main(): project = 'mitogen-load-testing' zone = 'europe-west1-d' - group_name = 'micro-debian9' + prefix = 'node-' client = googleapiclient.discovery.build('compute', 'v1') resp = client.instances().list(project=project, zone=zone).execute() ips = [] for inst in resp['items']: - if inst['status'] == 'RUNNING' and inst['name'].startswith(group_name): + if inst['status'] == 'RUNNING' and inst['name'].startswith(prefix): ips.extend( #bytes(config['natIP']) bytes(interface['networkIP']) diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 66264010..eea4baa4 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -17,3 +17,7 @@ print(" \"changed\": false,") print(" \"msg\": \"Here is my input\",") print(" \"input\": [%s]" % (input_json,)) print("}") + +# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module +# process to exit itself. So needless. +sys.exit(0) diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 70ee062d..f9c176c1 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -23,3 +23,7 @@ print(" \"__package__\": \"%s\"," % (__package__,)) print(" \"msg\": \"Here is my input\",") print(" \"input\": [%s]" % (input_json,)) print("}") + +# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module +# process to exit itself. So needless. +sys.exit(0) diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py new file mode 100644 index 00000000..7fe3fd1b --- /dev/null +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -0,0 +1,14 @@ +#!/usr/bin/python +# #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the +# process environment. + +import os + +try: + import json +except ImportError: + import simplejson as json + +print(json.dumps({ + 'cwd': os.getcwd() +})) diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py new file mode 100644 index 00000000..1fc31b4e --- /dev/null +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -0,0 +1,21 @@ +#!/usr/bin/python +# issue #590: I am an Ansible new-style Python module that tries to use +# ansible.module_utils.distro. + +import ansible +from ansible.module_utils.basic import AnsibleModule + +if ansible.__version__ > '2.8': + from ansible.module_utils import distro +else: + distro = None + +def main(): + module = AnsibleModule(argument_spec={}) + if ansible.__version__ > '2.8': + module.exit_json(info=distro.info()) + else: + module.exit_json(info={'id': None}) + +if __name__ == '__main__': + main() diff --git a/tests/ansible/mitogen_ansible_playbook.py b/tests/ansible/mitogen_ansible_playbook.py index 3af1791c..54fd4283 100755 --- a/tests/ansible/mitogen_ansible_playbook.py +++ b/tests/ansible/mitogen_ansible_playbook.py @@ -3,4 +3,8 @@ import os import subprocess import sys os.environ['ANSIBLE_STRATEGY'] = 'mitogen_linear' -subprocess.check_call(['./run_ansible_playbook.py'] + sys.argv[1:]) +os.execlp( + './run_ansible_playbook.py', + './run_ansible_playbook.py', + *sys.argv[1:] +) diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 123d87d9..81780bb3 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -8,3 +8,7 @@ - include: issue_154__module_state_leaks.yml - include: issue_177__copy_module_failing.yml - include: issue_332_ansiblemoduleerror_first_occurrence.yml +- include: issue_558_unarchive_failed.yml +- include: issue_590__sys_modules_crap.yml +- include: issue_591__setuptools_cwd_crash.yml +- include: issue_615__streaming_transfer.yml diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml index 0162c210..6f32af19 100644 --- a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -10,5 +10,4 @@ - assert: that: - - out.state == 'absent' - out.msg == 'file (/usr/bin/does-not-exist) is absent, cannot continue' diff --git a/tests/ansible/regression/issue_558_unarchive_failed.yml b/tests/ansible/regression/issue_558_unarchive_failed.yml new file mode 100644 index 00000000..c6b1c9f6 --- /dev/null +++ b/tests/ansible/regression/issue_558_unarchive_failed.yml @@ -0,0 +1,13 @@ +# _execute_module() would unconditionally delete shell.tmpdir without +# respecting the passed in 'tmp' parameter on Ansible 2.3. + +- name: regression/issue_558_unarchive_failed.yml + hosts: test-targets + tasks: + - file: state=absent path=/tmp/foo + - file: state=directory path=/tmp/foo + - unarchive: + src: "{{git_basedir}}/tests/data/unarchive_test.tar" + dest: /tmp/foo + # garbage doesn't work with BSD tar + when: ansible_system != 'Darwin' diff --git a/tests/ansible/regression/issue_590__sys_modules_crap.yml b/tests/ansible/regression/issue_590__sys_modules_crap.yml new file mode 100644 index 00000000..41130b68 --- /dev/null +++ b/tests/ansible/regression/issue_590__sys_modules_crap.yml @@ -0,0 +1,12 @@ + +- hosts: test-targets + tasks: + - meta: end_play + when: ansible_version.full < '2.8' + + - custom_python_uses_distro: + register: out + + - assert: + that: + - "'id' in out.info" diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml new file mode 100644 index 00000000..fc73825c --- /dev/null +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -0,0 +1,24 @@ +# #591: process CWD is not reset before start of module execution. This is +# usually fine, except for modules importing setuptools early, which attempts +# to call getcwd() before AnsibleModule has had a chance to clean up the +# process environment. + +- hosts: test-targets + tasks: + - meta: end_play + when: not is_mitogen + + - custom_python_run_script: + script: | + import os + try: + os.chdir(module.tmpdir) + except: + # Ansible 2.3. + os.chdir(os.path.dirname(__file__)) + + # Will crash if process has a nonexistent CWD. + - custom_python_os_getcwd: + script: | + import os + self._connection.get_chain().call(os.getcwd) diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml new file mode 100644 index 00000000..aa7c62c4 --- /dev/null +++ b/tests/ansible/regression/issue_615__streaming_transfer.yml @@ -0,0 +1,21 @@ +# issue #615: 'fetch' with become: was internally using slurp. + +- hosts: target + any_errors_fatal: True + gather_facts: no + become: true + vars: + mitogen_ssh_compression: false + tasks: + - shell: | + dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; + chmod go= /tmp/512mb.zero + + - fetch: + src: /tmp/512mb.zero + dest: /tmp/fetch-out + + - file: + path: /tmp/fetch-out + state: absent + delegate_to: localhost diff --git a/tests/ansible/run_ansible_playbook.py b/tests/ansible/run_ansible_playbook.py index b5b459a1..467eaffc 100755 --- a/tests/ansible/run_ansible_playbook.py +++ b/tests/ansible/run_ansible_playbook.py @@ -51,7 +51,11 @@ else: os.path.join(GIT_BASEDIR, 'tests/ansible/hosts') ) -args = ['ansible-playbook'] +if 'ANSIBLE_ARGV' in os.environ: + args = eval(os.environ['ANSIBLE_ARGV']) +else: + args = ['ansible-playbook'] + args += ['-e', json.dumps(extra)] args += sys.argv[1:] os.execvp(args[0], args) diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 8ee96085..9572717f 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -1,6 +1,7 @@ import multiprocessing import os +import sys import tempfile import mock @@ -63,32 +64,32 @@ class FixedPolicyTest(testlib.TestCase): def test_assign_muxprocess_1core(self): # Uniprocessor . policy = self.klass(cpu_count=1) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) def test_assign_muxprocess_2core(self): # Small SMP gets dedicated core. policy = self.klass(cpu_count=2) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) - policy.assign_muxprocess() + policy.assign_muxprocess(0) def test_assign_muxprocess_3core(self): # Small SMP gets a dedicated core. policy = self.klass(cpu_count=3) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) def test_assign_muxprocess_4core(self): # Big SMP gets a dedicated core. policy = self.klass(cpu_count=4) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) - policy.assign_muxprocess() + policy.assign_muxprocess(0) self.assertEquals(0x1, policy.mask) def test_assign_worker_1core(self): @@ -196,13 +197,13 @@ class LinuxPolicyTest(testlib.TestCase): tf = tempfile.NamedTemporaryFile() try: before = self._get_cpus() - self.policy._set_cpu(3) + self.policy._set_cpu(None, 3) my_cpu = self._get_cpus() - pid = mitogen.parent.detach_popen( + proc = mitogen.parent.popen( args=['cp', '/proc/self/status', tf.name] ) - os.waitpid(pid, 0) + proc.wait() his_cpu = self._get_cpus(tf.name) self.assertNotEquals(my_cpu, his_cpu) @@ -221,6 +222,11 @@ class MockLinuxPolicyTest(testlib.TestCase): for x in range(1, 4096, 32): policy.assign_subprocess() +MockLinuxPolicyTest = unittest2.skipIf( + condition=(not sys.platform.startswith('linuxPolicy')), + reason='select.select() not supported' +)(MockLinuxPolicyTest) + if __name__ == '__main__': unittest2.main() diff --git a/tests/ansible/tests/connection_test.py b/tests/ansible/tests/connection_test.py index 401cbe9e..54ea3d99 100644 --- a/tests/ansible/tests/connection_test.py +++ b/tests/ansible/tests/connection_test.py @@ -13,22 +13,73 @@ import ansible.errors import ansible.playbook.play_context import mitogen.core +import mitogen.utils + import ansible_mitogen.connection import ansible_mitogen.plugins.connection.mitogen_local import ansible_mitogen.process + import testlib -LOGGER_NAME = ansible_mitogen.target.LOG.name +class MuxProcessMixin(object): + no_zombie_check = True + @classmethod + def setUpClass(cls): + #mitogen.utils.log_to_file() + cls.model = ansible_mitogen.process.get_classic_worker_model( + _init_logging=False + ) + ansible_mitogen.process.set_worker_model(cls.model) + cls.model.on_strategy_start() + super(MuxProcessMixin, cls).setUpClass() -# TODO: fixtureize -import mitogen.utils -mitogen.utils.log_to_file() -ansible_mitogen.process.MuxProcess.start(_init_logging=False) + @classmethod + def tearDownClass(cls): + cls.model._test_reset() + super(MuxProcessMixin, cls).tearDownClass() + + +class ConnectionMixin(MuxProcessMixin): + klass = ansible_mitogen.plugins.connection.mitogen_local.Connection + + def make_connection(self): + play_context = ansible.playbook.play_context.PlayContext() + return self.klass(play_context, new_stdin=False) + def wait_for_completion(self): + # put_data() is asynchronous, must wait for operation to happen. Do + # that by making RPC for some junk that must run on the thread once op + # completes. + self.conn.get_chain().call(os.getpid) -class OptionalIntTest(unittest2.TestCase): + def setUp(self): + super(ConnectionMixin, self).setUp() + self.conn = self.make_connection() + + def tearDown(self): + self.conn.close() + super(ConnectionMixin, self).tearDown() + + +class MuxShutdownTest(ConnectionMixin, testlib.TestCase): + def test_connection_failure_raised(self): + # ensure if a WorkerProcess tries to connect to a MuxProcess that has + # already shut down, it fails with a graceful error. + path = self.model._muxes[0].path + os.rename(path, path + '.tmp') + try: + #e = self.assertRaises(ansible.errors.AnsibleError, + #lambda: self.conn._connect() + #) + e = 1 + print(e) + finally: + os.rename(path + '.tmp', path) + + +class OptionalIntTest(testlib.TestCase): func = staticmethod(ansible_mitogen.connection.optional_int) def test_already_int(self): @@ -48,29 +99,22 @@ class OptionalIntTest(unittest2.TestCase): self.assertEquals(None, self.func({1:2})) -class ConnectionMixin(object): - klass = ansible_mitogen.plugins.connection.mitogen_local.Connection - - def make_connection(self): - play_context = ansible.playbook.play_context.PlayContext() - return self.klass(play_context, new_stdin=False) +class FetchFileTest(ConnectionMixin, testlib.TestCase): + def test_success(self): + with tempfile.NamedTemporaryFile(prefix='mitotest') as ifp: + with tempfile.NamedTemporaryFile(prefix='mitotest') as ofp: + ifp.write(b'x' * (1048576 * 4)) + ifp.flush() + ifp.seek(0) - def wait_for_completion(self): - # put_data() is asynchronous, must wait for operation to happen. Do - # that by making RPC for some junk that must run on the thread once op - # completes. - self.conn.get_chain().call(os.getpid) - - def setUp(self): - super(ConnectionMixin, self).setUp() - self.conn = self.make_connection() - - def tearDown(self): - self.conn.close() - super(ConnectionMixin, self).tearDown() + self.conn.fetch_file(ifp.name, ofp.name) + # transfer_file() uses os.rename rather than direct data + # overwrite, so we must reopen. + with open(ofp.name, 'rb') as fp: + self.assertEquals(ifp.read(), fp.read()) -class PutDataTest(ConnectionMixin, unittest2.TestCase): +class PutDataTest(ConnectionMixin, testlib.TestCase): def test_out_path(self): path = tempfile.mktemp(prefix='mitotest') contents = mitogen.core.b('contents') @@ -91,7 +135,7 @@ class PutDataTest(ConnectionMixin, unittest2.TestCase): os.unlink(path) -class PutFileTest(ConnectionMixin, unittest2.TestCase): +class PutFileTest(ConnectionMixin, testlib.TestCase): @classmethod def setUpClass(cls): super(PutFileTest, cls).setUpClass() diff --git a/tests/ansible/tests/env_file_watcher_test.py b/tests/ansible/tests/env_file_watcher_test.py new file mode 100644 index 00000000..8803a6c2 --- /dev/null +++ b/tests/ansible/tests/env_file_watcher_test.py @@ -0,0 +1,74 @@ +import os +import sys +import tempfile + +import mock +import unittest2 +import testlib + +from mitogen.core import b +import ansible_mitogen.runner + + +klass = ansible_mitogen.runner.EnvironmentFileWatcher +environb = getattr(os, 'environb', os.environ) + + +class WatcherTest(testlib.TestCase): + def setUp(self): + self.original_env = environb.copy() + self.tf = tempfile.NamedTemporaryFile() + + def tearDown(self): + self.tf.close() + environb.clear() + environb.update(self.original_env) + + def test_missing_file(self): + # just ensure it doesn't crash + watcher = klass('/nonexistent') + watcher.check() + + def test_file_becomes_missing(self): + # just ensure it doesn't crash + watcher = klass(self.tf.name) + watcher.check() + os.unlink(self.tf.name) + watcher.check() + open(self.tf.name,'wb').close() + + def test_key_deleted(self): + environb[b('SOMEKEY')] = b('123') + self.tf.write(b('SOMEKEY=123\n')) + self.tf.flush() + watcher = klass(self.tf.name) + self.tf.seek(0) + self.tf.truncate(0) + watcher.check() + self.assertTrue(b('SOMEKEY') not in environb) + + def test_key_added(self): + watcher = klass(self.tf.name) + self.tf.write(b('SOMEKEY=123\n')) + self.tf.flush() + watcher.check() + self.assertEqual(environb[b('SOMEKEY')], b('123')) + + def test_key_shadowed_nuchange(self): + environb[b('SOMEKEY')] = b('234') + self.tf.write(b('SOMEKEY=123\n')) + self.tf.flush() + watcher = klass(self.tf.name) + watcher.check() + self.assertEqual(environb[b('SOMEKEY')], b('234')) + + def test_binary_key_added(self): + watcher = klass(self.tf.name) + self.tf.write(b('SOMEKEY=\xff\xff\xff\n')) + self.tf.flush() + watcher.check() + self.assertEqual(environb[b('SOMEKEY')], b('\xff\xff\xff')) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/bench/fork.py b/tests/bench/fork.py index b2f2382c..af5cb3a7 100644 --- a/tests/bench/fork.py +++ b/tests/bench/fork.py @@ -3,13 +3,13 @@ Measure latency of .fork() setup/teardown. """ import mitogen -import time +import mitogen.core @mitogen.main() def main(router): - t0 = time.time() + t0 = mitogen.core.now() for x in xrange(200): - t = time.time() + t = mitogen.core.now() ctx = router.fork() ctx.shutdown(wait=True) - print '++', 1000 * ((time.time() - t0) / (1.0+x)) + print '++', 1000 * ((mitogen.core.now() - t0) / (1.0+x)) diff --git a/tests/bench/large_messages.py b/tests/bench/large_messages.py index 24220023..e977e36d 100644 --- a/tests/bench/large_messages.py +++ b/tests/bench/large_messages.py @@ -4,7 +4,9 @@ import subprocess import time import socket + import mitogen +import mitogen.core @mitogen.main() @@ -15,12 +17,12 @@ def main(router): s = ' ' * n print('bytes in %.2fMiB string...' % (n/1048576.0),) - t0 = time.time() + t0 = mitogen.core.now() for x in range(10): - tt0 = time.time() + tt0 = mitogen.core.now() assert n == c.call(len, s) - print('took %dms' % (1000 * (time.time() - tt0),)) - t1 = time.time() + print('took %dms' % (1000 * (mitogen.core.now() - tt0),)) + t1 = mitogen.core.now() print('total %dms / %dms avg / %.2fMiB/sec' % ( 1000 * (t1 - t0), (1000 * (t1 - t0)) / (x + 1), diff --git a/tests/bench/latch_roundtrip.py b/tests/bench/latch_roundtrip.py index 49314fb9..1198aa48 100644 --- a/tests/bench/latch_roundtrip.py +++ b/tests/bench/latch_roundtrip.py @@ -6,6 +6,7 @@ import threading import time import mitogen +import mitogen.core import mitogen.utils import ansible_mitogen.affinity @@ -33,8 +34,8 @@ t2.start() ready.get() ready.get() -t0 = time.time() +t0 = mitogen.core.now() l1.put(None) t1.join() t2.join() -print('++', int(1e6 * ((time.time() - t0) / (1.0+X))), 'usec') +print('++', int(1e6 * ((mitogen.core.now() - t0) / (1.0+X))), 'usec') diff --git a/tests/bench/local.py b/tests/bench/local.py index 2808d803..aefeb84d 100644 --- a/tests/bench/local.py +++ b/tests/bench/local.py @@ -5,6 +5,7 @@ Measure latency of .local() setup. import time import mitogen +import mitogen.core import mitogen.utils import ansible_mitogen.affinity @@ -15,10 +16,10 @@ mitogen.utils.setup_gil() @mitogen.main() def main(router): - t0=time.time() + t0 = mitogen.core.now() for x in range(100): - t = time.time() + t = mitogen.core.now() f = router.local()# debug=True) - tt = time.time() + tt = mitogen.core.now() print(x, 1000 * (tt - t)) - print('%.03f ms' % (1000 * (time.time() - t0) / (1.0 + x))) + print('%.03f ms' % (1000 * (mitogen.core.now() - t0) / (1.0 + x))) diff --git a/tests/bench/megatime.py b/tests/bench/megatime.py index 6f5f3b5d..40cd9986 100755 --- a/tests/bench/megatime.py +++ b/tests/bench/megatime.py @@ -4,12 +4,14 @@ import sys import os import time +import mitogen.core + times = [] for x in range(5): - t0 = time.time() + t0 = mitogen.core.now() os.spawnvp(os.P_WAIT, sys.argv[1], sys.argv[1:]) - t = time.time() - t0 + t = mitogen.core.now() - t0 times.append(t) print('+++', t) diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py index 8d86d75b..8f31b1a2 100644 --- a/tests/bench/roundtrip.py +++ b/tests/bench/roundtrip.py @@ -5,6 +5,7 @@ Measure latency of local RPC. import time import mitogen +import mitogen.core import mitogen.utils import ansible_mitogen.affinity @@ -23,7 +24,7 @@ def do_nothing(): def main(router): f = router.fork() f.call(do_nothing) - t0 = time.time() + t0 = mitogen.core.now() for x in xrange(20000): f.call(do_nothing) - print('++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec') + print('++', int(1e6 * ((mitogen.core.now() - t0) / (1.0+x))), 'usec') diff --git a/tests/bench/service.py b/tests/bench/service.py index 6d866b5c..267ae3f6 100644 --- a/tests/bench/service.py +++ b/tests/bench/service.py @@ -4,8 +4,9 @@ Measure latency of local service RPC. import time -import mitogen.service import mitogen +import mitogen.core +import mitogen.service class MyService(mitogen.service.Service): @@ -17,7 +18,7 @@ class MyService(mitogen.service.Service): @mitogen.main() def main(router): f = router.fork() - t0 = time.time() + t0 = mitogen.core.now() for x in range(1000): f.call_service(service_name=MyService, method_name='ping') - print('++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec') + print('++', int(1e6 * ((mitogen.core.now() - t0) / (1.0+x))), 'usec') diff --git a/tests/bench/ssh-roundtrip.py b/tests/bench/ssh-roundtrip.py new file mode 100644 index 00000000..06c596c0 --- /dev/null +++ b/tests/bench/ssh-roundtrip.py @@ -0,0 +1,36 @@ +""" +Measure latency of SSH RPC. +""" + +import sys +import time + +import mitogen +import mitogen.core +import mitogen.utils +import ansible_mitogen.affinity + +mitogen.utils.setup_gil() +ansible_mitogen.affinity.policy.assign_worker() + +try: + xrange +except NameError: + xrange = range + +def do_nothing(): + pass + +@mitogen.main() +def main(router): + f = router.ssh(hostname=sys.argv[1]) + f.call(do_nothing) + t0 = mitogen.core.now() + end = mitogen.core.now() + 5.0 + i = 0 + while mitogen.core.now() < end: + f.call(do_nothing) + i += 1 + t1 = mitogen.core.now() + + print('++', float(1e3 * (t1 - t0) / (1.0+i)), 'ms') diff --git a/tests/bench/throughput.py b/tests/bench/throughput.py index 42604826..acb51afa 100644 --- a/tests/bench/throughput.py +++ b/tests/bench/throughput.py @@ -8,6 +8,7 @@ import tempfile import time import mitogen +import mitogen.core import mitogen.service import ansible_mitogen.affinity @@ -35,9 +36,9 @@ def run_test(router, fp, s, context): size = fp.tell() print('Testing %s...' % (s,)) context.call(prepare) - t0 = time.time() + t0 = mitogen.core.now() context.call(transfer, router.myself(), fp.name) - t1 = time.time() + t1 = mitogen.core.now() print('%s took %.2f ms to transfer %.2f MiB, %.2f MiB/s' % ( s, 1000 * (t1 - t0), size / 1048576.0, (size / (t1 - t0) / 1048576.0), diff --git a/tests/broker_test.py b/tests/broker_test.py index 23839a54..2212d8aa 100644 --- a/tests/broker_test.py +++ b/tests/broker_test.py @@ -1,4 +1,5 @@ +import time import threading import mock diff --git a/tests/buildah_test.py b/tests/buildah_test.py new file mode 100644 index 00000000..874205cd --- /dev/null +++ b/tests/buildah_test.py @@ -0,0 +1,28 @@ +import os + +import mitogen + +import unittest2 + +import testlib + + +class ConstructorTest(testlib.RouterMixin, testlib.TestCase): + def test_okay(self): + buildah_path = testlib.data_path('stubs/stub-buildah.py') + context = self.router.buildah( + container='container_name', + buildah_path=buildah_path, + ) + stream = self.router.stream_by_id(context.context_id) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[0], buildah_path) + self.assertEquals(argv[1], 'run') + self.assertEquals(argv[2], '--') + self.assertEquals(argv[3], 'container_name') + self.assertEquals(argv[4], stream.conn.options.python_path) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/connection_test.py b/tests/connection_test.py new file mode 100644 index 00000000..619594d9 --- /dev/null +++ b/tests/connection_test.py @@ -0,0 +1,77 @@ + +import os +import signal +import sys +import tempfile +import threading +import time + +import unittest2 +import testlib + +import mitogen.core +import mitogen.parent + + +class ConnectionTest(testlib.RouterMixin, testlib.TestCase): + def test_broker_shutdown_while_connect_in_progress(self): + # if Broker.shutdown() is called while a connection attempt is in + # progress, the connection should be torn down. + + path = tempfile.mktemp(prefix='broker_shutdown_sem_') + open(path, 'wb').close() + + os.environ['BROKER_SHUTDOWN_SEMAPHORE'] = path + result = [] + + def thread(): + python_path = testlib.data_path('broker_shutdown_test_python.py') + try: + result.append(self.router.local(python_path=python_path)) + except Exception: + result.append(sys.exc_info()[1]) + + th = threading.Thread(target=thread) + th.start() + + while os.path.exists(path): + time.sleep(0.05) + + self.broker.shutdown() + th.join() + + exc, = result + self.assertTrue(isinstance(exc, mitogen.parent.CancelledError)) + self.assertEquals(mitogen.parent.BROKER_SHUTDOWN_MSG, exc.args[0]) + + +@mitogen.core.takes_econtext +def do_detach(econtext): + econtext.detach() + while 1: + time.sleep(1) + logging.getLogger('mitogen').error('hi') + + +class DetachReapTest(testlib.RouterMixin, testlib.TestCase): + def test_subprocess_preserved_on_shutdown(self): + c1 = self.router.local() + pid = c1.call(os.getpid) + + l = mitogen.core.Latch() + mitogen.core.listen(c1, 'disconnect', l.put) + c1.call_no_reply(do_detach) + l.get() + + self.broker.shutdown() + self.broker.join() + + os.kill(pid, 0) # succeeds if process still alive + + # now clean up + os.kill(pid, signal.SIGTERM) + os.waitpid(pid, 0) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/serialization_test.py b/tests/context_test.py similarity index 50% rename from tests/serialization_test.py rename to tests/context_test.py index 6cf5f8b7..4bc4bd2e 100644 --- a/tests/serialization_test.py +++ b/tests/context_test.py @@ -8,40 +8,7 @@ from mitogen.core import b import testlib -class EvilObject(object): - pass - - -def roundtrip(v): - msg = mitogen.core.Message.pickled(v) - return mitogen.core.Message(data=msg.data).unpickle() - - -class EvilObjectTest(testlib.TestCase): - def test_deserialization_fails(self): - msg = mitogen.core.Message.pickled(EvilObject()) - e = self.assertRaises(mitogen.core.StreamError, - lambda: msg.unpickle() - ) - - -class BlobTest(testlib.TestCase): - klass = mitogen.core.Blob - - # Python 3 pickle protocol 2 does weird stuff depending on whether an empty - # or nonempty bytes is being serialized. For non-empty, it yields a - # _codecs.encode() call. For empty, it yields a bytes() call. - - def test_nonempty_bytes(self): - v = mitogen.core.Blob(b('dave')) - self.assertEquals(b('dave'), roundtrip(v)) - - def test_empty_bytes(self): - v = mitogen.core.Blob(b('')) - self.assertEquals(b(''), roundtrip(v)) - - -class ContextTest(testlib.RouterMixin, testlib.TestCase): +class PickleTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.core.Context # Ensure Context can be round-tripped by regular pickle in addition to diff --git a/tests/create_child_test.py b/tests/create_child_test.py new file mode 100644 index 00000000..26f10d57 --- /dev/null +++ b/tests/create_child_test.py @@ -0,0 +1,345 @@ + +import fcntl +import os +import stat +import sys +import time +import tempfile + +import mock +import unittest2 + +import mitogen.core +import mitogen.parent +from mitogen.core import b + +import testlib + + +def _osx_mode(n): + """ + fstat(2) on UNIX sockets on OSX return different mode bits depending on + which side is being inspected, so zero those bits for comparison. + """ + if sys.platform == 'darwin': + n &= ~int('0777', 8) + return n + + +def run_fd_check(func, fd, mode, on_start=None): + """ + Run ``tests/data/fd_check.py`` using `func`. The subprocess writes + information about the `fd` it received to a temporary file. + + :param func: + Function like `create_child()` used to start child. + :param fd: + FD child should read/write from, and report information about. + :param mode: + "read" or "write", depending on whether the FD is readable or writeable + from the perspective of the child. If "read", `on_start()` should write + "TEST" to it and the child reads "TEST" from it, otherwise `on_start()` + should read "TEST" from it and the child writes "TEST" to it. + :param on_start: + Function invoked as `on_start(proc)` + :returns: + Tuple of `(proc, info, on_start_result)`, where: + + * `proc`: the :class:`mitogen.parent.Process` returned by `func`. + * `info`: dict containing information returned by the child: + * `buf`: "TEST" that was read in "read" mode + * `flags`: :attr:`fcntl.F_GETFL` flags for `fd` + * `st_mode`: st_mode field from :func:`os.fstat` + * `st_dev`: st_dev field from :func:`os.fstat` + * `st_ino`: st_ino field from :func:`os.fstat` + * `ttyname`: :func:`os.ttyname` invoked on `fd`. + * `controlling_tty`: :func:os.ttyname` invoked on ``/dev/tty`` + from within the child. + """ + tf = tempfile.NamedTemporaryFile() + args = [ + sys.executable, + testlib.data_path('fd_check.py'), + tf.name, + str(fd), + mode, + ] + + proc = func(args=args) + os = None + if on_start: + os = on_start(proc) + proc.proc.wait() + try: + return proc, eval(tf.read()), os + finally: + tf.close() + + +def close_proc(proc): + proc.stdin.close() + proc.stdout.close() + if proc.stderr: + prco.stderr.close() + + +def wait_read(fp, n): + poller = mitogen.core.Poller() + try: + poller.start_receive(fp.fileno()) + for _ in poller.poll(): + return os.read(fp.fileno(), n) + assert False + finally: + poller.close() + + +class StdinSockMixin(object): + def test_stdin(self): + proc, info, _ = run_fd_check(self.func, 0, 'read', + lambda proc: proc.stdin.send(b('TEST'))) + st = os.fstat(proc.stdin.fileno()) + self.assertTrue(stat.S_ISSOCK(st.st_mode)) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, _osx_mode(info['st_mode'])) + flags = fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(info['buf'], 'TEST') + self.assertTrue(info['flags'] & os.O_RDWR) + + +class StdoutSockMixin(object): + def test_stdout(self): + proc, info, buf = run_fd_check(self.func, 1, 'write', + lambda proc: wait_read(proc.stdout, 4)) + st = os.fstat(proc.stdout.fileno()) + self.assertTrue(stat.S_ISSOCK(st.st_mode)) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, _osx_mode(info['st_mode'])) + flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + self.assertTrue(info['flags'] & os.O_RDWR) + + +class CreateChildTest(StdinSockMixin, StdoutSockMixin, testlib.TestCase): + func = staticmethod(mitogen.parent.create_child) + + def test_stderr(self): + proc, info, _ = run_fd_check(self.func, 2, 'write') + st = os.fstat(sys.stderr.fileno()) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, info['st_mode']) + self.assertEquals(st.st_ino, info['st_ino']) + + +class CreateChildMergedTest(StdinSockMixin, StdoutSockMixin, + testlib.TestCase): + def func(self, *args, **kwargs): + kwargs['merge_stdio'] = True + return mitogen.parent.create_child(*args, **kwargs) + + def test_stderr(self): + proc, info, buf = run_fd_check(self.func, 2, 'write', + lambda proc: wait_read(proc.stdout, 4)) + self.assertEquals(None, proc.stderr) + st = os.fstat(proc.stdout.fileno()) + self.assertTrue(stat.S_ISSOCK(st.st_mode)) + flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + self.assertTrue(info['flags'] & os.O_RDWR) + + +class CreateChildStderrPipeTest(StdinSockMixin, StdoutSockMixin, + testlib.TestCase): + def func(self, *args, **kwargs): + kwargs['stderr_pipe'] = True + return mitogen.parent.create_child(*args, **kwargs) + + def test_stderr(self): + proc, info, buf = run_fd_check(self.func, 2, 'write', + lambda proc: wait_read(proc.stderr, 4)) + st = os.fstat(proc.stderr.fileno()) + self.assertTrue(stat.S_ISFIFO(st.st_mode)) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, info['st_mode']) + flags = fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) + self.assertFalse(flags & os.O_WRONLY) + self.assertFalse(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + self.assertTrue(info['flags'] & os.O_WRONLY) + + +class TtyCreateChildTest(testlib.TestCase): + func = staticmethod(mitogen.parent.tty_create_child) + + def test_dev_tty_open_succeeds(self): + # In the early days of UNIX, a process that lacked a controlling TTY + # would acquire one simply by opening an existing TTY. Linux and OS X + # continue to follow this behaviour, however at least FreeBSD moved to + # requiring an explicit ioctl(). Linux supports it, but we don't yet + # use it there and anyway the behaviour will never change, so no point + # in fixing things that aren't broken. Below we test that + # getpass-loving apps like sudo and ssh get our slave PTY when they + # attempt to open /dev/tty, which is what they both do on attempting to + # read a password. + tf = tempfile.NamedTemporaryFile() + try: + proc = self.func([ + 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) + ]) + mitogen.core.set_block(proc.stdin.fileno()) + # read(3) below due to https://bugs.python.org/issue37696 + self.assertEquals(mitogen.core.b('hi\n'), proc.stdin.read(3)) + waited_pid, status = os.waitpid(proc.pid, 0) + self.assertEquals(proc.pid, waited_pid) + self.assertEquals(0, status) + self.assertEquals(mitogen.core.b(''), tf.read()) + proc.stdout.close() + finally: + tf.close() + + def test_stdin(self): + proc, info, _ = run_fd_check(self.func, 0, 'read', + lambda proc: proc.stdin.write(b('TEST'))) + st = os.fstat(proc.stdin.fileno()) + self.assertTrue(stat.S_ISCHR(st.st_mode)) + self.assertTrue(stat.S_ISCHR(info['st_mode'])) + + self.assertTrue(isinstance(info['ttyname'], + mitogen.core.UnicodeType)) + self.assertTrue(os.isatty(proc.stdin.fileno())) + + flags = fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(info['flags'] & os.O_RDWR) + self.assertTrue(info['buf'], 'TEST') + + def test_stdout(self): + proc, info, buf = run_fd_check(self.func, 1, 'write', + lambda proc: wait_read(proc.stdout, 4)) + + st = os.fstat(proc.stdout.fileno()) + self.assertTrue(stat.S_ISCHR(st.st_mode)) + self.assertTrue(stat.S_ISCHR(info['st_mode'])) + + self.assertTrue(isinstance(info['ttyname'], + mitogen.core.UnicodeType)) + self.assertTrue(os.isatty(proc.stdout.fileno())) + + flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(info['flags'] & os.O_RDWR) + + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + + def test_stderr(self): + # proc.stderr is None in the parent since there is no separate stderr + # stream. In the child, FD 2/stderr is connected to the TTY. + proc, info, buf = run_fd_check(self.func, 2, 'write', + lambda proc: wait_read(proc.stdout, 4)) + + st = os.fstat(proc.stdout.fileno()) + self.assertTrue(stat.S_ISCHR(st.st_mode)) + self.assertTrue(stat.S_ISCHR(info['st_mode'])) + + self.assertTrue(isinstance(info['ttyname'], + mitogen.core.UnicodeType)) + self.assertTrue(os.isatty(proc.stdout.fileno())) + + flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(info['flags'] & os.O_RDWR) + + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + + def test_dev_tty_open_succeeds(self): + # In the early days of UNIX, a process that lacked a controlling TTY + # would acquire one simply by opening an existing TTY. Linux and OS X + # continue to follow this behaviour, however at least FreeBSD moved to + # requiring an explicit ioctl(). Linux supports it, but we don't yet + # use it there and anyway the behaviour will never change, so no point + # in fixing things that aren't broken. Below we test that + # getpass-loving apps like sudo and ssh get our slave PTY when they + # attempt to open /dev/tty, which is what they both do on attempting to + # read a password. + tf = tempfile.NamedTemporaryFile() + try: + proc = self.func([ + 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) + ]) + self.assertEquals(mitogen.core.b('hi\n'), wait_read(proc.stdout, 3)) + waited_pid, status = os.waitpid(proc.pid, 0) + self.assertEquals(proc.pid, waited_pid) + self.assertEquals(0, status) + self.assertEquals(mitogen.core.b(''), tf.read()) + proc.stdout.close() + finally: + tf.close() + + +class StderrDiagTtyMixin(object): + def test_stderr(self): + # proc.stderr is the PTY master, FD 2 in the child is the PTY slave + proc, info, buf = run_fd_check(self.func, 2, 'write', + lambda proc: wait_read(proc.stderr, 4)) + + st = os.fstat(proc.stderr.fileno()) + self.assertTrue(stat.S_ISCHR(st.st_mode)) + self.assertTrue(stat.S_ISCHR(info['st_mode'])) + + self.assertTrue(isinstance(info['ttyname'], + mitogen.core.UnicodeType)) + self.assertTrue(os.isatty(proc.stderr.fileno())) + + flags = fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(info['flags'] & os.O_RDWR) + + self.assertTrue(flags & os.O_RDWR) + self.assertTrue(buf, 'TEST') + + +class HybridTtyCreateChildTest(StdinSockMixin, StdoutSockMixin, + StderrDiagTtyMixin, testlib.TestCase): + func = staticmethod(mitogen.parent.hybrid_tty_create_child) + + + +if 0: + # issue #410 + class SelinuxHybridTtyCreateChildTest(StderrDiagTtyMixin, testlib.TestCase): + func = staticmethod(mitogen.parent.selinux_hybrid_tty_create_child) + + def test_stdin(self): + proc, info, buf = run_fd_check(self.func, 0, 'read', + lambda proc: proc.transmit_side.write('TEST')) + st = os.fstat(proc.transmit_side.fd) + self.assertTrue(stat.S_ISFIFO(st.st_mode)) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, info['st_mode']) + flags = fcntl.fcntl(proc.transmit_side.fd, fcntl.F_GETFL) + self.assertTrue(flags & os.O_WRONLY) + self.assertTrue(buf, 'TEST') + self.assertFalse(info['flags'] & os.O_WRONLY) + self.assertFalse(info['flags'] & os.O_RDWR) + + def test_stdout(self): + proc, info, buf = run_fd_check(self.func, 1, 'write', + lambda proc: wait_read(proc.receive_side, 4)) + st = os.fstat(proc.receive_side.fd) + self.assertTrue(stat.S_ISFIFO(st.st_mode)) + self.assertEquals(st.st_dev, info['st_dev']) + self.assertEquals(st.st_mode, info['st_mode']) + flags = fcntl.fcntl(proc.receive_side.fd, fcntl.F_GETFL) + self.assertFalse(flags & os.O_WRONLY) + self.assertFalse(flags & os.O_RDWR) + self.assertTrue(info['flags'] & os.O_WRONLY) + self.assertTrue(buf, 'TEST') + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/data/broker_shutdown_test_python.py b/tests/data/broker_shutdown_test_python.py new file mode 100755 index 00000000..f1e20c16 --- /dev/null +++ b/tests/data/broker_shutdown_test_python.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# Delete a semaphore file to allow the main thread to wake up, then sleep for +# 30 seconds before starting the real Python. +import os +import time +import sys +os.unlink(os.environ['BROKER_SHUTDOWN_SEMAPHORE']) +time.sleep(30) +os.execl(sys.executable, sys.executable, *sys.argv[1:]) diff --git a/tests/data/docker/README.md b/tests/data/docker/README.md new file mode 100644 index 00000000..d3d37d52 --- /dev/null +++ b/tests/data/docker/README.md @@ -0,0 +1,7 @@ + + +# doas-debian.tar.gz + +A dynamically linked copy of the OpenBSD ``doas`` tool for Debian, port is from +https://github.com/multiplexd/doas (the slicer69 port is broken, it reads the +password from stdin). diff --git a/tests/data/docker/doas-debian.tar.gz b/tests/data/docker/doas-debian.tar.gz new file mode 100644 index 00000000..2deb72ff Binary files /dev/null and b/tests/data/docker/doas-debian.tar.gz differ diff --git a/tests/data/docker/mitogen__permdenied.profile b/tests/data/docker/mitogen__permdenied.profile new file mode 100644 index 00000000..4a2be07e --- /dev/null +++ b/tests/data/docker/mitogen__permdenied.profile @@ -0,0 +1,4 @@ + +mkdir -p bad +chmod 0 bad +cd bad diff --git a/tests/data/docker/ssh_login_banner.txt b/tests/data/docker/ssh_login_banner.txt index 1ae4cd03..8a03fbe4 100644 --- a/tests/data/docker/ssh_login_banner.txt +++ b/tests/data/docker/ssh_login_banner.txt @@ -19,3 +19,5 @@ incidents to law enforcement officials. ************************************************************** NOTE: This system is connected to DOMAIN.COM, please use your password. + +ستتم محاكمة المعتدين. هذا يختبر التدويل diff --git a/tests/data/fd_check.py b/tests/data/fd_check.py new file mode 100755 index 00000000..0a87a95e --- /dev/null +++ b/tests/data/fd_check.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import fcntl +import os +import sys + + +def ttyname(fd): + try: + t = os.ttyname(fd) + if hasattr(t, 'decode'): + t = t.decode() + return t + except OSError: + return None + + +def controlling_tty(): + try: + fp = open('/dev/tty') + try: + return ttyname(fp.fileno()) + finally: + fp.close() + except (IOError, OSError): + return None + + +fd = int(sys.argv[2]) +st = os.fstat(fd) + +if sys.argv[3] == 'write': + os.write(fd, u'TEST'.encode()) + buf = u'' +else: + buf = os.read(fd, 4).decode() + +open(sys.argv[1], 'w').write(repr({ + 'buf': buf, + 'flags': fcntl.fcntl(fd, fcntl.F_GETFL), + 'st_mode': st.st_mode, + 'st_dev': st.st_dev, + 'st_ino': st.st_ino, + 'ttyname': ttyname(fd), + 'controlling_tty': controlling_tty(), +})) diff --git a/tests/data/module_finder_testmod/__init__.py b/tests/data/importer/module_finder_testmod/__init__.py similarity index 100% rename from tests/data/module_finder_testmod/__init__.py rename to tests/data/importer/module_finder_testmod/__init__.py diff --git a/tests/data/module_finder_testmod/empty_mod.py b/tests/data/importer/module_finder_testmod/empty_mod.py similarity index 100% rename from tests/data/module_finder_testmod/empty_mod.py rename to tests/data/importer/module_finder_testmod/empty_mod.py diff --git a/tests/data/module_finder_testmod/regular_mod.py b/tests/data/importer/module_finder_testmod/regular_mod.py similarity index 100% rename from tests/data/module_finder_testmod/regular_mod.py rename to tests/data/importer/module_finder_testmod/regular_mod.py diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py similarity index 100% rename from tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py similarity index 100% rename from tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_rel_import.py similarity index 100% rename from tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_rel_import.py diff --git a/tests/data/pkg_like_plumbum/__init__.py b/tests/data/importer/pkg_like_ansible/__init__.py similarity index 100% rename from tests/data/pkg_like_plumbum/__init__.py rename to tests/data/importer/pkg_like_ansible/__init__.py diff --git a/tests/data/simple_pkg/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/__init__.py similarity index 100% rename from tests/data/simple_pkg/__init__.py rename to tests/data/importer/pkg_like_ansible/module_utils/__init__.py diff --git a/tests/data/importer/pkg_like_ansible/module_utils/distro/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/distro/__init__.py new file mode 100644 index 00000000..3a149657 --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/distro/__init__.py @@ -0,0 +1,5 @@ +# #590: a package that turns itself into a module. +I_AM = "the package that was replaced" +import sys +from pkg_like_ansible.module_utils.distro import _distro +sys.modules[__name__] = _distro diff --git a/tests/data/importer/pkg_like_ansible/module_utils/distro/_distro.py b/tests/data/importer/pkg_like_ansible/module_utils/distro/_distro.py new file mode 100644 index 00000000..9f113fef --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/distro/_distro.py @@ -0,0 +1 @@ +I_AM = "the module that replaced the package" diff --git a/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py new file mode 100644 index 00000000..f757e54c --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/__init__.py @@ -0,0 +1,5 @@ +# #590: a subpackage that turns itself into a module from elsewhere on sys.path. +I_AM = "the subpackage that was replaced with a system module" +import sys +import system_distro +sys.modules[__name__] = system_distro diff --git a/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py new file mode 100644 index 00000000..16c32b2a --- /dev/null +++ b/tests/data/importer/pkg_like_ansible/module_utils/sys_distro/_distro.py @@ -0,0 +1 @@ +I_AM = "the module inside the replaced subpackage" diff --git a/tests/data/webproject/webapp/__init__.py b/tests/data/importer/pkg_like_plumbum/__init__.py similarity index 100% rename from tests/data/webproject/webapp/__init__.py rename to tests/data/importer/pkg_like_plumbum/__init__.py diff --git a/tests/data/pkg_like_plumbum/colors.py b/tests/data/importer/pkg_like_plumbum/colors.py similarity index 100% rename from tests/data/pkg_like_plumbum/colors.py rename to tests/data/importer/pkg_like_plumbum/colors.py diff --git a/tests/data/webproject/webapp/migrations/__init__.py b/tests/data/importer/simple_pkg/__init__.py similarity index 100% rename from tests/data/webproject/webapp/migrations/__init__.py rename to tests/data/importer/simple_pkg/__init__.py diff --git a/tests/data/simple_pkg/a.py b/tests/data/importer/simple_pkg/a.py similarity index 100% rename from tests/data/simple_pkg/a.py rename to tests/data/importer/simple_pkg/a.py diff --git a/tests/data/simple_pkg/b.py b/tests/data/importer/simple_pkg/b.py similarity index 100% rename from tests/data/simple_pkg/b.py rename to tests/data/importer/simple_pkg/b.py diff --git a/tests/data/importer/simple_pkg/imports_replaces_self.py b/tests/data/importer/simple_pkg/imports_replaces_self.py new file mode 100644 index 00000000..b1b43813 --- /dev/null +++ b/tests/data/importer/simple_pkg/imports_replaces_self.py @@ -0,0 +1,6 @@ +# issue #590: this module imports a module that replaces itself in sys.modules +# during initialization. +import simple_pkg.replaces_self + +def subtract_one(n): + return simple_pkg.replaces_self.subtract_one(n) diff --git a/tests/data/simple_pkg/ping.py b/tests/data/importer/simple_pkg/ping.py similarity index 100% rename from tests/data/simple_pkg/ping.py rename to tests/data/importer/simple_pkg/ping.py diff --git a/tests/data/importer/simple_pkg/replaces_self.py b/tests/data/importer/simple_pkg/replaces_self.py new file mode 100644 index 00000000..5d853ebf --- /dev/null +++ b/tests/data/importer/simple_pkg/replaces_self.py @@ -0,0 +1,4 @@ +# issue #590: this module replaces itself in sys.modules during initialization. +import sys +import simple_pkg.b +sys.modules[__name__] = simple_pkg.b diff --git a/tests/data/six_brokenpkg/__init__.py b/tests/data/importer/six_brokenpkg/__init__.py similarity index 100% rename from tests/data/six_brokenpkg/__init__.py rename to tests/data/importer/six_brokenpkg/__init__.py diff --git a/tests/data/six_brokenpkg/_six.py b/tests/data/importer/six_brokenpkg/_six.py similarity index 100% rename from tests/data/six_brokenpkg/_six.py rename to tests/data/importer/six_brokenpkg/_six.py diff --git a/tests/data/importer/system_distro.py b/tests/data/importer/system_distro.py new file mode 100644 index 00000000..78fb1601 --- /dev/null +++ b/tests/data/importer/system_distro.py @@ -0,0 +1,2 @@ +# #590: a system module that replaces some subpackage +I_AM = "the system module that replaced the subpackage" diff --git a/tests/data/webproject/manage.py b/tests/data/importer/webproject/manage.py similarity index 100% rename from tests/data/webproject/manage.py rename to tests/data/importer/webproject/manage.py diff --git a/tests/data/webproject/serve_django_app.py b/tests/data/importer/webproject/serve_django_app.py similarity index 100% rename from tests/data/webproject/serve_django_app.py rename to tests/data/importer/webproject/serve_django_app.py diff --git a/tests/data/webproject/webproject/__init__.py b/tests/data/importer/webproject/webapp/__init__.py similarity index 100% rename from tests/data/webproject/webproject/__init__.py rename to tests/data/importer/webproject/webapp/__init__.py diff --git a/tests/data/webproject/webapp/admin.py b/tests/data/importer/webproject/webapp/admin.py similarity index 100% rename from tests/data/webproject/webapp/admin.py rename to tests/data/importer/webproject/webapp/admin.py diff --git a/tests/data/webproject/webapp/apps.py b/tests/data/importer/webproject/webapp/apps.py similarity index 100% rename from tests/data/webproject/webapp/apps.py rename to tests/data/importer/webproject/webapp/apps.py diff --git a/tests/data/importer/webproject/webapp/migrations/__init__.py b/tests/data/importer/webproject/webapp/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/webproject/webapp/models.py b/tests/data/importer/webproject/webapp/models.py similarity index 100% rename from tests/data/webproject/webapp/models.py rename to tests/data/importer/webproject/webapp/models.py diff --git a/tests/data/webproject/webapp/tests.py b/tests/data/importer/webproject/webapp/tests.py similarity index 100% rename from tests/data/webproject/webapp/tests.py rename to tests/data/importer/webproject/webapp/tests.py diff --git a/tests/data/webproject/webapp/views.py b/tests/data/importer/webproject/webapp/views.py similarity index 100% rename from tests/data/webproject/webapp/views.py rename to tests/data/importer/webproject/webapp/views.py diff --git a/tests/data/importer/webproject/webproject/__init__.py b/tests/data/importer/webproject/webproject/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/webproject/webproject/settings.py b/tests/data/importer/webproject/webproject/settings.py similarity index 100% rename from tests/data/webproject/webproject/settings.py rename to tests/data/importer/webproject/webproject/settings.py diff --git a/tests/data/webproject/webproject/urls.py b/tests/data/importer/webproject/webproject/urls.py similarity index 100% rename from tests/data/webproject/webproject/urls.py rename to tests/data/importer/webproject/webproject/urls.py diff --git a/tests/data/webproject/webproject/wsgi.py b/tests/data/importer/webproject/webproject/wsgi.py similarity index 100% rename from tests/data/webproject/webproject/wsgi.py rename to tests/data/importer/webproject/webproject/wsgi.py diff --git a/tests/data/iter_read_generator.py b/tests/data/iter_read_generator.py deleted file mode 100755 index 3fd3c08c..00000000 --- a/tests/data/iter_read_generator.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# I produce text every 100ms, for testing mitogen.core.iter_read() - -import sys -import time - - -i = 0 -while True: - i += 1 - sys.stdout.write(str(i)) - sys.stdout.flush() - time.sleep(0.1) diff --git a/tests/data/stubs/stub-buildah.py b/tests/data/stubs/stub-buildah.py new file mode 100755 index 00000000..558f117a --- /dev/null +++ b/tests/data/stubs/stub-buildah.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.environ['THIS_IS_STUB_BUILDAH'] = '1' +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 2:]) diff --git a/tests/data/stubs/stub-jexec.py b/tests/data/stubs/stub-jexec.py new file mode 100755 index 00000000..3f3e3bdc --- /dev/null +++ b/tests/data/stubs/stub-jexec.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import json +import os +import subprocess +import sys + +os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) +os.environ['THIS_IS_STUB_JEXEC'] = '1' + +# This must be a child process and not exec() since Mitogen replaces its stderr +# descriptor, causing the last user of the slave PTY to close it, resulting in +# the master side indicating EIO. +subprocess.call(sys.argv[sys.argv.index('somejail') + 1:]) +os._exit(0) diff --git a/tests/data/stubs/stub-su.py b/tests/data/stubs/stub-su.py index c32c91de..1f5e512d 100755 --- a/tests/data/stubs/stub-su.py +++ b/tests/data/stubs/stub-su.py @@ -4,6 +4,16 @@ import json import os import subprocess import sys +import time + +# #363: old input loop would fail to spot auth failure because of scheduling +# vs. su calling write() twice. +if 'DO_SLOW_AUTH_FAILURE' in os.environ: + os.write(2, u'su: '.encode()) + time.sleep(0.5) + os.write(2, u'incorrect password\n'.encode()) + os._exit(1) + os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) os.environ['THIS_IS_STUB_SU'] = '1' diff --git a/tests/data/unarchive_test.tar b/tests/data/unarchive_test.tar new file mode 100644 index 00000000..97c36407 Binary files /dev/null and b/tests/data/unarchive_test.tar differ diff --git a/tests/data/write_all_consumer.py b/tests/data/write_all_consumer.py deleted file mode 100755 index 4013ccdd..00000000 --- a/tests/data/write_all_consumer.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python -# I consume 65535 bytes every 10ms, for testing mitogen.core.write_all() - -import os -import time - -while True: - os.read(0, 65535) - time.sleep(0.01) diff --git a/tests/doas_test.py b/tests/doas_test.py index 0e27c2ab..73758476 100644 --- a/tests/doas_test.py +++ b/tests/doas_test.py @@ -2,6 +2,7 @@ import os import mitogen +import mitogen.doas import mitogen.parent import unittest2 @@ -27,5 +28,38 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals('1', context.call(os.getenv, 'THIS_IS_STUB_DOAS')) +class DoasTest(testlib.DockerMixin, testlib.TestCase): + # Only mitogen/debian-test has doas. + mitogen_test_distro = 'debian' + + def test_password_required(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + e = self.assertRaises(mitogen.core.StreamError, + lambda: self.router.doas(via=ssh) + ) + self.assertTrue(mitogen.doas.password_required_msg in str(e)) + + def test_password_incorrect(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + e = self.assertRaises(mitogen.core.StreamError, + lambda: self.router.doas(via=ssh, password='x') + ) + self.assertTrue(mitogen.doas.password_incorrect_msg in str(e)) + + def test_password_okay(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + context = self.router.doas(via=ssh, password='has_sudo_password') + self.assertEquals(0, context.call(os.getuid)) + + if __name__ == '__main__': unittest2.main() diff --git a/tests/docker_test.py b/tests/docker_test.py index 49c742ee..b5d15707 100644 --- a/tests/docker_test.py +++ b/tests/docker_test.py @@ -21,7 +21,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(argv[1], 'exec') self.assertEquals(argv[2], '--interactive') self.assertEquals(argv[3], 'container_name') - self.assertEquals(argv[4], stream.python_path) + self.assertEquals(argv[4], stream.conn.options.python_path) if __name__ == '__main__': diff --git a/tests/file_service_test.py b/tests/file_service_test.py index b9034bb1..45b621ac 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -22,15 +22,26 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): return recv, msg def test_unauthorized(self): + l1 = self.router.local() + service = self.klass(self.router) - recv, msg = self.replyable_msg() - service.fetch( - path='/etc/shadow', - sender=None, - msg=msg, + pool = mitogen.service.Pool( + router=self.router, + services=[service], + size=1, ) - e = self.assertRaises(mitogen.core.CallError, - lambda: recv.get().unpickle()) + try: + e = self.assertRaises(mitogen.core.CallError, + lambda: l1.call( + mitogen.service.FileService.get, + context=self.router.myself(), + path='/etc/shadow', + out_fp=None, + ) + ) + finally: + pool.stop() + expect = service.unregistered_msg % ('/etc/shadow',) self.assertTrue(expect in e.args[0]) @@ -85,30 +96,57 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): self._validate_response(recv.get().unpickle()) def test_prefix_authorized_abspath_bad(self): - recv = mitogen.core.Receiver(self.router) + l1 = self.router.local() + service = self.klass(self.router) service.register_prefix('/etc') - recv, msg = self.replyable_msg() - service.fetch( - path='/etc/foo/bar/../../../passwd', - sender=recv.to_sender(), - msg=msg, + + pool = mitogen.service.Pool( + router=self.router, + services=[service], + size=1, ) - self.assertEquals(None, recv.get().unpickle()) + path = '/etc/foo/bar/../../../passwd' + try: + e = self.assertRaises(mitogen.core.CallError, + lambda: l1.call( + mitogen.service.FileService.get, + context=self.router.myself(), + path=path, + out_fp=None, + ) + ) + finally: + pool.stop() + + expect = service.unregistered_msg % (path,) + self.assertTrue(expect in e.args[0]) + + def test_prefix_authorized_abspath_good(self): + l1 = self.router.local() - def test_prefix_authorized_abspath_bad(self): - recv = mitogen.core.Receiver(self.router) service = self.klass(self.router) service.register_prefix('/etc') - recv, msg = self.replyable_msg() - service.fetch( - path='/etc/../shadow', - sender=recv.to_sender(), - msg=msg, + path = '/etc/../shadow' + + pool = mitogen.service.Pool( + router=self.router, + services=[service], + size=1, ) - e = self.assertRaises(mitogen.core.CallError, - lambda: recv.get().unpickle()) - expect = service.unregistered_msg % ('/etc/../shadow',) + try: + e = self.assertRaises(mitogen.core.CallError, + lambda: l1.call( + mitogen.service.FileService.get, + context=self.router.myself(), + path=path, + out_fp=None + ) + ) + finally: + pool.stop() + + expect = service.unregistered_msg % (path,) self.assertTrue(expect in e.args[0]) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 470afc7a..53f98373 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -19,8 +19,10 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # * 3.x starting 2.7 def test_valid_syntax(self): - stream = mitogen.parent.Stream(self.router, 0, max_message_size=123) - args = stream.get_boot_command() + options = mitogen.parent.Options(max_message_size=123) + conn = mitogen.parent.Connection(options, self.router) + conn.context = mitogen.core.Context(None, 123) + args = conn.get_boot_command() # Executing the boot command will print "EC0" and expect to read from # stdin, which will fail because it's pointing at /dev/null, causing @@ -38,7 +40,8 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): ) stdout, stderr = proc.communicate() self.assertEquals(0, proc.returncode) - self.assertEquals(mitogen.parent.Stream.EC0_MARKER, stdout) + self.assertEquals(stdout, + mitogen.parent.BootstrapProtocol.EC0_MARKER+b('\n')) self.assertIn(b("Error -5 while decompressing data"), stderr) finally: fp.close() diff --git a/tests/image_prep/README.md b/tests/image_prep/README.md index d275672f..a970b319 100644 --- a/tests/image_prep/README.md +++ b/tests/image_prep/README.md @@ -11,10 +11,13 @@ code, the OS X config just has the user accounts. See ../README.md for a (mostly) description of the accounts created. + ## Building the containers ``./build_docker_images.sh`` +Requires Ansible 2.3.x.x in order to target CentOS 5 + ## Preparing an OS X box diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index dc0bbf53..65e898a1 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,5 +1,7 @@ - hosts: all + vars_files: + - shared_vars.yml strategy: linear gather_facts: false tasks: @@ -13,6 +15,8 @@ fi - hosts: all + vars_files: + - shared_vars.yml strategy: mitogen_free # Can't gather facts before here. gather_facts: true @@ -46,65 +50,103 @@ - when: ansible_virtualization_type != "docker" meta: end_play - - apt: + - name: Ensure requisite Debian packages are installed + apt: name: "{{packages.common + packages[distro][ver]}}" state: installed update_cache: true when: distro == "Debian" - - yum: + - name: Ensure requisite Red Hat packaed are installed + yum: name: "{{packages.common + packages[distro][ver]}}" state: installed update_cache: true when: distro == "CentOS" - - command: apt-get clean + - name: Clean up apt cache + command: apt-get clean when: distro == "Debian" - - command: yum clean all - when: distro == "CentOS" - - - shell: rm -rf {{item}}/* + - name: Clean up apt package lists + shell: rm -rf {{item}}/* with_items: - /var/cache/apt - /var/lib/apt/lists + when: distro == "Debian" - - copy: + - name: Clean up yum cache + command: yum clean all + when: distro == "CentOS" + + - name: Enable UTF-8 locale on Debian + copy: dest: /etc/locale.gen content: | en_US.UTF-8 UTF-8 fr_FR.UTF-8 UTF-8 when: distro == "Debian" - - shell: locale-gen + - name: Generate UTF-8 locale on Debian + shell: locale-gen when: distro == "Debian" - # Vanilla Ansible needs simplejson on CentOS 5. - - shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/ + - name: Write Unicode into /etc/environment + copy: + dest: /etc/environment + content: "UNICODE_SNOWMAN=\u2603\n" + + - name: Install prebuilt 'doas' binary + unarchive: + dest: / + src: ../data/docker/doas-debian.tar.gz + + - name: Make prebuilt 'doas' binary executable + file: + path: /usr/local/bin/doas + mode: 'u=rwxs,go=rx' + owner: root + group: root + + - name: Install doas.conf + copy: + dest: /etc/doas.conf + content: | + permit :mitogen__group + permit :root + + - name: Vanilla Ansible needs simplejson on CentOS 5. + shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/ when: distro == "CentOS" and ver == "5" - - synchronize: + - name: Vanilla Ansible needs simplejson on CentOS 5. + synchronize: dest: /usr/lib/python2.4/site-packages/simplejson/ src: ../../ansible_mitogen/compat/simplejson/ when: distro == "CentOS" and ver == "5" - - user: + - name: Set root user password and shell + user: name: root password: "{{ 'rootpassword' | password_hash('sha256') }}" shell: /bin/bash - - file: + - name: Ensure /var/run/sshd exists + file: path: /var/run/sshd state: directory - - command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key + - name: Generate SSH host key + command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key args: creates: /etc/ssh/ssh_host_rsa_key - - group: + - name: Ensure correct sudo group exists + group: name: "{{sudo_group[distro]}}" - - copy: + - name: Ensure /etc/sentinel exists + copy: dest: /etc/sentinel content: | i-am-mitogen-test-docker-image @@ -119,7 +161,8 @@ path: /etc/sudoers.d mode: 'u=rwx,go=' - - blockinfile: + - name: Install test-related sudo rules + blockinfile: path: /etc/sudoers block: | # https://www.toofishes.net/blog/trouble-sudoers-or-last-entry-wins/ @@ -131,31 +174,36 @@ Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw - # Prevent permission denied errors. - - file: + - name: Prevent permission denied errors. + file: path: /etc/sudoers.d/README state: absent - - lineinfile: + - name: Install CentOS wheel sudo rule + lineinfile: path: /etc/sudoers line: "%wheel ALL=(ALL) ALL" when: distro == "CentOS" - - lineinfile: + - name: Enable SSH banner + lineinfile: path: /etc/ssh/sshd_config line: Banner /etc/ssh/banner.txt - - lineinfile: + - name: Allow remote SSH root login + lineinfile: path: /etc/ssh/sshd_config line: PermitRootLogin yes regexp: '.*PermitRootLogin.*' - - lineinfile: + - name: Allow remote SSH root login + lineinfile: path: /etc/pam.d/sshd regexp: '.*session.*required.*pam_loginuid.so' line: session optional pam_loginuid.so - - copy: + - name: Install convenience script for running an straced Python + copy: mode: 'u+rwx,go=rx' dest: /usr/local/bin/pywrap content: | diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index a5b63c13..fbefd9c3 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -5,6 +5,8 @@ # - hosts: all + vars_files: + - shared_vars.yml gather_facts: true strategy: mitogen_free become: true @@ -20,6 +22,7 @@ - readonly_homedir - require_tty - require_tty_pw_required + - permdenied - slow_user - webapp - sudo1 @@ -72,14 +75,18 @@ - user: name: "mitogen__{{item}}" shell: /bin/bash - groups: "{{user_groups[item]|default(['mitogen__group'])}}" + groups: | + {{ + ['com.apple.access_ssh'] + + (user_groups[item] | default(['mitogen__group'])) + }} password: "{{item}}_password" with_items: "{{all_users}}" when: ansible_system == 'Darwin' - - name: Hide users from login window. - with_items: "{{all_users}}" + - name: Hide users from login window (Darwin). when: ansible_system == 'Darwin' + with_items: "{{all_users}}" osx_defaults: array_add: true domain: /Library/Preferences/com.apple.loginwindow @@ -87,6 +94,26 @@ key: HiddenUsersList value: ['mitogen_{{item}}'] + - name: Check if AccountsService is used + stat: + path: /var/lib/AccountsService/users + register: out + + - name: Hide users from login window (Linux). + when: ansible_system == 'Linux' and out.stat.exists + with_items: "{{all_users}}" + copy: + dest: /var/lib/AccountsService/users/mitogen__{{item}} + content: | + [User] + SystemAccount=true + + - name: Restart AccountsService (Linux). + when: ansible_system == 'Linux' and out.stat.exists + service: + name: accounts-daemon + restarted: true + - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" @@ -98,6 +125,14 @@ - bashrc - profile + - name: "Login throws permission denied errors (issue #271)" + copy: + dest: ~mitogen__permdenied/.{{item}} + src: ../data/docker/mitogen__permdenied.profile + with_items: + - bashrc + - profile + - name: Install pubkey for mitogen__has_sudo_pubkey block: - file: diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg index 8a8c47fa..60f2975e 100644 --- a/tests/image_prep/ansible.cfg +++ b/tests/image_prep/ansible.cfg @@ -4,3 +4,4 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy retry_files_enabled = false display_args_to_stdout = True no_target_syslog = True +host_key_checking = False diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py index 9fc89c05..76564297 100755 --- a/tests/image_prep/build_docker_images.py +++ b/tests/image_prep/build_docker_images.py @@ -26,7 +26,13 @@ label_by_id = {} for base_image, label in [ ('astj/centos5-vault', 'centos5'), # Python 2.4.3 - ('debian:stretch', 'debian'), # Python 2.7.13, 3.5.3 + # Debian containers later than debuerreotype/debuerreotype#48 no longer + # ship a stub 'initctl', causing (apparently) the Ansible service + # module run at the end of DebOps to trigger a full stop/start of SSHd. + # When SSHd is killed, Docker responds by destroying the container. + # Proper solution is to include a full /bin/init; Docker --init doesn't + # help. In the meantime, just use a fixed older version. + ('debian:stretch-20181112', 'debian'), # Python 2.7.13, 3.5.3 ('centos:6', 'centos6'), # Python 2.6.6 ('centos:7', 'centos7') # Python 2.7.5 ]: diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index 760da0f6..2c37c6bb 100644 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -1,14 +1,3 @@ -- hosts: all - gather_facts: false - tasks: - - set_fact: - # Hacktacular.. but easiest place for it with current structure. - sudo_group: - MacOSX: admin - Debian: sudo - Ubuntu: sudo - CentOS: wheel - - include: _container_setup.yml - include: _user_accounts.yml diff --git a/tests/image_prep/shared_vars.yml b/tests/image_prep/shared_vars.yml new file mode 100644 index 00000000..4be7babe --- /dev/null +++ b/tests/image_prep/shared_vars.yml @@ -0,0 +1,5 @@ +sudo_group: + MacOSX: admin + Debian: sudo + Ubuntu: sudo + CentOS: wheel diff --git a/tests/importer_test.py b/tests/importer_test.py index fc6f4bd6..c796f7d0 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -12,6 +12,7 @@ import mitogen.utils from mitogen.core import b import testlib +import simple_pkg.imports_replaces_self class ImporterMixin(testlib.RouterMixin): @@ -214,5 +215,13 @@ class Python24LineCacheTest(testlib.TestCase): pass +class SelfReplacingModuleTest(testlib.RouterMixin, testlib.TestCase): + # issue #590 + def test_importer_handles_self_replacement(self): + c = self.router.local() + self.assertEquals(0, + c.call(simple_pkg.imports_replaces_self.subtract_one, 1)) + + if __name__ == '__main__': unittest2.main() diff --git a/tests/iter_split_test.py b/tests/iter_split_test.py new file mode 100644 index 00000000..ee5e97d9 --- /dev/null +++ b/tests/iter_split_test.py @@ -0,0 +1,66 @@ + +import mock +import unittest2 + +import mitogen.core + +import testlib + +try: + next +except NameError: + def next(it): + return it.next() + + +class IterSplitTest(unittest2.TestCase): + func = staticmethod(mitogen.core.iter_split) + + def test_empty_buffer(self): + lst = [] + trailer, cont = self.func(buf='', delim='\n', func=lst.append) + self.assertTrue(cont) + self.assertEquals('', trailer) + self.assertEquals([], lst) + + def test_empty_line(self): + lst = [] + trailer, cont = self.func(buf='\n', delim='\n', func=lst.append) + self.assertTrue(cont) + self.assertEquals('', trailer) + self.assertEquals([''], lst) + + def test_one_line(self): + buf = 'xxxx\n' + lst = [] + trailer, cont = self.func(buf=buf, delim='\n', func=lst.append) + self.assertTrue(cont) + self.assertEquals('', trailer) + self.assertEquals(lst, ['xxxx']) + + def test_one_incomplete(self): + buf = 'xxxx\nyy' + lst = [] + trailer, cont = self.func(buf=buf, delim='\n', func=lst.append) + self.assertTrue(cont) + self.assertEquals('yy', trailer) + self.assertEquals(lst, ['xxxx']) + + def test_returns_false_immediately(self): + buf = 'xxxx\nyy' + func = lambda buf: False + trailer, cont = self.func(buf=buf, delim='\n', func=func) + self.assertFalse(cont) + self.assertEquals('yy', trailer) + + def test_returns_false_second_call(self): + buf = 'xxxx\nyy\nzz' + it = iter([True, False]) + func = lambda buf: next(it) + trailer, cont = self.func(buf=buf, delim='\n', func=func) + self.assertFalse(cont) + self.assertEquals('zz', trailer) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/jail_test.py b/tests/jail_test.py new file mode 100644 index 00000000..7239d32f --- /dev/null +++ b/tests/jail_test.py @@ -0,0 +1,33 @@ + +import os + +import mitogen +import mitogen.parent + +import unittest2 + +import testlib + + +class ConstructorTest(testlib.RouterMixin, testlib.TestCase): + jexec_path = testlib.data_path('stubs/stub-jexec.py') + + def test_okay(self): + context = self.router.jail( + jexec_path=self.jexec_path, + container='somejail', + ) + stream = self.router.stream_by_id(context.context_id) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[:4], [ + self.jexec_path, + 'somejail', + stream.conn.options.python_path, + '-c', + ]) + self.assertEquals('1', context.call(os.getenv, 'THIS_IS_STUB_JEXEC')) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/log_handler_test.py b/tests/log_handler_test.py new file mode 100644 index 00000000..c5d257a9 --- /dev/null +++ b/tests/log_handler_test.py @@ -0,0 +1,90 @@ + +import logging +import mock + +import unittest2 +import testlib +import mitogen.core +import mitogen.master +import mitogen.parent +import mitogen.utils +from mitogen.core import b + + +def ping(): + pass + + +class BufferingTest(testlib.TestCase): + klass = mitogen.core.LogHandler + + def record(self): + return logging.LogRecord( + name='name', + level=99, + pathname='pathname', + lineno=123, + msg='msg', + args=(), + exc_info=None, + ) + + def build(self): + context = mock.Mock() + return context, self.klass(context) + + def test_initially_buffered(self): + context, handler = self.build() + rec = self.record() + handler.emit(rec) + self.assertEquals(0, context.send.call_count) + self.assertEquals(1, len(handler._buffer)) + + def test_uncork(self): + context, handler = self.build() + rec = self.record() + handler.emit(rec) + handler.uncork() + + self.assertEquals(1, context.send.call_count) + self.assertEquals(None, handler._buffer) + + _, args, _ = context.send.mock_calls[0] + msg, = args + + self.assertEquals(mitogen.core.FORWARD_LOG, msg.handle) + self.assertEquals(b('name\x0099\x00msg'), msg.data) + + +class StartupTest(testlib.RouterMixin, testlib.TestCase): + def test_earliest_messages_logged(self): + log = testlib.LogCapturer() + log.start() + + c1 = self.router.local() + c1.shutdown(wait=True) + + logs = log.stop() + self.assertTrue('Python version is' in logs) + self.assertTrue('Parent is context 0 (master)' in logs) + + def test_earliest_messages_logged_via(self): + c1 = self.router.local(name='c1') + # ensure any c1-related msgs are processed before beginning capture. + c1.call(ping) + + log = testlib.LogCapturer() + log.start() + + c2 = self.router.local(via=c1, name='c2', debug=True) + c2.shutdown(wait=True) + + logs = log.stop() + self.assertTrue('Python version is' in logs) + + expect = 'Parent is context %s (%s)' % (c1.context_id, 'parent') + self.assertTrue(expect in logs) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/lxc_test.py b/tests/lxc_test.py index ae5990f6..f78846ff 100644 --- a/tests/lxc_test.py +++ b/tests/lxc_test.py @@ -38,7 +38,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): lxc_attach_path='true', ) ) - self.assertTrue(str(e).endswith(mitogen.lxc.Stream.eof_error_hint)) + self.assertTrue(str(e).endswith(mitogen.lxc.Connection.eof_error_hint)) if __name__ == '__main__': diff --git a/tests/lxd_test.py b/tests/lxd_test.py index e59da43c..c80f8251 100644 --- a/tests/lxd_test.py +++ b/tests/lxd_test.py @@ -30,7 +30,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): lxc_path='true', ) ) - self.assertTrue(str(e).endswith(mitogen.lxd.Stream.eof_error_hint)) + self.assertTrue(str(e).endswith(mitogen.lxd.Connection.eof_error_hint)) if __name__ == '__main__': diff --git a/tests/message_test.py b/tests/message_test.py new file mode 100644 index 00000000..79deb2c6 --- /dev/null +++ b/tests/message_test.py @@ -0,0 +1,545 @@ + +import sys +import struct + +import mock +import unittest2 + +import mitogen.core +import mitogen.master +import testlib + +from mitogen.core import b + + +class ConstructorTest(testlib.TestCase): + klass = mitogen.core.Message + + def test_dst_id_default(self): + self.assertEquals(self.klass().dst_id, None) + + def test_dst_id_explicit(self): + self.assertEquals(self.klass(dst_id=1111).dst_id, 1111) + + @mock.patch('mitogen.context_id', 1234) + def test_src_id_default(self): + self.assertEquals(self.klass().src_id, 1234) + + def test_src_id_explicit(self): + self.assertEquals(self.klass(src_id=4321).src_id, 4321) + + @mock.patch('mitogen.context_id', 5555) + def test_auth_id_default(self): + self.assertEquals(self.klass().auth_id, 5555) + + def test_auth_id_explicit(self): + self.assertEquals(self.klass(auth_id=2222).auth_id, 2222) + + def test_handle_default(self): + self.assertEquals(self.klass().handle, None) + + def test_handle_explicit(self): + self.assertEquals(self.klass(handle=1234).handle, 1234) + + def test_reply_to_default(self): + self.assertEquals(self.klass().reply_to, None) + + def test_reply_to_explicit(self): + self.assertEquals(self.klass(reply_to=8888).reply_to, 8888) + + def test_data_default(self): + m = self.klass() + self.assertEquals(m.data, b('')) + self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + + def test_data_explicit(self): + m = self.klass(data=b('asdf')) + self.assertEquals(m.data, b('asdf')) + self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + + def test_data_hates_unicode(self): + self.assertRaises(Exception, + lambda: self.klass(data=u'asdf')) + + +class PackTest(testlib.TestCase): + klass = mitogen.core.Message + + def test_header_format_sanity(self): + self.assertEquals(self.klass.HEADER_LEN, + struct.calcsize(self.klass.HEADER_FMT)) + + def test_header_length_correct(self): + s = self.klass(dst_id=123, handle=123).pack() + self.assertEquals(len(s), self.klass.HEADER_LEN) + + def test_magic(self): + s = self.klass(dst_id=123, handle=123).pack() + magic, = struct.unpack('>h', s[:2]) + self.assertEquals(self.klass.HEADER_MAGIC, magic) + + def test_dst_id(self): + s = self.klass(dst_id=123, handle=123).pack() + dst_id, = struct.unpack('>L', s[2:6]) + self.assertEquals(123, dst_id) + + def test_src_id(self): + s = self.klass(src_id=5432, dst_id=123, handle=123).pack() + src_id, = struct.unpack('>L', s[6:10]) + self.assertEquals(5432, src_id) + + def test_auth_id(self): + s = self.klass(auth_id=1919, src_id=5432, dst_id=123, handle=123).pack() + auth_id, = struct.unpack('>L', s[10:14]) + self.assertEquals(1919, auth_id) + + def test_handle(self): + s = self.klass(dst_id=123, handle=9999).pack() + handle, = struct.unpack('>L', s[14:18]) + self.assertEquals(9999, handle) + + def test_reply_to(self): + s = self.klass(dst_id=1231, handle=7777, reply_to=9132).pack() + reply_to, = struct.unpack('>L', s[18:22]) + self.assertEquals(9132, reply_to) + + def test_data_length_empty(self): + s = self.klass(dst_id=1231, handle=7777).pack() + data_length, = struct.unpack('>L', s[22:26]) + self.assertEquals(0, data_length) + + def test_data_length_present(self): + s = self.klass(dst_id=1231, handle=7777, data=b('hello')).pack() + data_length, = struct.unpack('>L', s[22:26]) + self.assertEquals(5, data_length) + + def test_data_empty(self): + s = self.klass(dst_id=1231, handle=7777).pack() + data = s[26:] + self.assertEquals(b(''), data) + + def test_data_present(self): + s = self.klass(dst_id=11, handle=77, data=b('hello')).pack() + data = s[26:] + self.assertEquals(b('hello'), data) + + +class IsDeadTest(testlib.TestCase): + klass = mitogen.core.Message + + def test_is_dead(self): + msg = self.klass(reply_to=mitogen.core.IS_DEAD) + self.assertTrue(msg.is_dead) + + def test_is_not_dead(self): + msg = self.klass(reply_to=5555) + self.assertFalse(msg.is_dead) + + +class DeadTest(testlib.TestCase): + klass = mitogen.core.Message + + def test_no_reason(self): + msg = self.klass.dead() + self.assertEquals(msg.reply_to, mitogen.core.IS_DEAD) + self.assertTrue(msg.is_dead) + self.assertEquals(msg.data, b('')) + + def test_with_reason(self): + msg = self.klass.dead(reason=u'oh no') + self.assertEquals(msg.reply_to, mitogen.core.IS_DEAD) + self.assertTrue(msg.is_dead) + self.assertEquals(msg.data, b('oh no')) + + +class EvilObject(object): + pass + + +class PickledTest(testlib.TestCase): + # getting_started.html#rpc-serialization-rules + klass = mitogen.core.Message + + def roundtrip(self, v, router=None): + msg = self.klass.pickled(v) + msg2 = self.klass(data=msg.data) + msg2.router = router + return msg2.unpickle() + + def test_bool(self): + for b in True, False: + self.assertEquals(b, self.roundtrip(b)) + + @unittest2.skipIf(condition=sys.version_info < (2, 6), + reason='bytearray missing on <2.6') + def test_bytearray(self): + ba = bytearray(b('123')) + self.assertRaises(mitogen.core.StreamError, + lambda: self.roundtrip(ba) + ) + + def test_bytes(self): + by = b('123') + self.assertEquals(by, self.roundtrip(by)) + + def test_dict(self): + d = {1: 2, u'a': 3, b('b'): 4, 'c': {}} + roundtrip = self.roundtrip(d) + self.assertEquals(d, roundtrip) + self.assertTrue(isinstance(roundtrip, dict)) + for k in d: + self.assertTrue(isinstance(roundtrip[k], type(d[k]))) + + def test_int(self): + self.assertEquals(123, self.klass.pickled(123).unpickle()) + + def test_list(self): + l = [1, u'b', b('c')] + roundtrip = self.roundtrip(l) + self.assertTrue(isinstance(roundtrip, list)) + self.assertEquals(l, roundtrip) + for k in range(len(l)): + self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + + @unittest2.skipIf(condition=sys.version_info > (3, 0), + reason='long missing in >3.x') + def test_long(self): + l = long(0xffffffffffff) + roundtrip = self.roundtrip(l) + self.assertEquals(l, roundtrip) + self.assertTrue(isinstance(roundtrip, long)) + + def test_tuple(self): + l = (1, u'b', b('c')) + roundtrip = self.roundtrip(l) + self.assertEquals(l, roundtrip) + self.assertTrue(isinstance(roundtrip, tuple)) + for k in range(len(l)): + self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + + def test_unicode(self): + u = u'abcd' + roundtrip = self.roundtrip(u) + self.assertEquals(u, roundtrip) + self.assertTrue(isinstance(roundtrip, mitogen.core.UnicodeType)) + + #### custom types. see also: types_test.py, call_error_test.py + + # Python 3 pickle protocol 2 does weird stuff depending on whether an empty + # or nonempty bytes is being serialized. For non-empty, it yields a + # _codecs.encode() call. For empty, it yields a bytes() call. + + def test_blob_nonempty(self): + v = mitogen.core.Blob(b('dave')) + roundtrip = self.roundtrip(v) + self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) + self.assertEquals(b('dave'), roundtrip) + + def test_blob_empty(self): + v = mitogen.core.Blob(b('')) + roundtrip = self.roundtrip(v) + self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) + self.assertEquals(b(''), v) + + def test_secret_nonempty(self): + s = mitogen.core.Secret(u'dave') + roundtrip = self.roundtrip(s) + self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) + self.assertEquals(u'dave', roundtrip) + + def test_secret_empty(self): + s = mitogen.core.Secret(u'') + roundtrip = self.roundtrip(s) + self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) + self.assertEquals(u'', roundtrip) + + def test_call_error(self): + ce = mitogen.core.CallError('nope') + ce2 = self.assertRaises(mitogen.core.CallError, + lambda: self.roundtrip(ce)) + self.assertEquals(ce.args[0], ce2.args[0]) + + def test_context(self): + router = mitogen.master.Router() + try: + c = router.context_by_id(1234) + roundtrip = self.roundtrip(c) + self.assertTrue(isinstance(roundtrip, mitogen.core.Context)) + self.assertEquals(c.context_id, 1234) + finally: + router.broker.shutdown() + router.broker.join() + + def test_sender(self): + router = mitogen.master.Router() + try: + recv = mitogen.core.Receiver(router) + sender = recv.to_sender() + roundtrip = self.roundtrip(sender, router=router) + self.assertTrue(isinstance(roundtrip, mitogen.core.Sender)) + self.assertEquals(roundtrip.context.context_id, mitogen.context_id) + self.assertEquals(roundtrip.dst_handle, sender.dst_handle) + finally: + router.broker.shutdown() + router.broker.join() + + #### + + def test_custom_object_deserialization_fails(self): + self.assertRaises(mitogen.core.StreamError, + lambda: self.roundtrip(EvilObject()) + ) + + +class ReplyTest(testlib.TestCase): + # getting_started.html#rpc-serialization-rules + klass = mitogen.core.Message + + def test_reply_calls_router_route(self): + msg = self.klass(src_id=1234, reply_to=9191) + router = mock.Mock() + msg.reply(123, router=router) + self.assertEquals(1, router.route.call_count) + + def test_reply_pickles_object(self): + msg = self.klass(src_id=1234, reply_to=9191) + router = mock.Mock() + msg.reply(123, router=router) + _, (reply,), _ = router.route.mock_calls[0] + self.assertEquals(reply.dst_id, 1234) + self.assertEquals(reply.unpickle(), 123) + + def test_reply_uses_preformatted_message(self): + msg = self.klass(src_id=1234, reply_to=9191) + router = mock.Mock() + my_reply = mitogen.core.Message.pickled(4444) + msg.reply(my_reply, router=router) + _, (reply,), _ = router.route.mock_calls[0] + self.assertTrue(my_reply is reply) + self.assertEquals(reply.dst_id, 1234) + self.assertEquals(reply.unpickle(), 4444) + + def test_reply_sets_dst_id(self): + msg = self.klass(src_id=1234, reply_to=9191) + router = mock.Mock() + msg.reply(123, router=router) + _, (reply,), _ = router.route.mock_calls[0] + self.assertEquals(reply.dst_id, 1234) + + def test_reply_sets_handle(self): + msg = self.klass(src_id=1234, reply_to=9191) + router = mock.Mock() + msg.reply(123, router=router) + _, (reply,), _ = router.route.mock_calls[0] + self.assertEquals(reply.handle, 9191) + + +class UnpickleTest(testlib.TestCase): + # mostly done by PickleTest, just check behaviour of parameters + klass = mitogen.core.Message + + def test_throw(self): + ce = mitogen.core.CallError('nope') + m = self.klass.pickled(ce) + ce2 = self.assertRaises(mitogen.core.CallError, + lambda: m.unpickle()) + self.assertEquals(ce.args[0], ce2.args[0]) + + def test_no_throw(self): + ce = mitogen.core.CallError('nope') + m = self.klass.pickled(ce) + ce2 = m.unpickle(throw=False) + self.assertEquals(ce.args[0], ce2.args[0]) + + def test_throw_dead(self): + m = self.klass.pickled('derp', reply_to=mitogen.core.IS_DEAD) + self.assertRaises(mitogen.core.ChannelError, + lambda: m.unpickle()) + + def test_no_throw_dead(self): + m = self.klass.pickled('derp', reply_to=mitogen.core.IS_DEAD) + self.assertEquals('derp', m.unpickle(throw_dead=False)) + + +class UnpickleCompatTest(testlib.TestCase): + # try weird variations of pickles from different Python versions. + klass = mitogen.core.Message + + def check(self, value, encoded, **kwargs): + if isinstance(encoded, mitogen.core.UnicodeType): + encoded = encoded.encode('latin1') + m = self.klass(data=encoded) + m.router = mitogen.master.Router() + try: + return m.unpickle(**kwargs) + finally: + m.router.broker.shutdown() + m.router.broker.join() + + def test_py24_bytes(self): + self.check('test', + ('\x80\x02U\x04testq\x00.')) + + def test_py24_unicode(self): + self.check(u'test', + ('\x80\x02X\x04\x00\x00\x00testq\x00.')) + + def test_py24_int(self): + self.check(123, + ('\x80\x02K{.')) + + def test_py24_long(self): + self.check(17592186044415, + ('\x80\x02\x8a\x06\xff\xff\xff\xff\xff\x0f.')) + + def test_py24_dict(self): + self.check({}, + ('\x80\x02}q\x00.')) + + def test_py24_tuple(self): + self.check((1, 2, u'b'), + ('\x80\x02K\x01K\x02X\x01\x00\x00\x00bq\x00\x87q\x01.')) + + def test_py24_bool(self): + self.check(True, + ('\x80\x02\x88.')) + + def test_py24_list(self): + self.check([1, 2, u'b'], + ('\x80\x02]q\x00(K\x01K\x02X\x01\x00\x00\x00bq\x01e.')) + + def test_py24_blob(self): + self.check(mitogen.core.mitogen.core.Blob(b('bigblob')), + ('\x80\x02cmitogen.core\nBlob\nq\x00U\x07bigblobq\x01\x85q\x02Rq\x03.')) + + def test_py24_secret(self): + self.check(mitogen.core.Secret(u'mypassword'), + ('\x80\x02cmitogen.core\nSecret\nq\x00X\n\x00\x00\x00mypasswordq\x01\x85q\x02Rq\x03.')) + + def test_py24_call_error(self): + self.check(mitogen.core.CallError('big error'), + ('\x80\x02cmitogen.core\n_unpickle_call_error\nq\x00X\t\x00\x00\x00big errorq\x01\x85q\x02R.'), throw=False) + + def test_py24_context(self): + self.check(mitogen.core.Context(1234, None), + ('\x80\x02cmitogen.core\n_unpickle_context\nq\x00M\xd2\x04N\x86q\x01Rq\x02.')) + + def test_py24_sender(self): + self.check(mitogen.core.Sender(mitogen.core.Context(55555, None), 4444), + ('\x80\x02cmitogen.core\n_unpickle_sender\nq\x00M\x03\xd9M\\\x11\x86q\x01Rq\x02.')) + + def test_py27_bytes(self): + self.check(b('test'), + ('\x80\x02U\x04testq\x01.')) + + def test_py27_unicode(self): + self.check(u'test', + ('\x80\x02X\x04\x00\x00\x00testq\x01.')) + + def test_py27_int(self): + self.check(123, + ('\x80\x02K{.')) + + def test_py27_long(self): + self.check(17592186044415, + ('\x80\x02\x8a\x06\xff\xff\xff\xff\xff\x0f.')) + + def test_py27_dict(self): + self.check({}, + ('\x80\x02}q\x01.')) + + def test_py27_tuple(self): + self.check((1, 2, u'b'), + ('\x80\x02K\x01K\x02X\x01\x00\x00\x00b\x87q\x01.')) + + def test_py27_bool(self): + self.check(True, + ('\x80\x02\x88.')) + + def test_py27_list(self): + self.check([1, 2, u'b'], + ('\x80\x02]q\x01(K\x01K\x02X\x01\x00\x00\x00be.')) + + def test_py27_blob(self): + self.check(mitogen.core.mitogen.core.Blob(b('bigblob')), + ('\x80\x02cmitogen.core\nBlob\nq\x01U\x07bigblob\x85Rq\x02.')) + + def test_py27_secret(self): + self.check(mitogen.core.Secret(u'mypassword'), + ('\x80\x02cmitogen.core\nSecret\nq\x01X\n\x00\x00\x00mypassword\x85Rq\x02.')) + + def test_py27_call_error(self): + self.check(mitogen.core.CallError(u'big error',), + ('\x80\x02cmitogen.core\n_unpickle_call_error\nq\x01X\t\x00\x00\x00big errorq\x02\x85Rq\x03.'), throw=False) + + def test_py27_context(self): + self.check(mitogen.core.Context(1234, None), + ('\x80\x02cmitogen.core\n_unpickle_context\nq\x01M\xd2\x04N\x86Rq\x02.')) + + def test_py27_sender(self): + self.check(mitogen.core.Sender(mitogen.core.Context(55555, None), 4444), + ('\x80\x02cmitogen.core\n_unpickle_sender\nq\x01M\x03\xd9M\\\x11\x86Rq\x02.')) + + def test_py36_bytes(self): + self.check(b('test'), + ('\x80\x02c_codecs\nencode\nq\x00X\x04\x00\x00\x00testq\x01X\x06\x00\x00\x00latin1q\x02\x86q\x03Rq\x04.')) + + def test_py36_unicode(self): + self.check('test', + ('\x80\x02X\x04\x00\x00\x00testq\x00.')) + + def test_py36_int(self): + self.check(123, + ('\x80\x02K{.')) + + def test_py36_long(self): + self.check(17592186044415, + ('\x80\x02\x8a\x06\xff\xff\xff\xff\xff\x0f.')) + + def test_py36_dict(self): + self.check({}, + ('\x80\x02}q\x00.')) + + def test_py36_tuple(self): + self.check((1, 2, u'b'), + ('\x80\x02K\x01K\x02X\x01\x00\x00\x00bq\x00\x87q\x01.')) + + def test_py36_bool(self): + self.check(True, + ('\x80\x02\x88.')) + + def test_py36_list(self): + self.check([1, 2, u'b'], + ('\x80\x02]q\x00(K\x01K\x02X\x01\x00\x00\x00bq\x01e.')) + + def test_py36_blob(self): + self.check(mitogen.core.mitogen.core.Blob(b('bigblob')), + ('\x80\x02cmitogen.core\nBlob\nq\x00c_codecs\nencode\nq\x01X\x07\x00\x00\x00bigblobq\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06Rq\x07.')) + + def test_py36_secret(self): + self.check(mitogen.core.Secret('mypassword'), + ('\x80\x02cmitogen.core\nSecret\nq\x00X\n\x00\x00\x00mypasswordq\x01\x85q\x02Rq\x03.')) + + def test_py36_call_error(self): + self.check(mitogen.core.CallError('big error'), + ('\x80\x02cmitogen.core\n_unpickle_call_error\nq\x00X\t\x00\x00\x00big errorq\x01\x85q\x02Rq\x03.'), throw=False) + + def test_py36_context(self): + self.check(mitogen.core.Context(1234, None), + ('\x80\x02cmitogen.core\n_unpickle_context\nq\x00M\xd2\x04N\x86q\x01Rq\x02.')) + + def test_py36_sender(self): + self.check(mitogen.core.Sender(mitogen.core.Context(55555, None), 4444), + ('\x80\x02cmitogen.core\n_unpickle_sender\nq\x00M\x03\xd9M\\\x11\x86q\x01Rq\x02.')) + + +class ReprTest(testlib.TestCase): + klass = mitogen.core.Message + + def test_repr(self): + # doesn't crash + repr(self.klass.pickled('test')) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/mitogen_protocol_test.py b/tests/mitogen_protocol_test.py new file mode 100644 index 00000000..834fb437 --- /dev/null +++ b/tests/mitogen_protocol_test.py @@ -0,0 +1,34 @@ + +import unittest2 +import mock + +import mitogen.core + +import testlib + + +class ReceiveOneTest(testlib.TestCase): + klass = mitogen.core.MitogenProtocol + + def test_corruption(self): + broker = mock.Mock() + router = mock.Mock() + stream = mock.Mock() + + protocol = self.klass(router, 1) + protocol.stream = stream + + junk = mitogen.core.b('x') * mitogen.core.Message.HEADER_LEN + + capture = testlib.LogCapturer() + capture.start() + protocol.on_receive(broker, junk) + capture.stop() + + self.assertEquals(1, stream.on_disconnect.call_count) + expect = self.klass.corrupt_msg % (stream.name, junk) + self.assertTrue(expect in capture.raw()) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 409adc6d..fc3a17de 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -5,8 +5,10 @@ import sys import unittest2 import mitogen.master +from mitogen.core import b import testlib +from testlib import MODS_DIR class ConstructorTest(testlib.TestCase): @@ -51,10 +53,10 @@ class IsStdlibNameTest(testlib.TestCase): class GetMainModuleDefectivePython3x(testlib.TestCase): - klass = mitogen.master.ModuleFinder + klass = mitogen.master.DefectivePython3xMainMethod def call(self, fullname): - return self.klass()._get_main_module_defective_python_3x(fullname) + return self.klass().find(fullname) def test_builtin(self): self.assertEquals(None, self.call('sys')) @@ -77,23 +79,23 @@ class GetMainModuleDefectivePython3x(testlib.TestCase): self.assertFalse(is_pkg) -class GetModuleViaPkgutilTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder +class PkgutilMethodTest(testlib.TestCase): + klass = mitogen.master.PkgutilMethod def call(self, fullname): - return self.klass()._get_module_via_pkgutil(fullname) + return self.klass().find(fullname) def test_empty_source_pkg(self): path, src, is_pkg = self.call('module_finder_testmod') self.assertEquals(path, - testlib.data_path('module_finder_testmod/__init__.py')) + os.path.join(MODS_DIR, 'module_finder_testmod/__init__.py')) self.assertEquals(mitogen.core.b(''), src) self.assertTrue(is_pkg) def test_empty_source_module(self): path, src, is_pkg = self.call('module_finder_testmod.empty_mod') self.assertEquals(path, - testlib.data_path('module_finder_testmod/empty_mod.py')) + os.path.join(MODS_DIR, 'module_finder_testmod/empty_mod.py')) self.assertEquals(mitogen.core.b(''), src) self.assertFalse(is_pkg) @@ -101,23 +103,29 @@ class GetModuleViaPkgutilTest(testlib.TestCase): from module_finder_testmod import regular_mod path, src, is_pkg = self.call('module_finder_testmod.regular_mod') self.assertEquals(path, - testlib.data_path('module_finder_testmod/regular_mod.py')) + os.path.join(MODS_DIR, 'module_finder_testmod/regular_mod.py')) self.assertEquals(mitogen.core.to_text(src), inspect.getsource(regular_mod)) self.assertFalse(is_pkg) -class GetModuleViaSysModulesTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder +class SysModulesMethodTest(testlib.TestCase): + klass = mitogen.master.SysModulesMethod def call(self, fullname): - return self.klass()._get_module_via_sys_modules(fullname) + return self.klass().find(fullname) def test_main(self): import __main__ path, src, is_pkg = self.call('__main__') self.assertEquals(path, __main__.__file__) - self.assertEquals(src, open(path, 'rb').read()) + + # linecache adds a line ending to the final line if one is missing. + actual_src = open(path, 'rb').read() + if actual_src[-1:] != b('\n'): + actual_src += b('\n') + + self.assertEquals(src, actual_src) self.assertFalse(is_pkg) def test_dylib_fails(self): @@ -133,10 +141,10 @@ class GetModuleViaSysModulesTest(testlib.TestCase): class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder + klass = mitogen.master.ParentEnumerationMethod def call(self, fullname): - return self.klass()._get_module_via_parent_enumeration(fullname) + return self.klass().find(fullname) def test_main_fails(self): import __main__ @@ -157,13 +165,67 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): # plumbum has been eating too many rainbow-colored pills import pkg_like_plumbum.colors path, src, is_pkg = self.call('pkg_like_plumbum.colors') - self.assertEquals(path, - testlib.data_path('pkg_like_plumbum/colors.py')) + modpath = os.path.join(MODS_DIR, 'pkg_like_plumbum/colors.py') + self.assertEquals(path, modpath) - s = open(testlib.data_path('pkg_like_plumbum/colors.py'), 'rb').read() - self.assertEquals(src, s) + self.assertEquals(src, open(modpath, 'rb').read()) self.assertFalse(is_pkg) + def test_ansible_module_utils_distro_succeeds(self): + # #590: a package that turns itself into a module. + import pkg_like_ansible.module_utils.distro as d + self.assertEquals(d.I_AM, "the module that replaced the package") + self.assertEquals( + sys.modules['pkg_like_ansible.module_utils.distro'].__name__, + 'pkg_like_ansible.module_utils.distro._distro' + ) + + # ensure we can resolve the subpackage. + path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro') + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/distro/__init__.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, True) + + # ensure we can resolve a child of the subpackage. + path, src, is_pkg = self.call( + 'pkg_like_ansible.module_utils.distro._distro' + ) + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/distro/_distro.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, False) + + def test_ansible_module_utils_system_distro_succeeds(self): + # #590: a package that turns itself into a module. + # #590: a package that turns itself into a module. + import pkg_like_ansible.module_utils.sys_distro as d + self.assertEquals(d.I_AM, "the system module that replaced the subpackage") + self.assertEquals( + sys.modules['pkg_like_ansible.module_utils.sys_distro'].__name__, + 'system_distro' + ) + + # ensure we can resolve the subpackage. + path, src, is_pkg = self.call('pkg_like_ansible.module_utils.sys_distro') + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/sys_distro/__init__.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, True) + + # ensure we can resolve a child of the subpackage. + path, src, is_pkg = self.call( + 'pkg_like_ansible.module_utils.sys_distro._distro' + ) + modpath = os.path.join(MODS_DIR, + 'pkg_like_ansible/module_utils/sys_distro/_distro.py') + self.assertEquals(path, modpath) + self.assertEquals(src, open(modpath, 'rb').read()) + self.assertEquals(is_pkg, False) + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder @@ -235,7 +297,7 @@ class FindRelatedTest(testlib.TestCase): if sys.version_info > (2, 6): class DjangoMixin(object): - WEBPROJECT_PATH = testlib.data_path('webproject') + WEBPROJECT_PATH = os.path.join(MODS_DIR, 'webproject') # TODO: rip out Django and replace with a static tree of weird imports # that don't depend on .. Django! The hack below is because the version diff --git a/tests/parent_test.py b/tests/parent_test.py index 00bddb4d..d6efe998 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -12,12 +12,18 @@ import unittest2 import testlib from testlib import Popen__terminate +import mitogen.core import mitogen.parent +try: + file +except NameError: + from io import FileIO as file + def wait_for_child(pid, timeout=1.0): - deadline = time.time() + timeout - while timeout < time.time(): + deadline = mitogen.core.now() + timeout + while timeout < mitogen.core.now(): try: target_pid, status = os.waitpid(pid, os.WNOHANG) if target_pid == pid: @@ -49,7 +55,7 @@ def wait_for_empty_output_queue(sync_recv, context): while True: # Now wait for the RPC to exit the output queue. stream = router.stream_by_id(context.context_id) - if broker.defer_sync(lambda: stream.pending_bytes()) == 0: + if broker.defer_sync(lambda: stream.protocol.pending_bytes()) == 0: return time.sleep(0.1) @@ -69,35 +75,17 @@ class GetDefaultRemoteNameTest(testlib.TestCase): self.assertEquals("ECORP_Administrator@box:123", self.func()) -class WstatusToStrTest(testlib.TestCase): - func = staticmethod(mitogen.parent.wstatus_to_str) +class ReturncodeToStrTest(testlib.TestCase): + func = staticmethod(mitogen.parent.returncode_to_str) def test_return_zero(self): - pid = os.fork() - if not pid: - os._exit(0) - (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) - self.assertEquals(self.func(status), - 'exited with return code 0') + self.assertEquals(self.func(0), 'exited with return code 0') def test_return_one(self): - pid = os.fork() - if not pid: - os._exit(1) - (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) - self.assertEquals( - self.func(status), - 'exited with return code 1' - ) + self.assertEquals(self.func(1), 'exited with return code 1') def test_sigkill(self): - pid = os.fork() - if not pid: - time.sleep(600) - os.kill(pid, signal.SIGKILL) - (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) - self.assertEquals( - self.func(status), + self.assertEquals(self.func(-signal.SIGKILL), 'exited due to signal %s (SIGKILL)' % (int(signal.SIGKILL),) ) @@ -107,20 +95,20 @@ class WstatusToStrTest(testlib.TestCase): class ReapChildTest(testlib.RouterMixin, testlib.TestCase): def test_connect_timeout(self): # Ensure the child process is reaped if the connection times out. - stream = mitogen.parent.Stream( - router=self.router, - remote_id=1234, + options = mitogen.parent.Options( old_router=self.router, max_message_size=self.router.max_message_size, python_path=testlib.data_path('python_never_responds.py'), connect_timeout=0.5, ) + + conn = mitogen.parent.Connection(options, router=self.router) self.assertRaises(mitogen.core.TimeoutError, - lambda: stream.connect() + lambda: conn.connect(context=mitogen.core.Context(None, 1234)) ) - wait_for_child(stream.pid) + wait_for_child(conn.proc.pid) e = self.assertRaises(OSError, - lambda: os.kill(stream.pid, 0) + lambda: os.kill(conn.proc.pid, 0) ) self.assertEquals(e.args[0], errno.ESRCH) @@ -133,7 +121,7 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): connect_timeout=3, ) ) - prefix = "EOF on stream; last 300 bytes received: " + prefix = mitogen.parent.Connection.eof_error_msg self.assertTrue(e.args[0].startswith(prefix)) def test_via_eof(self): @@ -142,12 +130,12 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.local( via=local, - python_path='true', + python_path='echo', connect_timeout=3, ) ) - s = "EOF on stream; last 300 bytes received: " - self.assertTrue(s in e.args[0]) + expect = mitogen.parent.Connection.eof_error_msg + self.assertTrue(expect in e.args[0]) def test_direct_enoent(self): e = self.assertRaises(mitogen.core.StreamError, @@ -185,11 +173,15 @@ class OpenPtyTest(testlib.TestCase): func = staticmethod(mitogen.parent.openpty) def test_pty_returned(self): - master_fd, slave_fd = self.func() - self.assertTrue(isinstance(master_fd, int)) - self.assertTrue(isinstance(slave_fd, int)) - os.close(master_fd) - os.close(slave_fd) + master_fp, slave_fp = self.func() + try: + self.assertTrue(master_fp.isatty()) + self.assertTrue(isinstance(master_fp, file)) + self.assertTrue(slave_fp.isatty()) + self.assertTrue(isinstance(slave_fp, file)) + finally: + master_fp.close() + slave_fp.close() @mock.patch('os.openpty') def test_max_reached(self, openpty): @@ -204,154 +196,20 @@ class OpenPtyTest(testlib.TestCase): @mock.patch('os.openpty') def test_broken_linux_fallback(self, openpty): openpty.side_effect = OSError(errno.EPERM) - master_fd, slave_fd = self.func() + master_fp, slave_fp = self.func() try: - st = os.fstat(master_fd) + st = os.fstat(master_fp.fileno()) self.assertEquals(5, os.major(st.st_rdev)) - flags = fcntl.fcntl(master_fd, fcntl.F_GETFL) + flags = fcntl.fcntl(master_fp.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) - st = os.fstat(slave_fd) + st = os.fstat(slave_fp.fileno()) self.assertEquals(136, os.major(st.st_rdev)) - flags = fcntl.fcntl(slave_fd, fcntl.F_GETFL) + flags = fcntl.fcntl(slave_fp.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) finally: - os.close(master_fd) - os.close(slave_fd) - - -class TtyCreateChildTest(testlib.TestCase): - func = staticmethod(mitogen.parent.tty_create_child) - - def test_dev_tty_open_succeeds(self): - # In the early days of UNIX, a process that lacked a controlling TTY - # would acquire one simply by opening an existing TTY. Linux and OS X - # continue to follow this behaviour, however at least FreeBSD moved to - # requiring an explicit ioctl(). Linux supports it, but we don't yet - # use it there and anyway the behaviour will never change, so no point - # in fixing things that aren't broken. Below we test that - # getpass-loving apps like sudo and ssh get our slave PTY when they - # attempt to open /dev/tty, which is what they both do on attempting to - # read a password. - tf = tempfile.NamedTemporaryFile() - try: - pid, fd, _ = self.func([ - 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) - ]) - deadline = time.time() + 5.0 - for line in mitogen.parent.iter_read([fd], deadline): - self.assertEquals(mitogen.core.b('hi\n'), line) - break - waited_pid, status = os.waitpid(pid, 0) - self.assertEquals(pid, waited_pid) - self.assertEquals(0, status) - self.assertEquals(mitogen.core.b(''), tf.read()) - os.close(fd) - finally: - tf.close() - - -class IterReadTest(testlib.TestCase): - func = staticmethod(mitogen.parent.iter_read) - - def make_proc(self): - # I produce text every 100ms. - args = [testlib.data_path('iter_read_generator.py')] - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - mitogen.core.set_nonblock(proc.stdout.fileno()) - return proc - - def test_no_deadline(self): - proc = self.make_proc() - try: - reader = self.func([proc.stdout.fileno()]) - for i, chunk in enumerate(reader): - self.assertEqual(1+i, int(chunk)) - if i > 2: - break - finally: - Popen__terminate(proc) - proc.stdout.close() - - def test_deadline_exceeded_before_call(self): - proc = self.make_proc() - reader = self.func([proc.stdout.fileno()], 0) - try: - got = [] - try: - for chunk in reader: - got.append(chunk) - assert 0, 'TimeoutError not raised' - except mitogen.core.TimeoutError: - self.assertEqual(len(got), 0) - finally: - Popen__terminate(proc) - proc.stdout.close() - - def test_deadline_exceeded_during_call(self): - proc = self.make_proc() - deadline = time.time() + 0.4 - - reader = self.func([proc.stdout.fileno()], deadline) - try: - got = [] - try: - for chunk in reader: - if time.time() > (deadline + 1.0): - assert 0, 'TimeoutError not raised' - got.append(chunk) - except mitogen.core.TimeoutError: - # Give a little wiggle room in case of imperfect scheduling. - # Ideal number should be 9. - self.assertLess(deadline, time.time()) - self.assertLess(1, len(got)) - self.assertLess(len(got), 20) - finally: - Popen__terminate(proc) - proc.stdout.close() - - -class WriteAllTest(testlib.TestCase): - func = staticmethod(mitogen.parent.write_all) - - def make_proc(self): - args = [testlib.data_path('write_all_consumer.py')] - proc = subprocess.Popen(args, stdin=subprocess.PIPE) - mitogen.core.set_nonblock(proc.stdin.fileno()) - return proc - - ten_ms_chunk = (mitogen.core.b('x') * 65535) - - def test_no_deadline(self): - proc = self.make_proc() - try: - self.func(proc.stdin.fileno(), self.ten_ms_chunk) - finally: - Popen__terminate(proc) - proc.stdin.close() - - def test_deadline_exceeded_before_call(self): - proc = self.make_proc() - try: - self.assertRaises(mitogen.core.TimeoutError, ( - lambda: self.func(proc.stdin.fileno(), self.ten_ms_chunk, 0) - )) - finally: - Popen__terminate(proc) - proc.stdin.close() - - def test_deadline_exceeded_during_call(self): - proc = self.make_proc() - try: - deadline = time.time() + 0.1 # 100ms deadline - self.assertRaises(mitogen.core.TimeoutError, ( - lambda: self.func(proc.stdin.fileno(), - self.ten_ms_chunk * 100, # 1s of data - deadline) - )) - finally: - Popen__terminate(proc) - proc.stdin.close() + master_fp.close() + slave_fp.close() class DisconnectTest(testlib.RouterMixin, testlib.TestCase): @@ -394,7 +252,7 @@ class DisconnectTest(testlib.RouterMixin, testlib.TestCase): c2 = self.router.local() # Let c1 call functions in c2. - self.router.stream_by_id(c1.context_id).auth_id = mitogen.context_id + self.router.stream_by_id(c1.context_id).protocol.auth_id = mitogen.context_id c1.call(mitogen.parent.upgrade_router) sync_recv = mitogen.core.Receiver(self.router) @@ -412,14 +270,14 @@ class DisconnectTest(testlib.RouterMixin, testlib.TestCase): def test_far_sibling_disconnected(self): # God mode: child of child notices child of child of parent has # disconnected. - c1 = self.router.local() - c11 = self.router.local(via=c1) + c1 = self.router.local(name='c1') + c11 = self.router.local(name='c11', via=c1) - c2 = self.router.local() - c22 = self.router.local(via=c2) + c2 = self.router.local(name='c2') + c22 = self.router.local(name='c22', via=c2) # Let c1 call functions in c2. - self.router.stream_by_id(c1.context_id).auth_id = mitogen.context_id + self.router.stream_by_id(c1.context_id).protocol.auth_id = mitogen.context_id c11.call(mitogen.parent.upgrade_router) sync_recv = mitogen.core.Receiver(self.router) diff --git a/tests/poller_test.py b/tests/poller_test.py index e2e3cdd7..3ed59ae3 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -42,8 +42,8 @@ class SockMixin(object): self.l2_sock, self.r2_sock = socket.socketpair() self.l2 = self.l2_sock.fileno() self.r2 = self.r2_sock.fileno() - for fd in self.l1, self.r1, self.l2, self.r2: - mitogen.core.set_nonblock(fd) + for fp in self.l1, self.r1, self.l2, self.r2: + mitogen.core.set_nonblock(fp) def fill(self, fd): """Make `fd` unwriteable.""" @@ -164,14 +164,14 @@ class CloseMixin(PollerMixin): class PollMixin(PollerMixin): def test_empty_zero_timeout(self): - t0 = time.time() + t0 = mitogen.core.now() self.assertEquals([], list(self.p.poll(0))) - self.assertTrue((time.time() - t0) < .1) # vaguely reasonable + self.assertTrue((mitogen.core.now() - t0) < .1) # vaguely reasonable def test_empty_small_timeout(self): - t0 = time.time() + t0 = mitogen.core.now() self.assertEquals([], list(self.p.poll(.2))) - self.assertTrue((time.time() - t0) >= .2) + self.assertTrue((mitogen.core.now() - t0) >= .2) class ReadableMixin(PollerMixin, SockMixin): @@ -354,17 +354,17 @@ class FileClosedMixin(PollerMixin, SockMixin): class TtyHangupMixin(PollerMixin): def test_tty_hangup_detected(self): # bug in initial select.poll() implementation failed to detect POLLHUP. - master_fd, slave_fd = mitogen.parent.openpty() + master_fp, slave_fp = mitogen.parent.openpty() try: - self.p.start_receive(master_fd) + self.p.start_receive(master_fp.fileno()) self.assertEquals([], list(self.p.poll(0))) - os.close(slave_fd) - slave_fd = None - self.assertEquals([master_fd], list(self.p.poll(0))) + slave_fp.close() + slave_fp = None + self.assertEquals([master_fp.fileno()], list(self.p.poll(0))) finally: - if slave_fd is not None: - os.close(slave_fd) - os.close(master_fd) + if slave_fp is not None: + slave_fp.close() + master_fp.close() class DistinctDataMixin(PollerMixin, SockMixin): diff --git a/tests/reaper_test.py b/tests/reaper_test.py new file mode 100644 index 00000000..e78fdbf2 --- /dev/null +++ b/tests/reaper_test.py @@ -0,0 +1,54 @@ + +import signal +import unittest2 +import testlib +import mock + +import mitogen.parent + + +class ReaperTest(testlib.TestCase): + @mock.patch('os.kill') + def test_calc_delay(self, kill): + broker = mock.Mock() + proc = mock.Mock() + proc.poll.return_value = None + reaper = mitogen.parent.Reaper(broker, proc, True, True) + self.assertEquals(50, int(1000 * reaper._calc_delay(0))) + self.assertEquals(86, int(1000 * reaper._calc_delay(1))) + self.assertEquals(147, int(1000 * reaper._calc_delay(2))) + self.assertEquals(254, int(1000 * reaper._calc_delay(3))) + self.assertEquals(437, int(1000 * reaper._calc_delay(4))) + self.assertEquals(752, int(1000 * reaper._calc_delay(5))) + self.assertEquals(1294, int(1000 * reaper._calc_delay(6))) + + @mock.patch('os.kill') + def test_reap_calls(self, kill): + broker = mock.Mock() + proc = mock.Mock() + proc.poll.return_value = None + + reaper = mitogen.parent.Reaper(broker, proc, True, True) + + reaper.reap() + self.assertEquals(0, kill.call_count) + + reaper.reap() + self.assertEquals(1, kill.call_count) + + reaper.reap() + reaper.reap() + reaper.reap() + self.assertEquals(1, kill.call_count) + + reaper.reap() + self.assertEquals(2, kill.call_count) + + self.assertEquals(kill.mock_calls, [ + mock.call(proc.pid, signal.SIGTERM), + mock.call(proc.pid, signal.SIGKILL), + ]) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/requirements.txt b/tests/requirements.txt index 327f563a..bbcdc7cc 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -13,3 +13,5 @@ unittest2==1.1.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings urllib3[secure]; python_version < '2.7.9' +# Last idna compatible with Python 2.6 was idna 2.7. +idna==2.7; python_version < '2.7' diff --git a/tests/responder_test.py b/tests/responder_test.py index dbc68a3c..2653589c 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -105,7 +105,7 @@ class BrokenModulesTest(testlib.TestCase): # unavailable. Should never happen in the real world. stream = mock.Mock() - stream.sent_modules = set() + stream.protocol.sent_modules = set() router = mock.Mock() router.stream_by_id = lambda n: stream @@ -143,7 +143,7 @@ class BrokenModulesTest(testlib.TestCase): import six_brokenpkg stream = mock.Mock() - stream.sent_modules = set() + stream.protocol.sent_modules = set() router = mock.Mock() router.stream_by_id = lambda n: stream @@ -158,14 +158,15 @@ class BrokenModulesTest(testlib.TestCase): self.assertEquals(1, len(router._async_route.mock_calls)) self.assertEquals(1, responder.get_module_count) - self.assertEquals(0, responder.good_load_module_count) - self.assertEquals(0, responder.good_load_module_size) - self.assertEquals(1, responder.bad_load_module_count) + self.assertEquals(1, responder.good_load_module_count) + self.assertEquals(0, responder.bad_load_module_count) call = router._async_route.mock_calls[0] msg, = call[1] self.assertEquals(mitogen.core.LOAD_MODULE, msg.handle) - self.assertIsInstance(msg.unpickle(), tuple) + + tup = msg.unpickle() + self.assertIsInstance(tup, tuple) class ForwardTest(testlib.RouterMixin, testlib.TestCase): diff --git a/tests/router_test.py b/tests/router_test.py index 80169e34..58ab637a 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -1,12 +1,17 @@ +import errno +import os +import sys import time import zlib import unittest2 import testlib +import mitogen.core import mitogen.master import mitogen.parent import mitogen.utils +from mitogen.core import b try: import Queue @@ -57,12 +62,12 @@ class SourceVerifyTest(testlib.RouterMixin, testlib.TestCase): recv = mitogen.core.Receiver(self.router) self.child2_msg.handle = recv.handle - self.broker.defer(self.router._async_route, - self.child2_msg, - in_stream=self.child1_stream) - - # Wait for IO loop to finish everything above. - self.sync_with_broker() + self.broker.defer_sync( + lambda: self.router._async_route( + self.child2_msg, + in_stream=self.child1_stream + ) + ) # Ensure message wasn't forwarded. self.assertTrue(recv.empty()) @@ -71,6 +76,34 @@ class SourceVerifyTest(testlib.RouterMixin, testlib.TestCase): expect = 'bad auth_id: got %r via' % (self.child2_msg.auth_id,) self.assertTrue(expect in log.stop()) + def test_parent_unaware_of_disconnect(self): + # Parent -> Child A -> Child B. B disconnects concurrent to Parent + # sending message. Parent does not yet know B has disconnected, A + # receives message from Parent with Parent's auth_id, for a stream that + # no longer exists. + c1 = self.router.local() + strm = self.router.stream_by_id(c1.context_id) + recv = mitogen.core.Receiver(self.router) + + self.broker.defer(lambda: + strm.protocol._send( + mitogen.core.Message( + dst_id=1234, # nonexistent child + handle=1234, + src_id=mitogen.context_id, + reply_to=recv.handle, + ) + ) + ) + + e = self.assertRaises(mitogen.core.ChannelError, + lambda: recv.get().unpickle() + ) + self.assertEquals(e.args[0], self.router.no_route_msg % ( + 1234, + c1.context_id, + )) + def test_bad_src_id(self): # Deliver a message locally from child2 with the correct auth_id, but # the wrong src_id. @@ -171,7 +204,7 @@ class CrashTest(testlib.BrokerMixin, testlib.TestCase): self.assertTrue(sem.get().is_dead) # Ensure it was logged. - expect = '_broker_main() crashed' + expect = 'broker crashed' self.assertTrue(expect in log.stop()) self.broker.join() @@ -254,12 +287,36 @@ class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase): self.assertTrue(expect in logs.stop()) + def test_remote_dead_message(self): + # Router should send dead message to original recipient when reply_to + # is unset. + router = self.klass(broker=self.broker, max_message_size=4096) + + # Try function call. Receiver should be woken by a dead message sent by + # router due to message size exceeded. + child = router.local() + recv = mitogen.core.Receiver(router) + + recv.to_sender().send(b('x') * 4097) + e = self.assertRaises(mitogen.core.ChannelError, + lambda: recv.get().unpickle() + ) + expect = router.too_large_msg % (4096,) + self.assertEquals(e.args[0], expect) + def test_remote_configured(self): router = self.klass(broker=self.broker, max_message_size=64*1024) remote = router.local() size = remote.call(return_router_max_message_size) self.assertEquals(size, 64*1024) + def test_remote_of_remote_configured(self): + router = self.klass(broker=self.broker, max_message_size=64*1024) + remote = router.local() + remote2 = router.local(via=remote) + size = remote2.call(return_router_max_message_size) + self.assertEquals(size, 64*1024) + def test_remote_exceeded(self): # Ensure new contexts receive a router with the same value. router = self.klass(broker=self.broker, max_message_size=64*1024) @@ -341,22 +398,43 @@ class NoRouteTest(testlib.RouterMixin, testlib.TestCase): )) +def test_siblings_cant_talk(router): + l1 = router.local() + l2 = router.local() + logs = testlib.LogCapturer() + logs.start() + + try: + l2.call(ping_context, l1) + except mitogen.core.CallError: + e = sys.exc_info()[1] + + msg = mitogen.core.Router.unidirectional_msg % ( + l2.context_id, + l1.context_id, + mitogen.context_id, + ) + assert msg in str(e) + assert 'routing mode prevents forward of ' in logs.stop() + + +@mitogen.core.takes_econtext +def test_siblings_cant_talk_remote(econtext): + mitogen.parent.upgrade_router(econtext) + test_siblings_cant_talk(econtext.router) + + class UnidirectionalTest(testlib.RouterMixin, testlib.TestCase): - def test_siblings_cant_talk(self): + def test_siblings_cant_talk_master(self): self.router.unidirectional = True - l1 = self.router.local() - l2 = self.router.local() - logs = testlib.LogCapturer() - logs.start() - e = self.assertRaises(mitogen.core.CallError, - lambda: l2.call(ping_context, l1)) + test_siblings_cant_talk(self.router) - msg = self.router.unidirectional_msg % ( - l2.context_id, - l1.context_id, - ) - self.assertTrue(msg in str(e)) - self.assertTrue('routing mode prevents forward of ' in logs.stop()) + def test_siblings_cant_talk_parent(self): + # ensure 'unidirectional' attribute is respected for contexts started + # by children. + self.router.unidirectional = True + parent = self.router.local() + parent.call(test_siblings_cant_talk_remote) def test_auth_id_can_talk(self): self.router.unidirectional = True @@ -364,8 +442,8 @@ class UnidirectionalTest(testlib.RouterMixin, testlib.TestCase): # treated like a parent. l1 = self.router.local() l1s = self.router.stream_by_id(l1.context_id) - l1s.auth_id = mitogen.context_id - l1s.is_privileged = True + l1s.protocol.auth_id = mitogen.context_id + l1s.protocol.is_privileged = True l2 = self.router.local() e = self.assertRaises(mitogen.core.CallError, @@ -378,12 +456,133 @@ class UnidirectionalTest(testlib.RouterMixin, testlib.TestCase): class EgressIdsTest(testlib.RouterMixin, testlib.TestCase): def test_egress_ids_populated(self): # Ensure Stream.egress_ids is populated on message reception. - c1 = self.router.local() - stream = self.router.stream_by_id(c1.context_id) - self.assertEquals(set(), stream.egress_ids) + c1 = self.router.local(name='c1') + c2 = self.router.local(name='c2') + + c1s = self.router.stream_by_id(c1.context_id) + try: + c1.call(ping_context, c2) + except mitogen.core.CallError: + # Fails because siblings cant call funcs in each other, but this + # causes messages to be sent. + pass + + self.assertEquals(c1s.protocol.egress_ids, set([ + mitogen.context_id, + c2.context_id, + ])) + + +class ShutdownTest(testlib.RouterMixin, testlib.TestCase): + # 613: tests for all the weird shutdown() variants we ended up with. + + def test_shutdown_wait_false(self): + l1 = self.router.local() + pid = l1.call(os.getpid) + + strm = self.router.stream_by_id(l1.context_id) + exitted = mitogen.core.Latch() + + # It is possible for Process 'exit' signal to fire immediately during + # processing of Stream 'disconnect' signal, so we must wait for both, + # otherwise ChannelError below will return 'respondent context has + # disconnected' rather than 'no route', because RouteMonitor hasn't run + # yet and the Receiver caught Context 'disconnect' signal instead of a + # dead message. + mitogen.core.listen(strm.conn.proc, 'exit', exitted.put) + mitogen.core.listen(strm, 'disconnect', exitted.put) + + l1.shutdown(wait=False) + exitted.get() + exitted.get() + + e = self.assertRaises(OSError, + lambda: os.waitpid(pid, 0)) + self.assertEquals(e.args[0], errno.ECHILD) + + e = self.assertRaises(mitogen.core.ChannelError, + lambda: l1.call(os.getpid)) + self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + l1.context_id, + mitogen.context_id, + )) + + def test_shutdown_wait_true(self): + l1 = self.router.local() + pid = l1.call(os.getpid) + + conn = self.router.stream_by_id(l1.context_id).conn + exitted = mitogen.core.Latch() + mitogen.core.listen(conn.proc, 'exit', exitted.put) + + l1.shutdown(wait=True) + exitted.get() + + e = self.assertRaises(OSError, + lambda: os.waitpid(pid, 0)) + self.assertEquals(e.args[0], errno.ECHILD) + + e = self.assertRaises(mitogen.core.ChannelError, + lambda: l1.call(os.getpid)) + self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + l1.context_id, + mitogen.context_id, + )) + + def test_disconnect_invalid_context(self): + self.router.disconnect( + mitogen.core.Context(self.router, 1234) + ) + + def test_disconnect_valid_context(self): + l1 = self.router.local() + pid = l1.call(os.getpid) + + strm = self.router.stream_by_id(l1.context_id) + + exitted = mitogen.core.Latch() + mitogen.core.listen(strm.conn.proc, 'exit', exitted.put) + self.router.disconnect_stream(strm) + exitted.get() + + e = self.assertRaises(OSError, + lambda: os.waitpid(pid, 0)) + self.assertEquals(e.args[0], errno.ECHILD) + + e = self.assertRaises(mitogen.core.ChannelError, + lambda: l1.call(os.getpid)) + self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + l1.context_id, + mitogen.context_id, + )) + + def test_disconnect_all(self): + l1 = self.router.local() + l2 = self.router.local() - c1.call(time.sleep, 0) - self.assertEquals(set([mitogen.context_id]), stream.egress_ids) + pids = [l1.call(os.getpid), l2.call(os.getpid)] + + exitted = mitogen.core.Latch() + for ctx in l1, l2: + strm = self.router.stream_by_id(ctx.context_id) + mitogen.core.listen(strm.conn.proc, 'exit', exitted.put) + + self.router.disconnect_all() + exitted.get() + exitted.get() + + for pid in pids: + e = self.assertRaises(OSError, + lambda: os.waitpid(pid, 0)) + self.assertEquals(e.args[0], errno.ECHILD) + + for ctx in l1, l2: + e = self.assertRaises(mitogen.core.ChannelError, + lambda: ctx.call(os.getpid)) + self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + ctx.context_id, + mitogen.context_id, + )) if __name__ == '__main__': diff --git a/tests/select_test.py b/tests/select_test.py index f08c9f3a..56e7e6cd 100644 --- a/tests/select_test.py +++ b/tests/select_test.py @@ -358,6 +358,18 @@ class GetReceiverTest(testlib.RouterMixin, testlib.TestCase): msg = select.get() self.assertEquals('123', msg.unpickle()) + def test_nonempty_multiple_items_before_add(self): + recv = mitogen.core.Receiver(self.router) + recv._on_receive(mitogen.core.Message.pickled('123')) + recv._on_receive(mitogen.core.Message.pickled('234')) + select = self.klass([recv], oneshot=False) + msg = select.get() + self.assertEquals('123', msg.unpickle()) + msg = select.get() + self.assertEquals('234', msg.unpickle()) + self.assertRaises(mitogen.core.TimeoutError, + lambda: select.get(block=False)) + def test_nonempty_after_add(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) @@ -415,6 +427,16 @@ class GetLatchTest(testlib.RouterMixin, testlib.TestCase): select = self.klass([latch]) self.assertEquals(123, select.get()) + def test_nonempty_multiple_items_before_add(self): + latch = mitogen.core.Latch() + latch.put(123) + latch.put(234) + select = self.klass([latch], oneshot=False) + self.assertEquals(123, select.get()) + self.assertEquals(234, select.get()) + self.assertRaises(mitogen.core.TimeoutError, + lambda: select.get(block=False)) + def test_nonempty_after_add(self): latch = mitogen.core.Latch() select = self.klass([latch]) diff --git a/tests/service_test.py b/tests/service_test.py index 3869f713..a3e75e14 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -15,6 +15,13 @@ class MyService(mitogen.service.Service): self._counter += 1 return self._counter, id(self) + @mitogen.service.expose(policy=mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'foo': int + }) + def test_arg_spec(self, foo): + return foo + @mitogen.service.expose(policy=mitogen.service.AllowParents()) def privileged_op(self): return 'privileged!' @@ -24,7 +31,6 @@ class MyService(mitogen.service.Service): return 'unprivileged!' - class MyService2(MyService): """ A uniquely named service that lets us test framework activation and class @@ -36,6 +42,44 @@ def call_service_in(context, service_name, method_name): return context.call_service(service_name, method_name) +class CallTest(testlib.RouterMixin, testlib.TestCase): + def test_local(self): + pool = mitogen.service.get_or_create_pool(router=self.router) + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService, 'privileged_op') + ) + pool.stop() + + def test_remote_bad_arg(self): + c1 = self.router.local() + self.assertRaises( + mitogen.core.CallError, + lambda: mitogen.service.call( + MyService.name(), + 'test_arg_spec', + foo='x', + call_context=c1 + ) + ) + + def test_local_unicode(self): + pool = mitogen.service.get_or_create_pool(router=self.router) + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService.name(), 'privileged_op') + ) + pool.stop() + + def test_remote(self): + c1 = self.router.local() + self.assertEquals( + 'privileged!', + mitogen.service.call(MyService, 'privileged_op', + call_context=c1) + ) + + class ActivationTest(testlib.RouterMixin, testlib.TestCase): def test_parent_can_activate(self): l1 = self.router.local() @@ -44,8 +88,8 @@ class ActivationTest(testlib.RouterMixin, testlib.TestCase): self.assertTrue(isinstance(id_, int)) def test_sibling_cannot_activate_framework(self): - l1 = self.router.local() - l2 = self.router.local() + l1 = self.router.local(name='l1') + l2 = self.router.local(name='l2') exc = self.assertRaises(mitogen.core.CallError, lambda: l2.call(call_service_in, l1, MyService2.name(), 'get_id')) self.assertTrue(mitogen.core.Router.refused_msg in exc.args[0]) diff --git a/tests/setns_test.py b/tests/setns_test.py new file mode 100644 index 00000000..d48179b1 --- /dev/null +++ b/tests/setns_test.py @@ -0,0 +1,46 @@ + +import os +import socket +import sys + +import mitogen +import mitogen.parent + +import unittest2 + +import testlib + + +class DockerTest(testlib.DockerMixin, testlib.TestCase): + def test_okay(self): + # Magic calls must happen as root. + try: + root = self.router.sudo() + except mitogen.core.StreamError: + raise unittest2.SkipTest("requires sudo to localhost root") + + via_ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + + via_setns = self.router.setns( + kind='docker', + container=self.dockerized_ssh.container_name, + via=root, + ) + + self.assertEquals( + via_ssh.call(socket.gethostname), + via_setns.call(socket.gethostname), + ) + + +DockerTest = unittest2.skipIf( + condition=sys.version_info < (2, 5), + reason="mitogen.setns unsupported on Python <2.4" +)(DockerTest) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 496710b8..273412e8 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -42,8 +42,6 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): class SshTest(testlib.DockerMixin, testlib.TestCase): - stream_class = mitogen.ssh.Stream - def test_debug_decoding(self): # ensure filter_debug_logs() decodes the logged string. capture = testlib.LogCapturer() @@ -60,6 +58,14 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): expect = "%s: debug1: Reading configuration data" % (context.name,) self.assertTrue(expect in s) + def test_bash_permission_denied(self): + # issue #271: only match Permission Denied at start of line. + context = self.docker_ssh( + username='mitogen__permdenied', + password='permdenied_password', + ssh_debug_level=3, + ) + def test_stream_name(self): context = self.docker_ssh( username='mitogen__has_sudo', @@ -85,27 +91,21 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): self.assertEquals(name, sudo.name) def test_password_required(self): - try: - context = self.docker_ssh( + e = self.assertRaises(mitogen.ssh.PasswordError, + lambda: self.docker_ssh( username='mitogen__has_sudo', ) - assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError: - e = sys.exc_info()[1] - - self.assertEqual(e.args[0], self.stream_class.password_required_msg) + ) + self.assertEqual(e.args[0], mitogen.ssh.password_required_msg) def test_password_incorrect(self): - try: - context = self.docker_ssh( + e = self.assertRaises(mitogen.ssh.PasswordError, + lambda: self.docker_ssh( username='mitogen__has_sudo', password='badpw', ) - assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError: - e = sys.exc_info()[1] - - self.assertEqual(e.args[0], self.stream_class.password_incorrect_msg) + ) + self.assertEqual(e.args[0], mitogen.ssh.password_incorrect_msg) def test_password_specified(self): context = self.docker_ssh( @@ -119,15 +119,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): ) def test_pubkey_required(self): - try: - context = self.docker_ssh( + e = self.assertRaises(mitogen.ssh.PasswordError, + lambda: self.docker_ssh( username='mitogen__has_sudo_pubkey', ) - assert 0, 'exception not thrown' - except mitogen.ssh.PasswordError: - e = sys.exc_info()[1] - - self.assertEqual(e.args[0], self.stream_class.password_required_msg) + ) + self.assertEqual(e.args[0], mitogen.ssh.password_required_msg) def test_pubkey_specified(self): context = self.docker_ssh( @@ -150,7 +147,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): check_host_keys='enforce', ) ) - self.assertEquals(e.args[0], mitogen.ssh.Stream.hostkey_failed_msg) + self.assertEquals(e.args[0], mitogen.ssh.hostkey_failed_msg) finally: fp.close() @@ -184,8 +181,6 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): class BannerTest(testlib.DockerMixin, testlib.TestCase): # Verify the ability to disambiguate random spam appearing in the SSHd's # login banner from a legitimate password prompt. - stream_class = mitogen.ssh.Stream - def test_verbose_enabled(self): context = self.docker_ssh( username='mitogen__has_sudo', @@ -210,8 +205,6 @@ class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): class StubCheckHostKeysTest(StubSshMixin, testlib.TestCase): - stream_class = mitogen.ssh.Stream - def test_check_host_keys_accept(self): # required=true, host_key_checking=accept context = self.stub_ssh(STUBSSH_MODE='ask', check_host_keys='accept') diff --git a/tests/stream_test.py b/tests/stream_test.py deleted file mode 100644 index d844e610..00000000 --- a/tests/stream_test.py +++ /dev/null @@ -1,33 +0,0 @@ - -import unittest2 -import mock - -import mitogen.core - -import testlib - - -class ReceiveOneTest(testlib.TestCase): - klass = mitogen.core.Stream - - def test_corruption(self): - broker = mock.Mock() - router = mock.Mock() - - stream = self.klass(router, 1) - junk = mitogen.core.b('x') * stream.HEADER_LEN - stream._input_buf = [junk] - stream._input_buf_len = len(junk) - - capture = testlib.LogCapturer() - capture.start() - ret = stream._receive_one(broker) - #self.assertEquals(1, broker.stop_receive.mock_calls) - capture.stop() - - self.assertFalse(ret) - self.assertTrue((self.klass.corrupt_msg % (junk,)) in capture.raw()) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/su_test.py b/tests/su_test.py index 2af17c6e..320f9cef 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -2,8 +2,7 @@ import os import mitogen -import mitogen.lxd -import mitogen.parent +import mitogen.su import unittest2 @@ -11,22 +10,64 @@ import testlib class ConstructorTest(testlib.RouterMixin, testlib.TestCase): - su_path = testlib.data_path('stubs/stub-su.py') + stub_su_path = testlib.data_path('stubs/stub-su.py') def run_su(self, **kwargs): context = self.router.su( - su_path=self.su_path, + su_path=self.stub_su_path, **kwargs ) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) return context, argv - def test_basic(self): context, argv = self.run_su() self.assertEquals(argv[1], 'root') self.assertEquals(argv[2], '-c') +class SuTest(testlib.DockerMixin, testlib.TestCase): + stub_su_path = testlib.data_path('stubs/stub-su.py') + + def test_slow_auth_failure(self): + # #363: old input loop would fail to spot auth failure because of + # scheduling vs. su calling write() twice. + os.environ['DO_SLOW_AUTH_FAILURE'] = '1' + try: + self.assertRaises(mitogen.su.PasswordError, + lambda: self.router.su(su_path=self.stub_su_path) + ) + finally: + del os.environ['DO_SLOW_AUTH_FAILURE'] + + def test_password_required(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + e = self.assertRaises(mitogen.core.StreamError, + lambda: self.router.su(via=ssh) + ) + self.assertTrue(mitogen.su.password_required_msg in str(e)) + + def test_password_incorrect(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + e = self.assertRaises(mitogen.core.StreamError, + lambda: self.router.su(via=ssh, password='x') + ) + self.assertTrue(mitogen.su.password_incorrect_msg in str(e)) + + def test_password_okay(self): + ssh = self.docker_ssh( + username='mitogen__has_sudo', + password='has_sudo_password', + ) + context = self.router.su(via=ssh, password='rootpassword') + self.assertEquals(0, context.call(os.getuid)) + + if __name__ == '__main__': unittest2.main() diff --git a/tests/sudo_test.py b/tests/sudo_test.py index 1d10ba9a..9ecf103d 100644 --- a/tests/sudo_test.py +++ b/tests/sudo_test.py @@ -2,8 +2,7 @@ import os import mitogen -import mitogen.lxd -import mitogen.parent +import mitogen.sudo import unittest2 @@ -79,7 +78,7 @@ class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.sudo(via=ssh) ) - self.assertTrue(mitogen.sudo.Stream.password_required_msg in str(e)) + self.assertTrue(mitogen.sudo.password_required_msg in str(e)) def test_password_incorrect(self): ssh = self.docker_ssh( @@ -91,7 +90,7 @@ class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.sudo(via=ssh, password='x') ) - self.assertTrue(mitogen.sudo.Stream.password_incorrect_msg in str(e)) + self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) def test_password_okay(self): ssh = self.docker_ssh( @@ -103,7 +102,7 @@ class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.sudo(via=ssh, password='rootpassword') ) - self.assertTrue(mitogen.sudo.Stream.password_incorrect_msg in str(e)) + self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) if __name__ == '__main__': diff --git a/tests/testlib.py b/tests/testlib.py index 37c3c654..b702fa05 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -41,7 +41,11 @@ except NameError: LOG = logging.getLogger(__name__) DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') +MODS_DIR = os.path.join(DATA_DIR, 'importer') + sys.path.append(DATA_DIR) +sys.path.append(MODS_DIR) + if mitogen.is_master: mitogen.utils.log_to_file() @@ -103,11 +107,11 @@ def wait_for_port( If a regex pattern is supplied try to find it in the initial data. Return None on success, or raise on error. """ - start = time.time() + start = mitogen.core.now() end = start + overall_timeout addr = (host, port) - while time.time() < end: + while mitogen.core.now() < end: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(connect_timeout) try: @@ -126,7 +130,7 @@ def wait_for_port( sock.settimeout(receive_timeout) data = mitogen.core.b('') found = False - while time.time() < end: + while mitogen.core.now() < end: try: resp = sock.recv(1024) except socket.timeout: @@ -188,7 +192,7 @@ def sync_with_broker(broker, timeout=10.0): """ sem = mitogen.core.Latch() broker.defer(sem.put, None) - sem.get(timeout=10.0) + sem.get(timeout=timeout) def log_fd_calls(): @@ -279,7 +283,11 @@ class LogCapturer(object): self.logger.level = logging.DEBUG def raw(self): - return self.sio.getvalue() + s = self.sio.getvalue() + # Python 2.x logging package hard-wires UTF-8 output. + if isinstance(s, mitogen.core.BytesType): + s = s.decode('utf-8') + return s def msgs(self): return self.handler.msgs @@ -323,17 +331,43 @@ class TestCase(unittest2.TestCase): for name in counts: assert counts[name] == 1, \ - 'Found %d copies of thread %r running after tests.' % (name,) + 'Found %d copies of thread %r running after tests.' % ( + counts[name], name + ) def _teardown_check_fds(self): mitogen.core.Latch._on_fork() if get_fd_count() != self._fd_count_before: - import os; os.system('lsof -p %s' % (os.getpid(),)) + import os; os.system('lsof +E -w -p %s | grep -vw mem' % (os.getpid(),)) assert 0, "%s leaked FDs. Count before: %s, after: %s" % ( self, self._fd_count_before, get_fd_count(), ) + # Some class fixtures (like Ansible MuxProcess) start persistent children + # for the duration of the class. + no_zombie_check = False + + def _teardown_check_zombies(self): + if self.no_zombie_check: + return + + try: + pid, status = os.waitpid(0, os.WNOHANG) + except OSError: + return # ECHILD + + if pid: + assert 0, "%s failed to reap subprocess %d (status %d)." % ( + self, pid, status + ) + + print('') + print('Children of unit test process:') + os.system('ps uww --ppid ' + str(os.getpid())) + assert 0, "%s leaked still-running subprocesses." % (self,) + def tearDown(self): + self._teardown_check_zombies() self._teardown_check_threads() self._teardown_check_fds() super(TestCase, self).tearDown() @@ -393,6 +427,11 @@ class DockerizedSshDaemon(object): raise ValueError('could not find SSH port in: %r' % (s,)) def start_container(self): + try: + subprocess__check_output(['docker', '--version']) + except Exception: + raise unittest2.SkipTest('Docker binary is unavailable') + self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),) args = [ 'docker', @@ -415,6 +454,22 @@ class DockerizedSshDaemon(object): def wait_for_sshd(self): wait_for_port(self.get_host(), self.port, pattern='OpenSSH') + def check_processes(self): + args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm='] + counts = {} + for comm in subprocess__check_output(args).decode().splitlines(): + comm = comm.strip() + counts[comm] = counts.get(comm, 0) + 1 + + if counts != {'ps': 1, 'sshd': 1}: + assert 0, ( + 'Docker container %r contained extra running processes ' + 'after test completed: %r' % ( + self.container_name, + counts + ) + ) + def close(self): args = ['docker', 'rm', '-f', self.container_name] subprocess__check_output(args) @@ -462,6 +517,7 @@ class DockerMixin(RouterMixin): @classmethod def tearDownClass(cls): + cls.dockerized_ssh.check_processes() cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() diff --git a/tests/timer_test.py b/tests/timer_test.py new file mode 100644 index 00000000..749405a4 --- /dev/null +++ b/tests/timer_test.py @@ -0,0 +1,201 @@ + +import time + +import mock +import unittest2 + +import mitogen.core +import mitogen.parent + +import testlib + + +class TimerListMixin(object): + klass = mitogen.parent.TimerList + + def setUp(self): + self.list = self.klass() + + +class GetTimeoutTest(TimerListMixin, testlib.TestCase): + def test_empty(self): + self.assertEquals(None, self.list.get_timeout()) + + def test_one_event(self): + self.list.schedule(2, lambda: None) + self.list._now = lambda: 1 + self.assertEquals(1, self.list.get_timeout()) + + def test_two_events_same_moment(self): + self.list.schedule(2, lambda: None) + self.list.schedule(2, lambda: None) + self.list._now = lambda: 1 + self.assertEquals(1, self.list.get_timeout()) + + def test_two_events(self): + self.list.schedule(2, lambda: None) + self.list.schedule(3, lambda: None) + self.list._now = lambda: 1 + self.assertEquals(1, self.list.get_timeout()) + + def test_two_events_expired(self): + self.list.schedule(2, lambda: None) + self.list.schedule(3, lambda: None) + self.list._now = lambda: 3 + self.assertEquals(0, self.list.get_timeout()) + + def test_two_events_in_past(self): + self.list.schedule(2, lambda: None) + self.list.schedule(3, lambda: None) + self.list._now = lambda: 30 + self.assertEquals(0, self.list.get_timeout()) + + def test_two_events_in_past(self): + self.list.schedule(2, lambda: None) + self.list.schedule(3, lambda: None) + self.list._now = lambda: 30 + self.assertEquals(0, self.list.get_timeout()) + + def test_one_cancelled(self): + t1 = self.list.schedule(2, lambda: None) + t2 = self.list.schedule(3, lambda: None) + self.list._now = lambda: 0 + t1.cancel() + self.assertEquals(3, self.list.get_timeout()) + + def test_two_cancelled(self): + t1 = self.list.schedule(2, lambda: None) + t2 = self.list.schedule(3, lambda: None) + self.list._now = lambda: 0 + t1.cancel() + t2.cancel() + self.assertEquals(None, self.list.get_timeout()) + + +class ScheduleTest(TimerListMixin, testlib.TestCase): + def test_in_past(self): + self.list._now = lambda: 30 + timer = self.list.schedule(29, lambda: None) + self.assertEquals(29, timer.when) + self.assertEquals(0, self.list.get_timeout()) + + def test_in_future(self): + self.list._now = lambda: 30 + timer = self.list.schedule(31, lambda: None) + self.assertEquals(31, timer.when) + self.assertEquals(1, self.list.get_timeout()) + + def test_same_moment(self): + self.list._now = lambda: 30 + timer = self.list.schedule(31, lambda: None) + timer2 = self.list.schedule(31, lambda: None) + self.assertEquals(31, timer.when) + self.assertEquals(31, timer2.when) + self.assertTrue(timer is not timer2) + self.assertEquals(1, self.list.get_timeout()) + + +class ExpireTest(TimerListMixin, testlib.TestCase): + def test_in_past(self): + timer = self.list.schedule(29, mock.Mock()) + self.assertTrue(timer.active) + self.list._now = lambda: 30 + self.list.expire() + self.assertEquals(1, len(timer.func.mock_calls)) + self.assertFalse(timer.active) + + def test_in_future(self): + timer = self.list.schedule(29, mock.Mock()) + self.assertTrue(timer.active) + self.list._now = lambda: 28 + self.list.expire() + self.assertEquals(0, len(timer.func.mock_calls)) + self.assertTrue(timer.active) + + def test_same_moment(self): + timer = self.list.schedule(29, mock.Mock()) + timer2 = self.list.schedule(29, mock.Mock()) + self.assertTrue(timer.active) + self.assertTrue(timer2.active) + self.list._now = lambda: 29 + self.list.expire() + self.assertEquals(1, len(timer.func.mock_calls)) + self.assertEquals(1, len(timer2.func.mock_calls)) + self.assertFalse(timer.active) + self.assertFalse(timer2.active) + + def test_cancelled(self): + self.list._now = lambda: 29 + timer = self.list.schedule(29, mock.Mock()) + timer.cancel() + self.assertEquals(None, self.list.get_timeout()) + self.list._now = lambda: 29 + self.list.expire() + self.assertEquals(0, len(timer.func.mock_calls)) + self.assertEquals(None, self.list.get_timeout()) + + +class CancelTest(TimerListMixin, testlib.TestCase): + def test_single_cancel(self): + self.list._now = lambda: 29 + timer = self.list.schedule(29, mock.Mock()) + self.assertTrue(timer.active) + timer.cancel() + self.assertFalse(timer.active) + self.list.expire() + self.assertEquals(0, len(timer.func.mock_calls)) + + def test_double_cancel(self): + self.list._now = lambda: 29 + timer = self.list.schedule(29, mock.Mock()) + timer.cancel() + self.assertFalse(timer.active) + timer.cancel() + self.assertFalse(timer.active) + self.list.expire() + self.assertEquals(0, len(timer.func.mock_calls)) + + +@mitogen.core.takes_econtext +def do_timer_test_econtext(econtext): + do_timer_test(econtext.broker) + + +def do_timer_test(broker): + now = mitogen.core.now() + latch = mitogen.core.Latch() + broker.defer(lambda: + broker.timers.schedule( + now + 0.250, + lambda: latch.put('hi'), + ) + ) + + assert 'hi' == latch.get() + assert mitogen.core.now() > (now + 0.250) + + +class BrokerTimerTest(testlib.TestCase): + klass = mitogen.master.Broker + + def test_call_later(self): + broker = self.klass() + try: + do_timer_test(broker) + finally: + broker.shutdown() + broker.join() + + def test_child_upgrade(self): + router = mitogen.master.Router() + try: + c = router.local() + c.call(mitogen.parent.upgrade_router) + c.call(do_timer_test_econtext) + finally: + router.broker.shutdown() + router.broker.join() + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/types_test.py b/tests/types_test.py index 8f120931..8e441c65 100644 --- a/tests/types_test.py +++ b/tests/types_test.py @@ -16,6 +16,11 @@ from mitogen.core import b import testlib +#### +#### see also message_test.py / PickledTest +#### + + class BlobTest(testlib.TestCase): klass = mitogen.core.Blob diff --git a/tests/unix_test.py b/tests/unix_test.py index 02dc11a4..cf3e595f 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -67,12 +67,12 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.unix.Listener def test_constructor_basic(self): - listener = self.klass(router=self.router) + listener = self.klass.build_stream(router=self.router) capture = testlib.LogCapturer() capture.start() try: - self.assertFalse(mitogen.unix.is_path_dead(listener.path)) - os.unlink(listener.path) + self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path)) + os.unlink(listener.protocol.path) # ensure we catch 0 byte read error log message self.broker.shutdown() self.broker.join() @@ -86,25 +86,28 @@ class ClientTest(testlib.TestCase): def _try_connect(self, path): # give server a chance to setup listener - for x in range(10): + timeout = mitogen.core.now() + 30.0 + while True: try: return mitogen.unix.connect(path) - except socket.error: - if x == 9: + except mitogen.unix.ConnectError: + if mitogen.core.now() > timeout: raise time.sleep(0.1) def _test_simple_client(self, path): router, context = self._try_connect(path) - self.assertEquals(0, context.context_id) - self.assertEquals(1, mitogen.context_id) - self.assertEquals(0, mitogen.parent_id) - resp = context.call_service(service_name=MyService, method_name='ping') - self.assertEquals(mitogen.context_id, resp['src_id']) - self.assertEquals(0, resp['auth_id']) - router.broker.shutdown() - router.broker.join() - os.unlink(path) + try: + self.assertEquals(0, context.context_id) + self.assertEquals(1, mitogen.context_id) + self.assertEquals(0, mitogen.parent_id) + resp = context.call_service(service_name=MyService, method_name='ping') + self.assertEquals(mitogen.context_id, resp['src_id']) + self.assertEquals(0, resp['auth_id']) + finally: + router.broker.shutdown() + router.broker.join() + os.unlink(path) @classmethod def _test_simple_server(cls, path): @@ -112,7 +115,7 @@ class ClientTest(testlib.TestCase): latch = mitogen.core.Latch() try: try: - listener = cls.klass(path=path, router=router) + listener = cls.klass.build_stream(path=path, router=router) pool = mitogen.service.Pool(router=router, services=[ MyService(latch=latch, router=router), ])