diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 3c3684f4..43d4f4cb 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +import collections import glob import os import signal @@ -44,6 +45,12 @@ with ci_lib.Fold('job_setup'): if not path.endswith('default.hosts'): ci_lib.run("ln -s %s %s", path, HOSTS_DIR) + distros = collections.defaultdict(list) + families = collections.defaultdict(list) + for container in containers: + distros[container['distro']].append(container['name']) + families[container['family']].append(container['name']) + inventory_path = os.path.join(HOSTS_DIR, 'target') with open(inventory_path, 'w') as fp: fp.write('[test-targets]\n') @@ -59,6 +66,16 @@ with ci_lib.Fold('job_setup'): for container in containers ) + for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]): + fp.write('\n[%s]\n' % distro) + fp.writelines('%s\n' % name for name in hostnames) + + for family, hostnames in sorted(families.items(), key=lambda t: t[0]): + fp.write('\n[%s]\n' % family) + fp.writelines('%s\n' % name for name in hostnames) + + fp.write('\n[linux:children]\ntest-targets\n') + ci_lib.dump_file(inventory_path) if not ci_lib.exists_in_path('sshpass'): diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 50f1a16b..ed516d72 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -1,17 +1,30 @@ +# Each step entry runs a task (Azure Pipelines analog of an Ansible module). +# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool -parameters: - name: '' - pool: '' - sign: false +# `{script: ...}` is shorthand for `{task: CmdLine@, inputs: {script: ...}}`. +# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines +# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines steps: - task: UsePythonVersion@0 displayName: Install python inputs: + githubToken: '$(GITHUB_PYVER_TOKEN)' versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') -- script: python -mpip install tox +- script: | + type python + python --version + displayName: Show python version + +- script: | + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + displayName: Install build deps + condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) + +- script: python -mpip install "tox<4.0" displayName: Install tooling - script: python -mtox -e "$(tox.env)" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 6f45397b..f672240d 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -9,78 +9,25 @@ #ANSIBLE_VERBOSITY: 3 jobs: -- job: Mac1015 +- job: mac11 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md - vmImage: macOS-10.15 - strategy: - matrix: - Mito_27: - python.version: '2.7' - tox.env: py27-mode_mitogen - Mito_36: - python.version: '3.6' - tox.env: py36-mode_mitogen - Mito_310: - python.version: '3.10' - tox.env: py310-mode_mitogen - - # TODO: test python3, python3 tests are broken - Loc_27_210: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 - Loc_27_3: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible3 - Loc_27_4: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible4 - - # NOTE: this hangs when ran in Ubuntu 18.04 - Van_27_210: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_3: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible3 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - -- job: Mac11 - # vanilla Ansible is really slow - timeoutInMinutes: 120 - steps: - - template: azure-pipelines-steps.yml - pool: - # https://github.com/actions/virtual-environments/blob/main/images/macos/ + # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md vmImage: macOS-11 strategy: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_37: - python.version: '3.7' - tox.env: py37-mode_mitogen - Mito_310: - python.version: '3.10' - tox.env: py310-mode_mitogen + Mito_311: + python.version: '3.11' + tox.env: py311-mode_mitogen # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_3: - tox.env: py27-mode_localhost-ansible3 Loc_27_4: tox.env: py27-mode_localhost-ansible4 @@ -89,10 +36,6 @@ jobs: tox.env: py27-mode_localhost-ansible2.10 STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_3: - tox.env: py27-mode_localhost-ansible3 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive Van_27_4: tox.env: py27-mode_localhost-ansible4 STRATEGY: linear @@ -100,38 +43,29 @@ jobs: - job: Linux pool: - # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md - vmImage: "Ubuntu 18.04" + # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md + vmImage: ubuntu-20.04 steps: - template: azure-pipelines-steps.yml strategy: matrix: Mito_27_centos6: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos6 Mito_27_centos7: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos7 Mito_27_centos8: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos8 Mito_27_debian9: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian9 Mito_27_debian10: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian10 Mito_27_debian11: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian11 Mito_27_ubuntu1604: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu1604 Mito_27_ubuntu1804: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu1804 Mito_27_ubuntu2004: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu2004 Mito_36_centos6: @@ -162,100 +96,58 @@ jobs: python.version: '3.6' tox.env: py36-mode_mitogen-distro_ubuntu2004 - Mito_310_centos6: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos6 - Mito_310_centos7: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos7 - Mito_310_centos8: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos8 - Mito_310_debian9: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian9 - Mito_310_debian10: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian10 - Mito_310_debian11: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian11 - Mito_310_ubuntu1604: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1604 - Mito_310_ubuntu1804: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1804 - Mito_310_ubuntu2004: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu2004 - - #DebOps_2460_27_27: - #python.version: '2.7' - #MODE: debops_common - #VER: 2.4.6.0 - - #DebOps_262_36_27: - #python.version: '3.6' - #MODE: debops_common - #VER: 2.6.2 - - #Ansible_2460_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - - #Ansible_2460_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.6.2 - - #Vanilla_262_27: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - #DISTROS: debian - #STRATEGY: linear + Mito_311_centos6: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos6 + Mito_311_centos7: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos7 + Mito_311_centos8: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos8 + Mito_311_debian9: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian9 + Mito_311_debian10: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian10 + Mito_311_debian11: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian11 + Mito_311_ubuntu1604: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu1604 + Mito_311_ubuntu1804: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu1804 + Mito_311_ubuntu2004: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu2004 Ans_27_210: - python.version: '2.7' tox.env: py27-mode_ansible-ansible2.10 - Ans_27_3: - python.version: '2.7' - tox.env: py27-mode_ansible-ansible3 Ans_27_4: - python.version: '2.7' tox.env: py27-mode_ansible-ansible4 Ans_36_210: python.version: '3.6' tox.env: py36-mode_ansible-ansible2.10 - Ans_36_3: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible3 Ans_36_4: python.version: '3.6' tox.env: py36-mode_ansible-ansible4 - Ans_310_210: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible2.10 - Ans_310_3: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible3 - Ans_310_4: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible4 - Ans_310_5: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible5 + Ans_311_210: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible2.10 + Ans_311_3: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible3 + Ans_311_4: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible4 + Ans_311_5: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible5 + Ans_311_6: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible6 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 000a7a83..3e716385 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -2,13 +2,19 @@ from __future__ import absolute_import from __future__ import print_function import atexit +import errno import os +import re import shlex import shutil -import subprocess import sys import tempfile +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess + try: import urlparse except ImportError: @@ -30,40 +36,30 @@ def print(*args, **kwargs): file.flush() -# -# check_output() monkeypatch cutpasted from testlib.py -# - -def subprocess__check_output(*popenargs, **kwargs): - # Missing from 2.6. - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, _ = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output - -if not hasattr(subprocess, 'check_output'): - subprocess.check_output = subprocess__check_output - +def _have_cmd(args): + try: + subprocess.run( + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + except subprocess.CallProcessError: + return False + return True -# ------------------ def have_apt(): - proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['apt', '--help']) + def have_brew(): - proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['brew', 'help']) def have_docker(): - proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['docker', 'info']) def _argv(s, *args): @@ -229,31 +225,20 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def image_for_distro(distro): - """Return the container image name or path for a test distro name. - - The returned value is suitable for use with `docker pull`. - - >>> image_for_distro('centos5') - 'public.ecr.aws/n5z0e8q9/centos5-test' - >>> image_for_distro('centos5-something_custom') - 'public.ecr.aws/n5z0e8q9/centos5-test' - """ - return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],) - - def make_containers(name_prefix='', port_offset=0): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian', 'centos6'] + >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] >>> pprint.pprint(make_containers()) - [{'distro': 'debian', + [{'distro': 'debian11', + 'family': 'debian', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/debian-test', - 'name': 'target-debian-1', + 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'name': 'target-debian11-1', 'port': 2201, 'python_path': '/usr/bin/python'}, {'distro': 'centos6', + 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', 'name': 'target-centos6-2', @@ -261,31 +246,39 @@ def make_containers(name_prefix='', port_offset=0): 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() - firstbit = lambda s: (s+'-').split('-')[0] - secondbit = lambda s: (s+'-').split('-')[1] - + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) i = 1 lst = [] for distro in DISTROS: - distro, star, count = distro.partition('*') - if star: + d = distro_pattern.match(distro).groupdict(default=None) + distro = d['distro'] + family = d['family'] + image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + + if d['py'] == 'py3': + python_path = '/usr/bin/python3' + else: + python_path = '/usr/bin/python' + + if d['count']: count = int(count) else: count = 1 for x in range(count): lst.append({ - "distro": firstbit(distro), - "image": image_for_distro(distro), + "distro": distro, "family": family, "image": image, "name": name_prefix + ("target-%s-%s" % (distro, i)), "hostname": docker_hostname, "port": BASE_PORT + i + port_offset, - "python_path": ( - '/usr/bin/python3' - if secondbit(distro) == 'py3' - else '/usr/bin/python' - ) + "python_path": python_path, }) i += 1 @@ -310,18 +303,24 @@ def proc_is_docker(pid): def get_interesting_procs(container_name=None): + """ + Return a list of (pid, line) tuples for processes considered interesting. + """ args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand='] if container_name is not None: args = ['docker', 'exec', container_name] + args out = [] - for line in subprocess__check_output(args).decode().splitlines(): + for line in subprocess.check_output(args).decode().splitlines(): ppid, pid, comm, rest = line.split(None, 3) if ( ( any(comm.startswith(s) for s in INTERESTING_COMMS) or 'mitogen:' in rest ) and + ( + 'WALinuxAgent' not in rest + ) and ( container_name is not None or (not proc_is_docker(pid)) diff --git a/ansible_mitogen/compat/simplejson/__init__.py b/ansible_mitogen/compat/simplejson/__init__.py deleted file mode 100644 index d5b4d399..00000000 --- a/ansible_mitogen/compat/simplejson/__init__.py +++ /dev/null @@ -1,318 +0,0 @@ -r"""JSON (JavaScript Object Notation) is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> import decimal - >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.0.9' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONEncoder', -] - -__author__ = 'Bob Ippolito ' - -from decoder import JSONDecoder -from encoder import JSONEncoder - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a non-negative integer, then JSON array elements and object - members will be pretty-printed with that indent level. An indent level - of 0 will only insert newlines. ``None`` is the most compact representation. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a non-negative integer, then JSON array elements and - object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - If the contents of ``fp`` is encoded with an ASCII based encoding other - than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must - be specified. Encodings that are not ASCII based (such as UCS-2) are - not allowed, and should be wrapped with - ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` - object and passed to ``loads()`` - - ``object_hook`` is an optional function that will be called with the - result of any object literal decode (a ``dict``). The return value of - ``object_hook`` will be used instead of the ``dict``. This feature - can be used to implement custom decoders (e.g. JSON-RPC class hinting). - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding - other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name - must be specified. Encodings that are not ASCII based (such as UCS-2) - are not allowed and should be decoded to ``unicode`` first. - - ``object_hook`` is an optional function that will be called with the - result of any object literal decode (a ``dict``). The return value of - ``object_hook`` will be used instead of the ``dict``. This feature - can be used to implement custom decoders (e.g. JSON-RPC class hinting). - - ``parse_float``, if specified, will be called with the string - of every JSON float to be decoded. By default this is equivalent to - float(num_str). This can be used to use another datatype or parser - for JSON floats (e.g. decimal.Decimal). - - ``parse_int``, if specified, will be called with the string - of every JSON int to be decoded. By default this is equivalent to - int(num_str). This can be used to use another datatype or parser - for JSON integers (e.g. float). - - ``parse_constant``, if specified, will be called with one of the - following strings: -Infinity, Infinity, NaN, null, true, false. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - return cls(encoding=encoding, **kw).decode(s) diff --git a/ansible_mitogen/compat/simplejson/decoder.py b/ansible_mitogen/compat/simplejson/decoder.py deleted file mode 100644 index b769ea48..00000000 --- a/ansible_mitogen/compat/simplejson/decoder.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -try: - from simplejson._speedups import scanstring as c_scanstring -except ImportError: - c_scanstring = None - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise ValueError( - errmsg("Unterminated string starting at", s, begin)) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise ValueError(errmsg(msg, s, end)) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise ValueError( - errmsg("Unterminated string starting at", s, begin)) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise ValueError(errmsg(msg, s, end)) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise ValueError(errmsg(msg, s, end)) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise ValueError(errmsg(msg, s, end)) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise ValueError(errmsg(msg, s, end)) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - pairs = {} - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - return pairs, end + 1 - elif nextchar != '"': - raise ValueError(errmsg("Expecting property name", s, end)) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise ValueError(errmsg("Expecting : delimiter", s, end)) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise ValueError(errmsg("Expecting object", s, end)) - pairs[key] = value - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise ValueError(errmsg("Expecting property name", s, end - 1)) - - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise ValueError(errmsg("Expecting object", s, end)) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise ValueError(errmsg("Expecting , delimiter", s, end)) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True): - """``encoding`` determines the encoding used to interpret any ``str`` - objects decoded by this instance (utf-8 by default). It has no - effect when decoding ``unicode`` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as ``unicode``. - - ``object_hook``, if specified, will be called with the result - of every JSON object decoded and its return value will be used in - place of the given ``dict``. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - ``parse_float``, if specified, will be called with the string - of every JSON float to be decoded. By default this is equivalent to - float(num_str). This can be used to use another datatype or parser - for JSON floats (e.g. decimal.Decimal). - - ``parse_int``, if specified, will be called with the string - of every JSON int to be decoded. By default this is equivalent to - int(num_str). This can be used to use another datatype or parser - for JSON integers (e.g. float). - - ``parse_constant``, if specified, will be called with one of the - following strings: -Infinity, Infinity, NaN. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise ValueError(errmsg("Extra data", s, end, len(s))) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning - with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise ValueError("No JSON object could be decoded") - return obj, end diff --git a/ansible_mitogen/compat/simplejson/encoder.py b/ansible_mitogen/compat/simplejson/encoder.py deleted file mode 100644 index cf582903..00000000 --- a/ansible_mitogen/compat/simplejson/encoder.py +++ /dev/null @@ -1,440 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re - -try: - from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii -except ImportError: - c_encode_basestring_ascii = None -try: - from simplejson._speedups import make_encoder as c_make_encoder -except ImportError: - c_make_encoder = None - -ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -# Assume this produces an infinity on all machines (probably not guaranteed) -INFINITY = float('1e66666') -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - def replace(match): - return ESCAPE_DCT[match.group(0)] - return '"' + ESCAPE.sub(replace, s) + '"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii - -class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a non-negative integer, then JSON array - elements and object members will be pretty-printed with that - indent level. An indent level of 0 will only insert newlines. - None is the most compact representation. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - return ''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): - # Check for specials. Note that this type of test is processor- and/or - # platform-specific, so do tests which don't depend on the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys: - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot) - return _iterencode(o, 0) - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - False=False, - True=True, - ValueError=ValueError, - basestring=basestring, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - long=long, - str=str, - tuple=tuple, - ): - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, basestring): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, (int, long)): - yield buf + str(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (' ' * (_indent * _current_indent_level)) - yield ']' - if markers is not None: - del markers[markerid] - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _sort_keys: - items = dct.items() - items.sort(key=lambda kv: kv[0]) - else: - items = dct.iteritems() - for key, value in items: - if isinstance(key, basestring): - pass - # JavaScript is weakly typed for these, so it makes sense to - # also allow them. Many encoders seem to do something like this. - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, (int, long)): - key = str(key) - elif _skipkeys: - continue - else: - raise TypeError("key " + repr(key) + " is not a string") - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, basestring): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, (int, long)): - yield str(value) - elif isinstance(value, float): - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (' ' * (_indent * _current_indent_level)) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, basestring): - yield _encoder(o) - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, (int, long)): - yield str(o) - elif isinstance(o, float): - yield _floatstr(o) - elif isinstance(o, (list, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - else: - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/ansible_mitogen/compat/simplejson/scanner.py b/ansible_mitogen/compat/simplejson/scanner.py deleted file mode 100644 index adbc6ec9..00000000 --- a/ansible_mitogen/compat/simplejson/scanner.py +++ /dev/null @@ -1,65 +0,0 @@ -"""JSON token scanner -""" -import re -try: - from simplejson._speedups import make_scanner as c_make_scanner -except ImportError: - c_make_scanner = None - -__all__ = ['make_scanner'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - - def _scan_once(string, idx): - try: - nextchar = string[idx] - except IndexError: - raise StopIteration - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise StopIteration - - return _scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 6df3dfcf..44caf9ac 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -484,6 +484,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): login_context = None #: Only sudo, su, and doas are supported for now. + # Ansible ConnectionBase attribute, removed in Ansible >= 2.8 become_methods = ['sudo', 'su', 'doas'] #: Dict containing init_child() return value as recorded at startup by @@ -521,15 +522,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): # set by `_get_task_vars()` for interpreter discovery _action = None - def __del__(self): - """ - Ansible cannot be trusted to always call close() e.g. the synchronize - action constructs a local connection like this. So provide a destructor - in the hopes of catching these cases. - """ - # https://github.com/dw/mitogen/issues/140 - self.close() - def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir): """ Invoked by ActionModuleMixin to indicate a new task is about to start @@ -684,6 +676,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): @property def connected(self): + """ + Ansible connection plugin property. Used by ansible-connection command. + """ return self.context is not None def _spec_from_via(self, proxied_inventory_name, via_spec): @@ -842,7 +837,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): the _connect_*() service calls defined above to cause the master process to establish the real connection on our behalf, or return a reference to the existing one. + + Ansible connection plugin method. """ + # In some Ansible connection plugins this method returns self. + # However nothing I've found uses it, it's not even assigned. if self.connected: return @@ -880,6 +879,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): Arrange for the mitogen.master.Router running in the worker to gracefully shut down, and wait for shutdown to complete. Safe to call multiple times. + + Ansible connection plugin method. """ self._put_connection() if self.binding: @@ -896,6 +897,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): any local state we hold for the connection, returns the Connection to the 'disconnected' state, and informs ContextService the connection is bad somehow, and should be shut down and discarded. + + Ansible connection plugin method. """ if self._play_context.remote_addr is None: # <2.5.6 incorrectly populate PlayContext for reset_connection @@ -1002,6 +1005,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): Data to supply on ``stdin`` of the process. :returns: (return code, stdout bytes, stderr bytes) + + Ansible connection plugin method. """ emulate_tty = (not in_data and sudoable) rc, stdout, stderr = self.get_chain().call( @@ -1027,6 +1032,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): Remote filesystem path to read. :param str out_path: Local filesystem path to write. + + Ansible connection plugin method. """ self._connect() ansible_mitogen.target.transfer_file( @@ -1076,6 +1083,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): Local filesystem path to read. :param str out_path: Remote filesystem path to write. + + Ansible connection plugin method. """ try: st = os.stat(in_path) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index cd05feac..1f4d8fc6 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -39,6 +39,7 @@ import ansible_mitogen.utils __all__ = [ 'action_loader', + 'become_loader', 'connection_loader', 'module_loader', 'module_utils_loader', @@ -48,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 12) +ANSIBLE_VERSION_MAX = (2, 13) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" @@ -90,6 +91,7 @@ assert_supported_release() from ansible.plugins.loader import action_loader +from ansible.plugins.loader import become_loader from ansible.plugins.loader import connection_loader from ansible.plugins.loader import module_loader from ansible.plugins.loader import module_utils_loader diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 86690f7d..0b1b7aab 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -323,6 +323,7 @@ class NewStylePlanner(ScriptPlanner): 'dnf', # issue #280; py-dnf/hawkey need therapy 'firewalld', # issue #570: ansible module_utils caches dbus conn 'ansible.legacy.dnf', # issue #776 + 'ansible.builtin.dnf', # issue #832 ]) def should_fork(self): diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index 5a6abaa9..75f2d42f 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -42,6 +42,24 @@ DOCUMENTATION = """ accepts. version_added: "2.5" options: + ssh_args: + type: str + vars: + - name: ssh_args + - name: ansible_ssh_args + - name: ansible_mitogen_ssh_args + ssh_common_args: + type: str + vars: + - name: ssh_args + - name: ansible_ssh_common_args + - name: ansible_mitogen_ssh_common_args + ssh_extra_args: + type: str + vars: + - name: ssh_args + - name: ansible_ssh_extra_args + - name: ansible_mitogen_ssh_extra_args """ try: diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 9276614b..63caa88a 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -180,42 +180,6 @@ def setup_pool(pool): LOG.debug('Service pool configured: size=%d', pool.size) -def _setup_simplejson(responder): - """ - We support serving simplejson for Python 2.4 targets on Ansible 2.3, at - least so the package's own CI Docker scripts can run without external - help, however newer versions of simplejson no longer support Python - 2.4. Therefore override any installed/loaded version with a - 2.4-compatible version we ship in the compat/ directory. - """ - responder.whitelist_prefix('simplejson') - - # issue #536: must be at end of sys.path, in case existing newer - # version is already loaded. - compat_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.append(compat_path) - - for fullname, is_pkg, suffix in ( - (u'simplejson', True, '__init__.py'), - (u'simplejson.decoder', False, 'decoder.py'), - (u'simplejson.encoder', False, 'encoder.py'), - (u'simplejson.scanner', False, 'scanner.py'), - ): - path = os.path.join(compat_path, 'simplejson', suffix) - fp = open(path, 'rb') - try: - source = fp.read() - finally: - fp.close() - - responder.add_source_override( - fullname=fullname, - path=path, - source=source, - is_pkg=is_pkg, - ) - - def _setup_responder(responder): """ Configure :class:`mitogen.master.ModuleResponder` to only permit @@ -223,7 +187,6 @@ def _setup_responder(responder): """ responder.whitelist_prefix('ansible') responder.whitelist_prefix('ansible_mitogen') - _setup_simplejson(responder) # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 31ccf1cf..c4cb71ff 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -41,6 +41,7 @@ __metaclass__ = type import atexit import imp +import json import os import re import shlex @@ -63,12 +64,6 @@ except ImportError: # Python 2.4 ctypes = None -try: - import json -except ImportError: - # Python 2.4 - import simplejson as json - try: # Cannot use cStringIO as it does not support Unicode. from StringIO import StringIO diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 2c65c516..7d907d62 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -38,6 +38,7 @@ __metaclass__ = type import errno import grp +import json import operator import os import pwd @@ -58,11 +59,6 @@ import mitogen.parent import mitogen.service from mitogen.core import b -try: - import json -except ImportError: - import simplejson as json - try: reduce except NameError: @@ -371,11 +367,6 @@ def init_child(econtext, log_level, candidate_temp_dirs): LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) - # issue #536: if the json module is available, remove simplejson from the - # importer whitelist to avoid confusing certain Ansible modules. - if json.__name__ == 'json': - econtext.importer.whitelist.remove('simplejson') - global _fork_parent if FORK_SUPPORTED: mitogen.parent.upgrade_router(econtext) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index b488b851..5fc78185 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -79,6 +79,7 @@ try: except ImportError: from ansible.vars.unsafe_proxy import AnsibleUnsafeText +import ansible_mitogen.loaders import mitogen.core @@ -435,7 +436,10 @@ class PlayContextSpec(Spec): return self._play_context.become_user def become_pass(self): - return optional_secret(self._play_context.become_pass) + become_method = self.become_method() + become_plugin = ansible_mitogen.loaders.become_loader.get(become_method) + become_pass = become_plugin.get_option('become_pass', hostvars=self._task_vars) + return optional_secret(become_pass) def password(self): return optional_secret(self._play_context.password) @@ -652,8 +656,8 @@ class MitogenViaSpec(Spec): def become_pass(self): return optional_secret( - self._host_vars.get('ansible_become_password') or - self._host_vars.get('ansible_become_pass') + self._host_vars.get('ansible_become_pass') or + self._host_vars.get('ansible_become_password') ) def password(self): @@ -749,7 +753,7 @@ class MitogenViaSpec(Spec): return self._host_vars.get('mitogen_kubectl_path') def mitogen_lxc_path(self): - return self.host_vars.get('mitogen_lxc_path') + return self._host_vars.get('mitogen_lxc_path') def mitogen_lxc_attach_path(self): return self._host_vars.get('mitogen_lxc_attach_path') diff --git a/docs/_templates/github.html b/docs/_templates/github.html index bb2b5ee5..e6ed304a 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,4 +1,4 @@


-Star +Star

diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index d3298074..5679537e 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged. .. _Ansible: https://www.ansible.com/ -.. _Bug reports: https://goo.gl/yLKZiJ +.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose Overview @@ -147,8 +147,10 @@ Noteworthy Differences * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. Mitogen 0.3.1+ supports - - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10 - - Ansible 5; with Python 3.8-3.10 + + - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 + - Ansible 5 and 6; with Python 3.8-3.11 + Verify your installation is running one of these versions by checking ``ansible --version`` output. @@ -1248,18 +1250,17 @@ with ``-vvv``. However, certain controller hangs may render ``MITOGEN_DUMP_THREAD_STACKS`` ineffective, or occur too infrequently for interactive reproduction. In these -cases `faulthandler `_ may be used: +cases :py:mod:`faulthandler` may be used with Python >= 3.3: -1. For Python 2, ``pip install faulthandler``. This is unnecessary on Python 3. -2. Once the hang occurs, observe the process tree using ``pstree`` or ``ps +1. Once the hang occurs, observe the process tree using ``pstree`` or ``ps --forest``. -3. The most likely process to be hung is the connection multiplexer, which can +2. The most likely process to be hung is the connection multiplexer, which can easily be identified as the parent of all SSH client processes. -4. Send ``kill -SEGV `` to the multiplexer PID, causing it to print all +3. Send ``kill -SEGV `` to the multiplexer PID, causing it to print all thread stacks. -5. `File a bug `_ including a copy - of the stacks, along with a description of the last task executing prior to - the hang. +4. `File a bug `_ + including a copy of the stacks and a description of the last task executing + before the hang It is possible the hang occurred in a process on a target. If ``strace`` is available, look for the host name not listed in Ansible output as reporting a @@ -1292,7 +1293,7 @@ Sample Profiles --------------- The summaries below may be reproduced using data and scripts maintained in the -`pcaps branch `_. Traces were +`pcaps branch `_. Traces were recorded using Ansible 2.5.14. @@ -1301,7 +1302,7 @@ Trivial Loop: Local Host This demonstrates Mitogen vs. SSH pipelining to the local machine running `bench/loop-100-items.yml -`_, +`_, executing a simple command 100 times. Most Ansible controller overhead is isolated, characterizing just module executor and connection layer performance. Mitogen requires **63x less bandwidth and 5.9x less time**. @@ -1329,7 +1330,7 @@ File Transfer: UK to France ~~~~~~~~~~~~~~~~~~~~~~~~~~~ `This playbook -`_ +`_ was used to compare file transfer performance over a ~26 ms link. It uses the ``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the target. diff --git a/docs/changelog.rst b/docs/changelog.rst index c08a91dd..038145b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,10 +15,28 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file -`directly from GitHub `_. +`directly from GitHub `_. +Unreleased +---------- -v0.3.3.dev0 +* :gh:issue:`987` Support Python 3.11 + + +v0.3.4 (2023-07-02) +------------------- + +* :gh:issue:`929` Support Ansible 6 and ansible-core 2.13 +* :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times +* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the + connection on destruction. This is expected to reduce cases of `mitogen.core.Error: An attempt + was made to enqueue a message with a Broker that has already exitted`. However it may result in + resource leaks. +* :gh:issue:`659` Removed :mod:`mitogen.compat.simplejson`, not needed with Python 2.7+, contained Python 3.x syntax errors +* :gh:issue:`983` CI: Removed PyPI faulthandler requirement from tests +* :gh:issue:`1001` CI: Fixed Debian 9 & 11 tests + +v0.3.3 (2022-06-03) ------------------- * :gh:issue:`906` Support packages dynamically inserted into sys.modules, e.g. `distro` >= 1.7.0 as `ansible.module_utils.distro`. @@ -53,7 +71,7 @@ v0.3.0 (2021-11-24) ------------------- This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. -`See here for details `_. +`See here for details `_. * :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports * :gh:issue:`770` better check for supported Ansible version @@ -74,7 +92,7 @@ v0.2.10 (2021-11-24) * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in - :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` + :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 * :gh:issue:`775` Add msvcrt to the default module deny list @@ -164,7 +182,7 @@ Mitogen for Ansible :linux:man7:`unix` sockets across privilege domains. * :gh:issue:`467`: an incompatibility running Mitogen under `Molecule - `_ was resolved. + `_ was resolved. * :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of connections, ``async`` tasks, tasks using custom :mod:`module_utils`, @@ -1216,9 +1234,8 @@ Core Library parameter may specify an argument vector prefix rather than a string program path. -* :gh:issue:`300`: the broker could crash on - OS X during shutdown due to scheduled `kqueue - `_ filter changes for +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to + scheduled :freebsd:man2:`kqueue` filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. diff --git a/docs/conf.py b/docs/conf.py index 47c4afbe..914f199e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,8 @@ -import os import sys sys.path.extend(['..', '.']) import mitogen -VERSION = '%s.%s.%s' % mitogen.__version__ +VERSION = '.'.join(str(part) for part in mitogen.__version__) author = u'Network Genomics' copyright = u'2021, the Mitogen authors' @@ -43,15 +42,15 @@ version = VERSION domainrefs = { 'gh:commit': { 'text': '%s', - 'url': 'https://github.com/dw/mitogen/commit/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/commit/%s', }, 'gh:issue': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/issues/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/issues/%s', }, 'gh:pull': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/pull/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/pull/%s', }, 'ans:mod': { 'text': '%s module', @@ -63,23 +62,23 @@ domainrefs = { }, 'freebsd:man2': { 'text': '%s(2)', - 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + 'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s', }, 'linux:man1': { 'text': '%s(1)', - 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + 'url': 'https://man7.org/linux/man-pages/man1/%s.1.html', }, 'linux:man2': { 'text': '%s(2)', - 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + 'url': 'https://man7.org/linux/man-pages/man2/%s.2.html', }, 'linux:man3': { 'text': '%s(3)', - 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + 'url': 'https://man7.org/linux/man-pages/man3/%s.3.html', }, 'linux:man7': { 'text': '%s(7)', - 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + 'url': 'https://man7.org/linux/man-pages/man7/%s.7.html', }, } diff --git a/docs/index.rst b/docs/index.rst index d33cf29f..32083db0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_ can be built, and while the API is quite friendly and comparable to `Fabric`_, ultimately it is not intended for direct use by consumer software. -.. _Salt: https://docs.saltstack.com/en/latest/ +.. _Salt: https://docs.saltproject.io/en/latest/ .. _Ansible: https://docs.ansible.com/ .. _Fabric: https://www.fabfile.org/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c4674fd..a2894c82 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ +docutils<0.18 +Jinja2<3 +MarkupSafe<2.1 Sphinx==2.1.2; python_version > '3.0' sphinxcontrib-programoutput==0.14; python_version > '3.0' alabaster==0.7.10; python_version > '3.0' diff --git a/mitogen/__init__.py b/mitogen/__init__.py index fc3eb926..db4e8b3e 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 3, 'dev0') +__version__ = (0, 3, 5, 'dev0') #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/master.py b/mitogen/master.py index 3a163a8b..4fb535f0 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -536,7 +536,7 @@ class PkgutilMethod(FinderMethod): try: path = loader.get_filename(fullname) - except (AttributeError, ImportError): + except (AttributeError, ImportError, ValueError): # - get_filename() may throw ImportError if pkgutil.find_loader() # picks a "parent" package's loader for some crap that's been # stuffed in sys.modules, for example in the case of urllib3: diff --git a/mitogen/parent.py b/mitogen/parent.py index 32aa3cb6..59ee1685 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux :linux:man2:`epoll` interface. + Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' diff --git a/mitogen/service.py b/mitogen/service.py index 0e5f6419..7fde9013 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -752,10 +752,12 @@ class PushFileService(Service): One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. - overridden_sources: optional dict containing source code to override path's source code - extra_sys_paths: loads additional sys paths for use in finding modules; beneficial - in situations like loading Ansible Collections because source code - dependencies come from different file paths than where the source lives + :param dict overridden_sources: + Optional dict containing source code to override path's source code + :param extra_sys_paths: + Loads additional sys paths for use in finding modules; beneficial + in situations like loading Ansible Collections because source code + dependencies come from different file paths than where the source lives """ for path in paths: overridden_source = None diff --git a/setup.py b/setup.py index 9d2ff36f..4d7fadfc 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index dfc2858e..c34dd219 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,4 +1,12 @@ [defaults] +any_errors_fatal = true +# callback_whitelist naming will be deprecated in ansible-core >= 2.15. +# callbacks_enabled naming was added in ansible-core 2.11 +# profile_tasks: Displays timing for each task and summary table of top N tasks +# timer: Displays "Playbook run took 0 days, 0 hours, ..." +callback_whitelist = + profile_tasks, + timer inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy @@ -6,15 +14,12 @@ inventory_plugins = lib/inventory action_plugins = lib/action callback_plugins = lib/callback stdout_callback = yaml -stdout_whitelist = - profile_roles, - timer, - yaml vars_plugins = lib/vars library = lib/modules filter_plugins = lib/filters module_utils = lib/module_utils retry_files_enabled = False +show_task_path_on_failure = true # Added in ansible-core 2.11 display_args_to_stdout = True forks = 100 @@ -30,14 +35,34 @@ transport = ssh no_target_syslog = True # Required by integration/ssh/timeouts.yml -timeout = 10 +timeout = 30 # On Travis, paramiko check fails due to host key checking enabled. host_key_checking = False +[inventory] +any_unparsed_is_failed = true +host_pattern_mismatch = error + [callback_profile_tasks] task_output_limit = 10 [ssh_connection] -ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s +# https://www.openssh.com/legacy.html +# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. +# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). +# Duplicated cases in +# - tests/ansible/ansible.cfg +# - tests/ansible/integration/connection_delegation/delegate_to_template.yml +# - tests/ansible/integration/connection_delegation/stack_construction.yml +# - tests/ansible/integration/process/unix_socket_cleanup.yml +# - tests/ansible/integration/ssh/variables.yml +# - tests/testlib.py +ssh_args = + -o ControlMaster=auto + -o ControlPersist=60s + -o ForwardAgent=yes + -o HostKeyAlgorithms=+ssh-rsa + -o PubkeyAcceptedKeyTypes=+ssh-rsa + -o UserKnownHostsFile=/dev/null pipelining = True diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index f6702f58..09534af5 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -1,7 +1,6 @@ - name: bench/file_transfer.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Make 32MiB file @@ -52,6 +51,8 @@ with_items: - /tmp/bigfile.out - /tmp/bigbigfile.out + tags: + - requires_local_sudo - name: Copy 32MiB file via localhost sudo delegate_to: localhost @@ -59,6 +60,8 @@ copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + tags: + - requires_local_sudo - name: Copy 320MiB file via localhost sudo delegate_to: localhost @@ -66,6 +69,8 @@ copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + tags: + - requires_local_sudo tags: - resource_intensive diff --git a/tests/ansible/bench/includes.yml b/tests/ansible/bench/includes.yml index 96079874..6241a485 100644 --- a/tests/ansible/bench/includes.yml +++ b/tests/ansible/bench/includes.yml @@ -1,4 +1,5 @@ -- hosts: test-targets +- name: bench/includes.yml + hosts: test-targets tasks: - include_tasks: _includes.yml with_sequence: start=1 end=1000 diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index e25ae552..2a8e7d6f 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -1,6 +1,5 @@ - -- hosts: all - any_errors_fatal: true +- name: bench/loop-100-copies.yml + hosts: all tasks: - name: Create file tree diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml index e711301d..ad298638 100644 --- a/tests/ansible/bench/loop-100-items.yml +++ b/tests/ansible/bench/loop-100-items.yml @@ -4,7 +4,8 @@ # # See also: loop-100-tasks.yml # -- hosts: all +- name: bench/loop-100-items.yml + hosts: all tasks: - command: hostname with_sequence: start=1 end="{{end|default(100)}}" diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml index 4a76c4fe..211cecd1 100644 --- a/tests/ansible/bench/loop-100-tasks.yml +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -8,7 +8,8 @@ # # See also: loop-100-items.yml # -- hosts: all +- name: bench/loop-100-tasks.yml + hosts: all tasks: - command: hostname - command: hostname diff --git a/tests/ansible/bench/loop-20-templates.yml b/tests/ansible/bench/loop-20-templates.yml index 17dc7777..4035ea34 100644 --- a/tests/ansible/bench/loop-20-templates.yml +++ b/tests/ansible/bench/loop-20-templates.yml @@ -1,12 +1,14 @@ - -- hosts: all +- name: bench/loop-20-templates.yml + hosts: all tasks: - - file: + - name: Create loop templates dir + file: dest: /tmp/templates state: "{{item}}" with_items: ["absent", "directory"] - - copy: + - name: Copy loop files + copy: dest: /tmp/templates/{{item}} mode: 0755 content: diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml new file mode 100644 index 00000000..cea46113 --- /dev/null +++ b/tests/ansible/hosts/group_vars/all.yml @@ -0,0 +1,2 @@ +--- +pkg_mgr_python_interpreter: python diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml new file mode 100644 index 00000000..7b9e34f6 --- /dev/null +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -0,0 +1,2 @@ +--- +pkg_mgr_python_interpreter: /usr/libexec/platform-python diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 856c6a0f..7d7b526a 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,49 +1,76 @@ # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. +[transport_config:children] +transport_config_undiscover +tc_python_path -# tansport() +[transport_config_undiscover:children] +tc_become +tc_become_method +tc_become_pass +tc_become_user +tc_password +tc_port +tc_remote_addr +tc_remote_user +tc_transport + +[transport_config_undiscover:vars] +# If python interpreter path is unset, Ansible tries to connect & discover it. +# That causes approx 10 seconds timeout per task - there's no host to connect to. +# This optimisation should not be relied in any test. +# Note: tc-python-path-* are intentionally not included. +ansible_python_interpreter = python3000 # Not expected to exist + +[tc_transport] tc-transport-unset tc-transport-local ansible_connection=local tc-transport-smart ansible_connection=smart -# python_path() +[tc_python_path] tc-python-path-unset tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python tc-python-path-local-unset ansible_connection=local tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c -# remote_addr() +[tc_remote_addr] tc-remote-addr-unset # defaults to inventory_hostname tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host tc-remote-addr-explicit-host ansible_host=ansi.host tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d -# password() +[tc_password] tc-password-unset tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass -tc-password-explicit-user ansible_password=ansi-pass +tc-password-explicit-pass ansible_password=ansi-pass tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a -# become() +[tc_remote_user] +tc-remote-user-unset # defaults to C.DEFAULT_REMOTE_USER +tc-remote-user-explicit-ssh ansible_ssh_user=ansi-ssh-user +tc-remote-user-explicit-user ansible_user=ansi-user +tc-remote-user-explicit-both ansible_user=a.b.c ansible_ssh_user=c.b.a + +[tc_become] tc-become-unset tc-become-set -# become_method() +[tc_become_method] tc-become-method-unset tc-become-method-su ansible_become_method=su -# become_user() +[tc_become_user] tc-become-user-unset tc-become-user-set ansible_become_user=ansi-become-user -# become_pass() +[tc_become_pass] tc-become-pass-unset tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass -tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a +tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword -# port() +[tc_port] tc-port-unset tc-port-explicit-port ansible_port=1234 tc-port-explicit-ssh ansible_ssh_port=4321 diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index 611ff3f2..5dadff9a 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -2,23 +2,23 @@ - name: integration/action/copy.yml hosts: test-targets - any_errors_fatal: true tasks: - - copy: + - name: Create tiny file + copy: dest: /tmp/copy-tiny-file content: this is a tiny file. delegate_to: localhost - - copy: + - name: Create large file + copy: dest: /tmp/copy-large-file # Must be larger than Connection.SMALL_SIZE_LIMIT. content: "{% for x in range(200000) %}x{% endfor %}" delegate_to: localhost - # end of making files - - - file: + - name: Cleanup copied files + file: state: absent path: "{{item}}" with_items: @@ -27,28 +27,31 @@ - /tmp/copy-tiny-inline-file.out - /tmp/copy-large-inline-file.out - # end of cleaning out files - - - copy: + - name: Copy large file + copy: dest: /tmp/copy-large-file.out src: /tmp/copy-large-file - - copy: + - name: Copy tiny file + copy: dest: /tmp/copy-tiny-file.out src: /tmp/copy-tiny-file - - copy: + - name: Copy tiny inline file + copy: dest: /tmp/copy-tiny-inline-file.out content: "tiny inline content" - - copy: + - name: Copy large inline file + copy: dest: /tmp/copy-large-inline-file.out content: | {% for x in range(200000) %}y{% endfor %} # stat results - - stat: + - name: Stat copied files + stat: path: "{{item}}" with_items: - /tmp/copy-tiny-file.out @@ -65,7 +68,8 @@ - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" fail_msg: stat={{stat}} - - file: + - name: Cleanup files + file: state: absent path: "{{item}}" with_items: diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index 1ba19e6f..fa4765f8 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -5,12 +5,7 @@ - name: integration/action/fixup_perms2__copy.yml hosts: test-targets - any_errors_fatal: true tasks: - # - # copy module (no mode). - # - - name: "Copy files (no mode)" copy: content: "" @@ -23,10 +18,6 @@ - out.stat.mode in ("0644", "0664") fail_msg: out={{out}} - # - # copy module (explicit mode). - # - - name: "Copy files from content: arg" copy: content: "" @@ -40,22 +31,20 @@ - out.stat.mode == "0400" fail_msg: out={{out}} - # - # copy module (existing disk files, no mode). - # - - - file: + - name: Cleanup local weird mode file + file: path: /tmp/weird-mode.out state: absent - - name: Create local test file. + - name: Create local weird mode file delegate_to: localhost copy: content: "weird mode" dest: "/tmp/weird-mode" mode: "1462" - - copy: + - name: Copy file with weird mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode.out" @@ -67,11 +56,8 @@ - out.stat.mode in ("0644", "0664") fail_msg: out={{out}} - # - # copy module (existing disk files, preserve mode). - # - - - copy: + - name: Copy file with weird mode, preserving mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode" mode: preserve @@ -84,11 +70,8 @@ - out.stat.mode == "1462" fail_msg: out={{out}} - # - # copy module (existing disk files, explicit mode). - # - - - copy: + - name: Copy file with weird mode, explicit mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode" mode: "1461" @@ -102,7 +85,8 @@ - out.stat.mode == "1461" fail_msg: out={{out}} - - file: + - name: Cleanup + file: state: absent path: "{{item}}" with_items: diff --git a/tests/ansible/integration/action/low_level_execute_command.yml b/tests/ansible/integration/action/low_level_execute_command.yml index 7f91bdf8..24649424 100644 --- a/tests/ansible/integration/action/low_level_execute_command.yml +++ b/tests/ansible/integration/action/low_level_execute_command.yml @@ -2,7 +2,6 @@ - name: integration/action/low_level_execute_command.yml hosts: test-targets - any_errors_fatal: true tasks: # "echo -en" to test we actually hit bash shell too. diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index bfebeffd..8e981250 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -9,7 +9,6 @@ - name: integration/action/make_tmp_path.yml hosts: test-targets - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/action/make_tmp_path__double.yml b/tests/ansible/integration/action/make_tmp_path__double.yml index 811e159d..fd66c13d 100644 --- a/tests/ansible/integration/action/make_tmp_path__double.yml +++ b/tests/ansible/integration/action/make_tmp_path__double.yml @@ -1,7 +1,8 @@ # issue #554: double calls to make_tmp_path() fail with assertion error. Ensure # they succeed and are cleaned up correctly. -- hosts: target +- name: integration/action/make_tmp_path__double.yml + hosts: test-targets tasks: - mitogen_action_script: script: | diff --git a/tests/ansible/integration/action/remote_expand_user.yml b/tests/ansible/integration/action/remote_expand_user.yml index 8e31f657..3a675635 100644 --- a/tests/ansible/integration/action/remote_expand_user.yml +++ b/tests/ansible/integration/action/remote_expand_user.yml @@ -3,7 +3,6 @@ - name: integration/action/remote_expand_user.yml hosts: test-targets - any_errors_fatal: true tasks: - name: "Find out root's homedir." # Runs first because it blats regular Ansible facts with junk, so diff --git a/tests/ansible/integration/action/remote_file_exists.yml b/tests/ansible/integration/action/remote_file_exists.yml index 2d206930..577e64a0 100644 --- a/tests/ansible/integration/action/remote_file_exists.yml +++ b/tests/ansible/integration/action/remote_file_exists.yml @@ -1,38 +1,33 @@ - name: integration/action/remote_file_exists.yml hosts: test-targets - any_errors_fatal: true tasks: - - - file: + - name: Ensure does-not-exist doesnt + file: path: /tmp/does-not-exist state: absent - - action_passthrough: method: _remote_file_exists args: ['/tmp/does-not-exist'] register: out - - assert: that: out.result == False fail_msg: out={{out}} - # --- - - - copy: + - name: Ensure does-exist does + copy: dest: /tmp/does-exist content: "I think, therefore I am" - - action_passthrough: method: _remote_file_exists args: ['/tmp/does-exist'] register: out - - assert: that: out.result == True fail_msg: out={{out}} - - file: + - name: Cleanup + file: path: /tmp/does-exist state: absent tags: diff --git a/tests/ansible/integration/action/remove_tmp_path.yml b/tests/ansible/integration/action/remove_tmp_path.yml index 19ae22e3..d90c90dc 100644 --- a/tests/ansible/integration/action/remove_tmp_path.yml +++ b/tests/ansible/integration/action/remove_tmp_path.yml @@ -4,14 +4,14 @@ # - name: integration/action/remove_tmp_path.yml hosts: test-targets - any_errors_fatal: true tasks: # # Use the copy module to cause a temporary directory to be created, and # return a result with a 'src' attribute pointing into that directory. # - - copy: + - name: Ensure remove_tmp_path_test + copy: dest: /tmp/remove_tmp_path_test content: "{{ 123123 | random }}" register: out diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index a6cd277e..aa87deaf 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -2,7 +2,6 @@ - name: integration/action/synchronize.yml hosts: test-targets - any_errors_fatal: true vars: ansible_user: mitogen__has_sudo_pubkey ansible_become_pass: has_sudo_pubkey_password @@ -13,23 +12,27 @@ ansible_password: '' tasks: # must copy git file to set proper file mode. - - copy: + - name: Copy synchronize-action-key + copy: dest: /tmp/synchronize-action-key src: ../../../data/docker/mitogen__has_sudo_pubkey.key mode: u=rw,go= delegate_to: localhost - - file: + - name: Cleanup sync-test + file: path: /tmp/sync-test state: absent delegate_to: localhost - - file: + - name: Create sync-test + file: path: /tmp/sync-test state: directory delegate_to: localhost - - copy: + - name: Create syn-test item + copy: dest: /tmp/sync-test/item content: "item!" delegate_to: localhost diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml index e0f42d0e..6d4faab5 100644 --- a/tests/ansible/integration/action/transfer_data.yml +++ b/tests/ansible/integration/action/transfer_data.yml @@ -1,49 +1,46 @@ - name: integration/action/transfer_data.yml hosts: test-targets - any_errors_fatal: true tasks: - - - file: + - name: Cleanup transfer data + file: path: /tmp/transfer-data state: absent - # Ensure it JSON-encodes dicts. - - action_passthrough: + - name: Create JSON transfer data + action_passthrough: method: _transfer_data kwargs: remote_path: /tmp/transfer-data data: { "I am JSON": true } - - - slurp: + - name: Slurp JSON transfer data + slurp: src: /tmp/transfer-data register: out - - assert: that: | out.content|b64decode == '{"I am JSON": true}' fail_msg: out={{out}} - - # Ensure it handles strings. - - action_passthrough: + - name: Create text transfer data + action_passthrough: method: _transfer_data kwargs: remote_path: /tmp/transfer-data data: "I am text." - - - slurp: + - name: Slurp text transfer data + slurp: src: /tmp/transfer-data register: out - - assert: that: out.content|b64decode == 'I am text.' fail_msg: out={{out}} - - file: + - name: Cleanup transfer data + file: path: /tmp/transfer-data state: absent tags: diff --git a/tests/ansible/integration/async/multiple_items_loop.yml b/tests/ansible/integration/async/multiple_items_loop.yml index 24d43083..5d070255 100644 --- a/tests/ansible/integration/async/multiple_items_loop.yml +++ b/tests/ansible/integration/async/multiple_items_loop.yml @@ -2,7 +2,6 @@ - name: integration/async/multiple_items_loop.yml hosts: test-targets - any_errors_fatal: true tasks: - name: start long running ops diff --git a/tests/ansible/integration/async/result_binary_producing_json.yml b/tests/ansible/integration/async/result_binary_producing_json.yml index b05b24bf..942b356a 100644 --- a/tests/ansible/integration/async/result_binary_producing_json.yml +++ b/tests/ansible/integration/async/result_binary_producing_json.yml @@ -2,7 +2,6 @@ - name: integration/async/result_binary_producing_json.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - block: @@ -38,7 +37,8 @@ retries: 100000 delay: 0 - - slurp: + - name: Slurp async busy-poll + slurp: src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result diff --git a/tests/ansible/integration/async/result_binary_producing_junk.yml b/tests/ansible/integration/async/result_binary_producing_junk.yml index afdf0c2f..734f0e12 100644 --- a/tests/ansible/integration/async/result_binary_producing_junk.yml +++ b/tests/ansible/integration/async/result_binary_producing_junk.yml @@ -2,7 +2,6 @@ - name: integration/async/result_binary_producing_junk.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - block: diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index 6eb31702..f327a965 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -2,17 +2,19 @@ - name: integration/async/result_shell_echo_hi.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - - shell: echo hi; echo there >&2 + - name: Async shell + shell: echo hi; echo there >&2 async: 100 poll: 0 register: job - - shell: sleep 1 + - name: Sleepy shell + shell: sleep 1 - - slurp: + - name: Slurp async shell + slurp: src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result diff --git a/tests/ansible/integration/async/runner_new_process.yml b/tests/ansible/integration/async/runner_new_process.yml index fbc7261f..0113b658 100644 --- a/tests/ansible/integration/async/runner_new_process.yml +++ b/tests/ansible/integration/async/runner_new_process.yml @@ -2,7 +2,6 @@ - name: integration/async/runner_new_process.yml hosts: test-targets - any_errors_fatal: true tasks: - name: get process ID. diff --git a/tests/ansible/integration/async/runner_one_job.yml b/tests/ansible/integration/async/runner_one_job.yml index 15e02efa..bfb50bfa 100644 --- a/tests/ansible/integration/async/runner_one_job.yml +++ b/tests/ansible/integration/async/runner_one_job.yml @@ -3,7 +3,6 @@ - name: integration/async/runner_one_job.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify output of a single async job. diff --git a/tests/ansible/integration/async/runner_timeout_then_polling.yml b/tests/ansible/integration/async/runner_timeout_then_polling.yml index 6fc46ba7..2f71ebe4 100644 --- a/tests/ansible/integration/async/runner_timeout_then_polling.yml +++ b/tests/ansible/integration/async/runner_timeout_then_polling.yml @@ -2,7 +2,6 @@ - name: integration/async/runner_timeout_then_polling.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify async-with-timeout-then-poll behaviour. diff --git a/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml b/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml index 6eda88aa..5678dbdd 100644 --- a/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml +++ b/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml @@ -1,12 +1,12 @@ - name: integration/async/runner_two_simultaneous_jobs.yml hosts: test-targets - any_errors_fatal: true tasks: # Start 2 duplicate jobs, verify they run concurrently. - - file: + - name: Cleanup semaphore file + file: path: /tmp/flurp state: absent diff --git a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml index a5cbabef..ba413372 100644 --- a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml +++ b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml @@ -2,7 +2,6 @@ - name: integration/async/runner_with_polling_and_timeout.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify async-with-polling-and-timeout behaviour. diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index 7bd6db5a..bd6a0aee 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -4,7 +4,6 @@ - name: integration/become/su_password.yml hosts: test-targets become_method: su - any_errors_fatal: true tasks: - name: Ensure su password absent but required. diff --git a/tests/ansible/integration/become/sudo_flags_failure.yml b/tests/ansible/integration/become/sudo_flags_failure.yml index fdbb712c..75ecfebf 100644 --- a/tests/ansible/integration/become/sudo_flags_failure.yml +++ b/tests/ansible/integration/become/sudo_flags_failure.yml @@ -1,6 +1,5 @@ - name: integration/become/sudo_flags_failure.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify behaviour for bad sudo flags. diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index bc3de7b8..e7a849c2 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -1,6 +1,5 @@ - name: integration/become/sudo_nonexistent.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify behaviour for non-existent accounts. diff --git a/tests/ansible/integration/become/sudo_nopassword.yml b/tests/ansible/integration/become/sudo_nopassword.yml index 110c95b4..98dd9cfd 100644 --- a/tests/ansible/integration/become/sudo_nopassword.yml +++ b/tests/ansible/integration/become/sudo_nopassword.yml @@ -2,7 +2,6 @@ - name: integration/become/sudo_nopassword.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify we aren't root diff --git a/tests/ansible/integration/become/sudo_password.yml b/tests/ansible/integration/become/sudo_password.yml index a0a1dec9..931cba27 100644 --- a/tests/ansible/integration/become/sudo_password.yml +++ b/tests/ansible/integration/become/sudo_password.yml @@ -2,7 +2,6 @@ - name: integration/become/sudo_password.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Ensure sudo password absent but required. diff --git a/tests/ansible/integration/become/sudo_requiretty.yml b/tests/ansible/integration/become/sudo_requiretty.yml index 0bdfaa26..19b49b28 100644 --- a/tests/ansible/integration/become/sudo_requiretty.yml +++ b/tests/ansible/integration/become/sudo_requiretty.yml @@ -2,7 +2,6 @@ - name: integration/become/sudo_requiretty.yml hosts: test-targets - any_errors_fatal: true tasks: # TODO: https://github.com/dw/mitogen/issues/692 diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml index 3092e199..fb5c6014 100644 --- a/tests/ansible/integration/connection/_put_file.yml +++ b/tests/ansible/integration/connection/_put_file.yml @@ -1,19 +1,23 @@ --- -- shell: dd if=/dev/urandom of=/tmp/{{file_name}} bs=1024 count={{file_size}} +- name: Create {{ file_name }} + shell: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size }} args: creates: /tmp/{{file_name}} delegate_to: localhost -- copy: +- name: Copy {{ file_name }} + copy: dest: /tmp/{{file_name}}.out src: /tmp/{{file_name}} -- stat: path=/tmp/{{file_name}} +- name: Stat created {{ file_name }} + stat: path=/tmp/{{ file_name }} register: original delegate_to: localhost -- stat: path=/tmp/{{file_name}}.out +- name: Stat copied {{ file_name }} + stat: path=/tmp/{{ file_name }}.out register: copied - assert: diff --git a/tests/ansible/integration/connection/become_same_user.yml b/tests/ansible/integration/connection/become_same_user.yml index 9f720e5c..5ff4f95b 100644 --- a/tests/ansible/integration/connection/become_same_user.yml +++ b/tests/ansible/integration/connection/become_same_user.yml @@ -4,7 +4,6 @@ - name: integration/connection/become_same_user.yml hosts: bsu-joe gather_facts: no - any_errors_fatal: true tasks: # bsu-joe's login user is joe, so become should be ignored. diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index 1c7b544a..5c74b0ed 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -9,10 +9,13 @@ - meta: end_play when: not is_mitogen - - delegate_to: localhost + - name: Run _disconnect_during_module.yml + delegate_to: localhost command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} integration/connection/_disconnect_during_module.yml args: chdir: ../.. diff --git a/tests/ansible/integration/connection/disconnect_resets_connection.yml b/tests/ansible/integration/connection/disconnect_resets_connection.yml index 3a6f712a..95f96910 100644 --- a/tests/ansible/integration/connection/disconnect_resets_connection.yml +++ b/tests/ansible/integration/connection/disconnect_resets_connection.yml @@ -12,7 +12,6 @@ - name: integration/connection/disconnect_resets_connection.yml hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/connection/exec_command.yml b/tests/ansible/integration/connection/exec_command.yml index 125def1b..f0ec1c1b 100644 --- a/tests/ansible/integration/connection/exec_command.yml +++ b/tests/ansible/integration/connection/exec_command.yml @@ -4,7 +4,6 @@ - name: integration/connection/exec_command.yml hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - connection_passthrough: method: exec_command diff --git a/tests/ansible/integration/connection/home_dir.yml b/tests/ansible/integration/connection/home_dir.yml index 1dbaa5d3..d41657bc 100644 --- a/tests/ansible/integration/connection/home_dir.yml +++ b/tests/ansible/integration/connection/home_dir.yml @@ -2,7 +2,6 @@ - name: integration/connection/home_dir.yml hosts: test-targets - any_errors_fatal: true tasks: - name: "Find out root's homedir." # Runs first because it blats regular Ansible facts with junk, so diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml index 90a13999..ce8f8fe0 100644 --- a/tests/ansible/integration/connection/put_large_file.yml +++ b/tests/ansible/integration/connection/put_large_file.yml @@ -4,7 +4,6 @@ - name: integration/connection/put_large_file.yml hosts: test-targets gather_facts: no - any_errors_fatal: true vars: file_name: large-file file_size: 512 diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml index 49be87a9..3e0eaaa9 100644 --- a/tests/ansible/integration/connection/put_small_file.yml +++ b/tests/ansible/integration/connection/put_small_file.yml @@ -4,7 +4,6 @@ - name: integration/connection/put_small_file.yml hosts: test-targets gather_facts: no - any_errors_fatal: true vars: file_name: small-file file_size: 123 diff --git a/tests/ansible/integration/connection/reset_become.yml b/tests/ansible/integration/connection/reset_become.yml index a36cbf1a..003c375e 100644 --- a/tests/ansible/integration/connection/reset_become.yml +++ b/tests/ansible/integration/connection/reset_become.yml @@ -1,7 +1,8 @@ # issue #633: Connection.reset() should ignore "become", and apply to the login # account. -- hosts: test-targets +- name: integration/connection/reset_become.yml + hosts: test-targets become: true gather_facts: false tasks: diff --git a/tests/ansible/integration/connection_delegation/all.yml b/tests/ansible/integration/connection_delegation/all.yml index 8d5ffe03..cb55bdc7 100644 --- a/tests/ansible/integration/connection_delegation/all.yml +++ b/tests/ansible/integration/connection_delegation/all.yml @@ -1,5 +1,5 @@ - import_playbook: delegate_to_template.yml - import_playbook: local_action.yml -- import_playbook: osa_container_standalone.yml -- import_playbook: osa_delegate_to_self.yml +#- import_playbook: osa_container_standalone.yml +#- import_playbook: osa_delegate_to_self.yml - import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index be083ff9..3776a7db 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -14,7 +14,6 @@ physical_hosts: ["cd-normal-alias", "cd-normal-normal"] hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen @@ -34,7 +33,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -45,14 +44,12 @@ 'python_path': ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -64,7 +61,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, @@ -75,14 +72,12 @@ 'python_path': ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', diff --git a/tests/ansible/integration/connection_delegation/local_action.yml b/tests/ansible/integration/connection_delegation/local_action.yml index 1c9a25ac..6176d770 100644 --- a/tests/ansible/integration/connection_delegation/local_action.yml +++ b/tests/ansible/integration/connection_delegation/local_action.yml @@ -1,7 +1,7 @@ - # issue #251: local_action with mitogen_via= builds wrong stack. -- hosts: cd-newuser-normal-normal +- name: integration/connection_delegation/local_action.yml + hosts: cd-newuser-normal-normal tasks: - meta: end_play when: not is_mitogen @@ -22,7 +22,7 @@ { 'enable_lru': true, 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'python_path': ["{{ansible_playbook_python}}"], 'remote_name': null, 'password': null, diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index ebf0f54f..279a7b2b 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -30,7 +30,6 @@ - hosts: cd-normal - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen @@ -42,7 +41,7 @@ right: [ { "kwargs": { - "connect_timeout": 10, + "connect_timeout": 30, "doas_path": null, "password": null, "python_path": ["/usr/bin/python"], @@ -71,7 +70,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -82,14 +81,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -116,7 +113,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -127,14 +124,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -159,7 +154,7 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, "python_path": ["/usr/bin/python"], @@ -172,7 +167,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-normal', 'identities_only': False, 'identity_file': null, @@ -183,14 +178,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -217,7 +210,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -228,14 +221,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -247,7 +238,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, @@ -258,14 +249,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -290,7 +279,7 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, "python_path": ["/usr/bin/python"], @@ -303,7 +292,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-newuser-normal-normal', 'identities_only': False, 'identity_file': null, @@ -314,14 +303,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -349,7 +336,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -360,14 +347,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -413,7 +398,7 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, 'python_path': ["/usr/bin/python"], @@ -424,7 +409,7 @@ }, { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, 'python_path': ["/usr/bin/python"], @@ -433,6 +418,6 @@ }, 'method': 'doas', }, - ] + ] tags: - stack_construction diff --git a/tests/ansible/integration/connection_loader/local_blemished.yml b/tests/ansible/integration/connection_loader/local_blemished.yml index 85224dbf..1452abd2 100644 --- a/tests/ansible/integration/connection_loader/local_blemished.yml +++ b/tests/ansible/integration/connection_loader/local_blemished.yml @@ -2,7 +2,6 @@ - name: integration/connection_loader/local_blemished.yml hosts: test-targets - any_errors_fatal: true tasks: - determine_strategy: diff --git a/tests/ansible/integration/connection_loader/paramiko_unblemished.yml b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml index 1bc5e2f6..74b3c743 100644 --- a/tests/ansible/integration/connection_loader/paramiko_unblemished.yml +++ b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml @@ -3,7 +3,6 @@ - name: integration/connection_loader/paramiko_unblemished.yml hosts: test-targets - any_errors_fatal: true tasks: - debug: msg: "skipped for now" diff --git a/tests/ansible/integration/connection_loader/ssh_blemished.yml b/tests/ansible/integration/connection_loader/ssh_blemished.yml index 58d35cee..c019f208 100644 --- a/tests/ansible/integration/connection_loader/ssh_blemished.yml +++ b/tests/ansible/integration/connection_loader/ssh_blemished.yml @@ -2,7 +2,6 @@ - name: integration/connection_loader__ssh_blemished.yml hosts: test-targets - any_errors_fatal: true tasks: - determine_strategy: diff --git a/tests/ansible/integration/context_service/disconnect_cleanup.yml b/tests/ansible/integration/context_service/disconnect_cleanup.yml index 48304120..2dd10b2c 100644 --- a/tests/ansible/integration/context_service/disconnect_cleanup.yml +++ b/tests/ansible/integration/context_service/disconnect_cleanup.yml @@ -3,7 +3,6 @@ - name: integration/context_service/disconnect_cleanup.yml hosts: test-targets[0] - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/context_service/lru_one_target.yml b/tests/ansible/integration/context_service/lru_one_target.yml index 2d6dc6cf..570581e3 100644 --- a/tests/ansible/integration/context_service/lru_one_target.yml +++ b/tests/ansible/integration/context_service/lru_one_target.yml @@ -2,7 +2,6 @@ - name: integration/context_service/lru_one_target.yml hosts: test-targets - any_errors_fatal: true vars: max_interps: "{{lookup('env', 'MITOGEN_MAX_INTERPRETERS')}}" ubound: "{{max_interps|int + 1}}" diff --git a/tests/ansible/integration/context_service/reconnection.yml b/tests/ansible/integration/context_service/reconnection.yml index 2d7dbacf..5ab2309f 100644 --- a/tests/ansible/integration/context_service/reconnection.yml +++ b/tests/ansible/integration/context_service/reconnection.yml @@ -3,7 +3,6 @@ - name: integration/context_service/reconnection.yml hosts: test-targets - any_errors_fatal: true tasks: - mitogen_shutdown_all: @@ -14,7 +13,8 @@ custom_python_detect_environment: register: old_become_env - - become: true + - name: Kill ssh process + become: true shell: | bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown" diff --git a/tests/ansible/integration/context_service/remote_name.yml b/tests/ansible/integration/context_service/remote_name.yml index cce209e4..79fb8dea 100644 --- a/tests/ansible/integration/context_service/remote_name.yml +++ b/tests/ansible/integration/context_service/remote_name.yml @@ -2,7 +2,6 @@ - name: integration/context_service/remote_name.yml hosts: test-targets[0] - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen @@ -11,7 +10,8 @@ - meta: end_play when: ansible_system != 'Linux' - - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' + - name: Get cmdline + shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out - debug: var=out @@ -20,7 +20,8 @@ - out.stdout is match('.*python([0-9.]+)?\(mitogen:[a-z]+@[^:]+:[0-9]+\)') fail_msg: out={{out}} - - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' + - name: Get cmdline, with mitogen_mask_remote_name + shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out vars: mitogen_mask_remote_name: true diff --git a/tests/ansible/integration/glibc_caches/resolv_conf.yml b/tests/ansible/integration/glibc_caches/resolv_conf.yml index e83ce38e..1b22b66c 100644 --- a/tests/ansible/integration/glibc_caches/resolv_conf.yml +++ b/tests/ansible/integration/glibc_caches/resolv_conf.yml @@ -15,12 +15,14 @@ - ansible_facts.virtualization_type == "docker" - ansible_facts.python.version_info[:2] >= [2, 5] - - shell: cp /etc/resolv.conf /tmp/resolv.conf + - name: Backup resolv.conf + shell: cp /etc/resolv.conf /tmp/resolv.conf when: - ansible_facts.virtualization_type == "docker" - ansible_facts.python.version_info[:2] >= [2, 5] - - shell: echo > /etc/resolv.conf + - name: Truncate resolv.conf + shell: echo > /etc/resolv.conf when: - ansible_facts.virtualization_type == "docker" - ansible_facts.python.version_info[:2] >= [2, 5] @@ -33,7 +35,8 @@ - ansible_facts.virtualization_type == "docker" - ansible_facts.python.version_info[:2] >= [2, 5] - - shell: cat /tmp/resolv.conf > /etc/resolv.conf + - name: Restore resolv.conf + shell: cat /tmp/resolv.conf > /etc/resolv.conf when: - ansible_facts.virtualization_type == "docker" - ansible_facts.python.version_info[:2] >= [2, 5] diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 201ef8b4..ea828aa8 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -3,7 +3,6 @@ - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - name: can only run these tests on ansible >= 2.8.0 diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index ca35a402..38d10124 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -3,7 +3,6 @@ - name: integration/interpreter_discovery/complex_args.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - name: create temp file to source diff --git a/tests/ansible/integration/local/cwd_preserved.yml b/tests/ansible/integration/local/cwd_preserved.yml index 24a34ba8..70aec613 100644 --- a/tests/ansible/integration/local/cwd_preserved.yml +++ b/tests/ansible/integration/local/cwd_preserved.yml @@ -5,10 +5,10 @@ # https://github.com/ansible/ansible/issues/14489 - name: integration/local/cwd_preserved.yml - any_errors_fatal: true hosts: test-targets tasks: - - connection: local + - name: Get local cwd + connection: local command: pwd register: pwd diff --git a/tests/ansible/integration/local/env_preserved.yml b/tests/ansible/integration/local/env_preserved.yml index 64794756..e074a490 100644 --- a/tests/ansible/integration/local/env_preserved.yml +++ b/tests/ansible/integration/local/env_preserved.yml @@ -1,8 +1,8 @@ - # Execution environment should be that of WorkerProcess -- # https://github.com/dw/mitogen/issues/297 -- hosts: localhost +- name: integration/local/env_preserved.yml + hosts: localhost connection: local tasks: - shell: "env | grep EVIL_VARS_PLUGIN" diff --git a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml index edd65db7..40da94d0 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml @@ -3,7 +3,6 @@ - name: integration/module_utils/adjacent_to_playbook.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_module: diff --git a/tests/ansible/integration/module_utils/adjacent_to_role.yml b/tests/ansible/integration/module_utils/adjacent_to_role.yml index 4430c2d9..c4fb813f 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_role.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_role.yml @@ -3,7 +3,6 @@ - name: integration/module_utils/adjacent_to_playbook.yml hosts: test-targets - any_errors_fatal: true roles: - modrole tags: diff --git a/tests/ansible/integration/module_utils/from_config_path.yml b/tests/ansible/integration/module_utils/from_config_path.yml index 0fb0f3ea..f4d9186b 100644 --- a/tests/ansible/integration/module_utils/from_config_path.yml +++ b/tests/ansible/integration/module_utils/from_config_path.yml @@ -2,7 +2,6 @@ - name: integration/module_utils/from_config_path.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_module: diff --git a/tests/ansible/integration/module_utils/from_config_path_pkg.yml b/tests/ansible/integration/module_utils/from_config_path_pkg.yml index 3453cf29..3e81f10f 100644 --- a/tests/ansible/integration/module_utils/from_config_path_pkg.yml +++ b/tests/ansible/integration/module_utils/from_config_path_pkg.yml @@ -2,7 +2,6 @@ - name: integration/module_utils/from_config_path.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_pkg: diff --git a/tests/ansible/integration/module_utils/overrides_builtin.yml b/tests/ansible/integration/module_utils/overrides_builtin.yml index 86d88312..9d841e2e 100644 --- a/tests/ansible/integration/module_utils/overrides_builtin.yml +++ b/tests/ansible/integration/module_utils/overrides_builtin.yml @@ -1,7 +1,6 @@ - name: integration/module_utils/overrides_builtin.yml hosts: test-targets - any_errors_fatal: true roles: - overrides_modrole tags: diff --git a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py index 78f2d71d..faf8a9a5 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py +++ b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils import external3 diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index 688fc45a..4fc4e04c 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import json import ansible.module_utils.basic diff --git a/tests/ansible/integration/playbook_semantics/become_flags.yml b/tests/ansible/integration/playbook_semantics/become_flags.yml index ac762643..9c6c8d90 100644 --- a/tests/ansible/integration/playbook_semantics/become_flags.yml +++ b/tests/ansible/integration/playbook_semantics/become_flags.yml @@ -4,7 +4,6 @@ - name: integration/playbook_semantics/become_flags.yml hosts: test-targets - any_errors_fatal: true tasks: - name: "without -E" @@ -19,7 +18,6 @@ - become_flags - hosts: test-targets - any_errors_fatal: true become_flags: -E tasks: - name: "with -E" diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index 7536258f..c8d9e607 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -1,6 +1,5 @@ - name: integration/playbook_semantics/delegate_to.yml hosts: test-targets - any_errors_fatal: true tasks: # # delegate_to, no sudo diff --git a/tests/ansible/integration/playbook_semantics/environment.yml b/tests/ansible/integration/playbook_semantics/environment.yml index 01e8ce04..eea000bd 100644 --- a/tests/ansible/integration/playbook_semantics/environment.yml +++ b/tests/ansible/integration/playbook_semantics/environment.yml @@ -2,9 +2,9 @@ - name: integration/playbook_semantics/environment.yml hosts: test-targets - any_errors_fatal: true tasks: - - shell: echo $SOME_ENV + - name: Echo $SOME_ENV + shell: echo $SOME_ENV environment: SOME_ENV: 123 register: result diff --git a/tests/ansible/integration/playbook_semantics/with_items.yml b/tests/ansible/integration/playbook_semantics/with_items.yml index 12a92bba..4de29c4b 100644 --- a/tests/ansible/integration/playbook_semantics/with_items.yml +++ b/tests/ansible/integration/playbook_semantics/with_items.yml @@ -3,7 +3,6 @@ - name: integration/playbook_semantics/with_items.yml hosts: test-targets - any_errors_fatal: true tasks: # TODO: https://github.com/dw/mitogen/issues/692 diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index 6aeac609..b5d40b7d 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -1,5 +1,5 @@ - -- hosts: test-targets[0] +- name: integration/process/unix_socket_cleanup.yml + hosts: test-targets[0] tasks: - mitogen_action_script: script: | @@ -9,8 +9,12 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -c local -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -c local -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets args: chdir: ../.. register: out diff --git a/tests/ansible/integration/runner/_etc_environment_global.yml b/tests/ansible/integration/runner/_etc_environment_global.yml index 7b769ef4..2d883d06 100644 --- a/tests/ansible/integration/runner/_etc_environment_global.yml +++ b/tests/ansible/integration/runner/_etc_environment_global.yml @@ -1,18 +1,21 @@ # /etc/environment -- file: +- name: Remove /etc/environment + file: path: /etc/environment state: absent become: true -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV without + shell: echo $MAGIC_ETC_ENV register: echo - assert: that: echo.stdout == "" fail_msg: echo={{echo}} -- copy: +- name: Create /etc/environment + copy: dest: /etc/environment content: | MAGIC_ETC_ENV=555 @@ -23,14 +26,16 @@ #- mitogen_shutdown_all: #when: not is_mitogen -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV with + shell: echo $MAGIC_ETC_ENV register: echo - assert: that: echo.stdout == "555" fail_msg: echo={{echo}} -- file: +- name: Cleanup /etc/environment + file: path: /etc/environment state: absent become: true @@ -40,7 +45,8 @@ - mitogen_shutdown_all: when: not is_mitogen -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV without+shutdown + shell: echo $MAGIC_ETC_ENV register: echo - assert: diff --git a/tests/ansible/integration/runner/_etc_environment_user.yml b/tests/ansible/integration/runner/_etc_environment_user.yml index 9d9b831a..939ee8dc 100644 --- a/tests/ansible/integration/runner/_etc_environment_user.yml +++ b/tests/ansible/integration/runner/_etc_environment_user.yml @@ -1,6 +1,7 @@ # ~/.pam_environment -- file: +- name: Remove pam_environment + file: path: ~/.pam_environment state: absent @@ -11,7 +12,8 @@ that: echo.stdout == "" fail_msg: echo={{echo}} -- copy: +- name: Copy pam_environment + copy: dest: ~/.pam_environment content: | MAGIC_PAM_ENV=321 @@ -23,7 +25,8 @@ that: echo.stdout == "321" fail_msg: echo={{echo}} -- file: +- name: Cleanup pam_environment + file: path: ~/.pam_environment state: absent diff --git a/tests/ansible/integration/runner/atexit.yml b/tests/ansible/integration/runner/atexit.yml index 65673561..75509877 100644 --- a/tests/ansible/integration/runner/atexit.yml +++ b/tests/ansible/integration/runner/atexit.yml @@ -9,20 +9,16 @@ vars: path: /tmp/atexit-should-delete-this tasks: - - # - # Verify a run with a healthy atexit handler. Broken handlers cause an - # exception to be raised. - # - - - custom_python_run_script: + - name: Verify a run with a healthy atexit handler + custom_python_run_script: script: | import atexit, os, shutil path = '{{path}}' os.mkdir(path, int('777', 8)) atexit.register(shutil.rmtree, path) - - stat: + - name: Stat atexit file + stat: path: "{{path}}" register: out diff --git a/tests/ansible/integration/runner/builtin_command_module.yml b/tests/ansible/integration/runner/builtin_command_module.yml index b567bee6..06e058cd 100644 --- a/tests/ansible/integration/runner/builtin_command_module.yml +++ b/tests/ansible/integration/runner/builtin_command_module.yml @@ -1,10 +1,10 @@ - name: integration/runner/builtin_command_module.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - - command: hostname + - name: Run hostname + command: hostname with_sequence: start=1 end={{end|default(1)}} register: out diff --git a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml index d2d2281c..ebb7966e 100644 --- a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml +++ b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml @@ -1,7 +1,6 @@ # https://github.com/dw/mitogen/issues/291 - name: integration/runner/custom_bash_hashbang_argument.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_old_style_module: diff --git a/tests/ansible/integration/runner/custom_bash_old_style_module.yml b/tests/ansible/integration/runner/custom_bash_old_style_module.yml index 122b8fde..81ab5d07 100644 --- a/tests/ansible/integration/runner/custom_bash_old_style_module.yml +++ b/tests/ansible/integration/runner/custom_bash_old_style_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_bash_old_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_old_style_module: diff --git a/tests/ansible/integration/runner/custom_bash_want_json_module.yml b/tests/ansible/integration/runner/custom_bash_want_json_module.yml index b4242e9a..24d9a064 100644 --- a/tests/ansible/integration/runner/custom_bash_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_bash_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_bash_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_want_json_module: foo: true diff --git a/tests/ansible/integration/runner/custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml index 6dfeefc3..4c6e2468 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_json.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_binary_producing_json.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - block: diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml index 97fc92fe..c2fb7ccd 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_junk.yml @@ -24,7 +24,6 @@ - hosts: test-targets - any_errors_fatal: true tasks: - assert: that: diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml index 353cc58d..a666ad68 100644 --- a/tests/ansible/integration/runner/custom_binary_single_null.yml +++ b/tests/ansible/integration/runner/custom_binary_single_null.yml @@ -10,7 +10,6 @@ - custom_binary_single_null - hosts: test-targets - any_errors_fatal: true tasks: - assert: that: diff --git a/tests/ansible/integration/runner/custom_perl_json_args_module.yml b/tests/ansible/integration/runner/custom_perl_json_args_module.yml index 68230420..76ead985 100644 --- a/tests/ansible/integration/runner/custom_perl_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_perl_json_args_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_perl_json_args_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_perl_json_args_module: foo: true diff --git a/tests/ansible/integration/runner/custom_perl_want_json_module.yml b/tests/ansible/integration/runner/custom_perl_want_json_module.yml index 44c39e64..2e520329 100644 --- a/tests/ansible/integration/runner/custom_perl_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_perl_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_perl_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_perl_want_json_module: foo: true diff --git a/tests/ansible/integration/runner/custom_python_json_args_module.yml b/tests/ansible/integration/runner/custom_python_json_args_module.yml index 33a03ba0..d72ff0fa 100644 --- a/tests/ansible/integration/runner/custom_python_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_python_json_args_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_python_json_args_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_json_args_module: foo: true diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 1562e307..0c620dac 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -1,7 +1,6 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_new_style_missing_interpreter: foo: true diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index b22fdc14..e2384f81 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: # without Mitogen Ansible 2.10 hangs on this play - meta: end_play diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml index 4ddec8c5..f2a3eefd 100644 --- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml +++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml @@ -2,7 +2,6 @@ - name: integration/runner/custom_python_prehistoric_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_prehistoric_module: register: out diff --git a/tests/ansible/integration/runner/custom_python_want_json_module.yml b/tests/ansible/integration/runner/custom_python_want_json_module.yml index d704ee11..a9bbd5b2 100644 --- a/tests/ansible/integration/runner/custom_python_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_python_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_python_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_want_json_module: foo: true diff --git a/tests/ansible/integration/runner/custom_script_interpreter.yml b/tests/ansible/integration/runner/custom_script_interpreter.yml index 03ea6718..9849912e 100644 --- a/tests/ansible/integration/runner/custom_script_interpreter.yml +++ b/tests/ansible/integration/runner/custom_script_interpreter.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_script_interpreter.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_old_style_module: diff --git a/tests/ansible/integration/runner/environment_isolation.yml b/tests/ansible/integration/runner/environment_isolation.yml index 90110d01..b195a469 100644 --- a/tests/ansible/integration/runner/environment_isolation.yml +++ b/tests/ansible/integration/runner/environment_isolation.yml @@ -1,52 +1,38 @@ # issue #309: ensure process environment is restored after a module runs. - - name: integration/runner/environment_isolation.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - - # --- - # Verify custom env setting is cleared out. - # --- - - # Verify sane state first. - - custom_python_detect_environment: + - name: Verify custom env setting is cleared, control + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined fail_msg: out={{out}} - - - shell: echo 'hi' + - name: Verify custom env setting is cleared, with evil_key + shell: echo 'hi' environment: evil_key: evil - - # Verify environment was cleaned up. - - custom_python_detect_environment: + - name: Verify custom env setting is cleared, without evil_key + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined fail_msg: out={{out}} - - # --- - # Verify non-explicit module env mutations are cleared out. - # --- - - # Verify sane state first. - - custom_python_detect_environment: + - name: Verify non-explicit module env mutations are cleared, control + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined fail_msg: out={{out}} - - - custom_python_modify_environ: + - name: Verify non-explicit module env mutations are cleared, mutate evil_key + custom_python_modify_environ: key: evil_key val: evil - - # Verify environment was cleaned up. - - custom_python_detect_environment: + - name: Verify non-explicit module env mutations are cleared, without evil_key + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index 3a44c753..6ed3e9c2 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -4,7 +4,6 @@ - name: integration/runner/etc_environment.yml hosts: test-targets[0] - any_errors_fatal: true gather_facts: true tasks: - include: _etc_environment_user.yml diff --git a/tests/ansible/integration/runner/forking_active.yml b/tests/ansible/integration/runner/forking_active.yml index 6a0e322d..5198ee1c 100644 --- a/tests/ansible/integration/runner/forking_active.yml +++ b/tests/ansible/integration/runner/forking_active.yml @@ -1,6 +1,5 @@ - name: integration/runner/forking_active.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify mitogen_task_isolation=fork triggers forking. diff --git a/tests/ansible/integration/runner/forking_correct_parent.yml b/tests/ansible/integration/runner/forking_correct_parent.yml index 7fe54e7b..8f360bb7 100644 --- a/tests/ansible/integration/runner/forking_correct_parent.yml +++ b/tests/ansible/integration/runner/forking_correct_parent.yml @@ -1,7 +1,6 @@ - name: integration/runner/forking_correct_parent.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify mitogen_task_isolation=fork forks from "virginal fork parent", not diff --git a/tests/ansible/integration/runner/forking_inactive.yml b/tests/ansible/integration/runner/forking_inactive.yml index 37282f82..91355489 100644 --- a/tests/ansible/integration/runner/forking_inactive.yml +++ b/tests/ansible/integration/runner/forking_inactive.yml @@ -2,7 +2,6 @@ - name: integration/runner/forking_inactive.yml hosts: test-targets - any_errors_fatal: true tasks: - name: get process ID. diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index 676012d9..979cdf21 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -3,10 +3,13 @@ hosts: test-targets[0] connection: local tasks: - - connection: local + - name: Run missing_module + connection: local command: | ansible -vvv - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} test-targets -m missing_module args: diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index d697905a..d4aba5d2 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -3,10 +3,15 @@ - name: integration/ssh/timeouts.yml hosts: test-targets tasks: - - connection: local + - name: Cause Ansible connection timeout + connection: local + environment: + ANSIBLE_SSH_TIMEOUT: 10 command: | ansible -vvv - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} test-targets -m custom_python_detect_environment -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password @@ -16,7 +21,8 @@ ignore_errors: true when: is_mitogen - - assert: + - name: Verify connection timeout occurred + assert: that: - | '"changed": false' in out.stdout diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index d05ac288..6041d304 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -13,12 +13,16 @@ -o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}" tasks: - - name: ansible_ssh_user - # Remaining tests just use "ansible_user". + - name: ansible_ssh_user, ansible_ssh_pass shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_ssh_user=mitogen__has_sudo -e ansible_ssh_pass=has_sudo_password args: @@ -26,10 +30,16 @@ register: out when: is_mitogen - - shell: > + - name: ansible_ssh_user, wrong ansible_ssh_pass + shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_ssh_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password args: @@ -39,16 +49,22 @@ when: is_mitogen - assert: - that: out.rc == 4 # unreachable + that: + - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS fail_msg: out={{out}} when: is_mitogen - - name: ansible_ssh_pass + - name: ansible_user, ansible_ssh_pass shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_pass=has_sudo_password args: @@ -56,10 +72,16 @@ register: out when: is_mitogen - - shell: > + - name: ansible_user, wrong ansible_ssh_pass + shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password args: @@ -69,16 +91,22 @@ when: is_mitogen - assert: - that: out.rc == 4 # unreachable + that: + - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS fail_msg: out={{out}} when: is_mitogen - - name: ansible_password + - name: ansible_user, ansible_password shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_password=has_sudo_password args: @@ -86,10 +114,16 @@ register: out when: is_mitogen - - shell: > + - name: ansible_user, wrong ansible_password + shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_password=wrong_password args: @@ -99,21 +133,27 @@ when: is_mitogen - assert: - that: out.rc == 4 # unreachable + that: + - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS fail_msg: out={{out}} when: is_mitogen - - name: ansible_ssh_private_key_file + - name: setup ansible_ssh_private_key_file shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key args: chdir: ../.. - - name: ansible_ssh_private_key_file + - name: ansible_user, ansible_ssh_private_key_file shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo_pubkey -e ansible_ssh_private_key_file=../data/docker/mitogen__has_sudo_pubkey.key args: @@ -121,10 +161,16 @@ register: out when: is_mitogen - - shell: > + - name: ansible_user, wrong ansible_ssh_private_key_file + shell: > + ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_private_key_file=/dev/null args: @@ -134,6 +180,7 @@ when: is_mitogen - assert: - that: out.rc == 4 # unreachable + that: + - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS fail_msg: out={{out}} when: is_mitogen diff --git a/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml b/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml index b2ae7641..e4df9378 100644 --- a/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml +++ b/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml @@ -3,7 +3,6 @@ - name: integration/strategy/_mixed_mitogen_vanilla.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear run_once: true tasks: @@ -22,7 +21,6 @@ - name: integration/strategy/_mixed_mitogen_vanilla.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: @@ -40,7 +38,6 @@ - name: integration/strategy/_mixed_mitogen_vanilla.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear tasks: - custom_python_detect_environment: diff --git a/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml index e073d11a..d2bdfdb8 100644 --- a/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml @@ -3,7 +3,6 @@ - name: integration/strategy/_mixed_vanilla_mitogen.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: @@ -21,7 +20,6 @@ - name: integration/strategy/_mixed_vanilla_mitogen.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear tasks: - custom_python_detect_environment: @@ -39,7 +37,6 @@ - name: integration/strategy/_mixed_vanilla_mitogen.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index 0f462fea..d183564f 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -1,12 +1,13 @@ - name: integration/strategy/mixed_vanilla_mitogen.yml (linear->mitogen->linear) hosts: test-targets[0] - any_errors_fatal: true tasks: - connection: local command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} -vvv integration/strategy/_mixed_mitogen_vanilla.yml args: @@ -16,7 +17,9 @@ - connection: local command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} -vvv integration/strategy/_mixed_vanilla_mitogen.yml args: diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index 5303f7c7..47777ba8 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -2,7 +2,6 @@ - name: integration/stub_connections/kubectl.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index ec79cdac..abe354b6 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -2,7 +2,6 @@ - name: integration/stub_connections/lxc.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index cb03232f..5314f82c 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -2,7 +2,6 @@ - name: integration/stub_connections/lxd.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 5e545935..87764d12 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -2,7 +2,6 @@ - name: integration/stub_connections/mitogen_doas.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index 36f69814..6da0e5d6 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -2,7 +2,6 @@ - name: integration/stub_connections/mitogen_sudo.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index 44717504..75d4207b 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -13,7 +13,8 @@ - include: _end_play_if_not_sudo_linux.yml - - command: | + - name: Run stub-lxc-info.py + command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, -c setns diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index 841f7805..3f3bc291 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -13,7 +13,8 @@ - include: _end_play_if_not_sudo_linux.yml - - command: | + - name: Run ansible stub-lxc.py + command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, -c setns diff --git a/tests/ansible/integration/transport_config/become_method.yml b/tests/ansible/integration/transport_config/become_method.yml index a6eac9ef..e4efbabd 100644 --- a/tests/ansible/integration/transport_config/become_method.yml +++ b/tests/ansible/integration/transport_config/become_method.yml @@ -2,7 +2,7 @@ # No become-method set. -- name: integration/transport_config/become-method.yml +- name: integration/transport_config/become_method.yml hosts: tc-become-method-unset become: true tasks: diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml index 6287e708..317e0522 100644 --- a/tests/ansible/integration/transport_config/become_pass.yml +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -2,7 +2,7 @@ # No become-pass set, defaults to "root" -- name: integration/transport_config/become-pass.yml +- name: integration/transport_config/become_pass.yml hosts: tc-become-pass-unset become: true tasks: @@ -119,9 +119,6 @@ fail_msg: out={{out}} - -# ansible_become_pass & ansible_become_password set, password used to take precedence -# but it's possible since https://github.com/ansible/ansible/pull/69629/files#r428376864, now it doesn't - hosts: tc-become-pass-both become: true tasks: @@ -132,7 +129,9 @@ - out.result|length == 2 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "c.b.a" + # Ansible >= 2.10 builtin become plugins (e.g. sudo, su) give priority + # to ansible_become_pass over ansible_become_password. + - out.result[1].kwargs.password == "bpass" fail_msg: out={{out}} @@ -147,6 +146,6 @@ - out.result|length == 3 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "a.b.c" + - out.result[1].kwargs.password == "bpass" - out.result[2].method == "ssh" fail_msg: out={{out}} diff --git a/tests/ansible/integration/transport_config/become_user.yml b/tests/ansible/integration/transport_config/become_user.yml index 3c8397ad..1a8b9fc9 100644 --- a/tests/ansible/integration/transport_config/become_user.yml +++ b/tests/ansible/integration/transport_config/become_user.yml @@ -2,7 +2,7 @@ # No become-user set, defaults to "root" -- name: integration/transport_config/become-user.yml +- name: integration/transport_config/become_user.yml hosts: tc-become-user-unset become: true tasks: diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml index 6014ffae..340eae79 100644 --- a/tests/ansible/integration/transport_config/port.yml +++ b/tests/ansible/integration/transport_config/port.yml @@ -41,7 +41,7 @@ - out.result[0].kwargs.port == 4321 fail_msg: out={{out}} -- hosts: tc-port-explicit-unset +- hosts: tc-port-unset vars: {mitogen_via: tc-port-explicit-ssh} tasks: - include: ../_mitogen_only.yml @@ -50,9 +50,9 @@ that: - out.result|length == 2 - out.result[0].method == "ssh" - - out.result[1].kwargs.port == 4321 + - out.result[0].kwargs.port == 4321 - out.result[1].method == "ssh" - - out.result[0].kwargs.port == None + - out.result[1].kwargs.port == None fail_msg: out={{out}} # ansible_port= diff --git a/tests/ansible/integration/transport_config/remote_user.yml b/tests/ansible/integration/transport_config/remote_user.yml index b873fcbe..5ddc5aa3 100644 --- a/tests/ansible/integration/transport_config/remote_user.yml +++ b/tests/ansible/integration/transport_config/remote_user.yml @@ -60,7 +60,7 @@ left: out.result[0].kwargs.username right: "ansi-user" -- hosts: tc-remote-user-explicit-host +- hosts: tc-remote-user-explicit-user vars: {mitogen_via: tc-remote-user-unset} tasks: - include: ../_mitogen_only.yml diff --git a/tests/ansible/integration/transport_config/transport__smart.yml b/tests/ansible/integration/transport_config/transport__smart.yml index a4c1959f..f263d67e 100644 --- a/tests/ansible/integration/transport_config/transport__smart.yml +++ b/tests/ansible/integration/transport_config/transport__smart.yml @@ -4,7 +4,7 @@ # When no ansible_connection= is set, transport comes via ansible.cfg ("smart" # is parsed away to either paramiko or ssh). -- name: integration/transport_config/transport.yml +- name: integration/transport_config/transport__smart.yml hosts: tc-transport-smart tasks: - include: ../_mitogen_only.yml diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py index 5923f4b5..72264cf6 100644 --- a/tests/ansible/lib/action/assert_equal.py +++ b/tests/ansible/lib/action/assert_equal.py @@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase TEMPLATE_KWARGS = {} -_argspec = inspect.getargspec(ansible.template.Templar.template) +try: + # inspect.getfullargspec() Added: 3.0 + _argspec = inspect.getfullargspec(ansible.template.Templar.template) +except AttributeError: + # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11 + _argspec = inspect.getargspec(ansible.template.Templar.template) if 'bare_deprecated' in _argspec.args: TEMPLATE_KWARGS['bare_deprecated'] = False diff --git a/tests/ansible/lib/callback/profile_tasks.py b/tests/ansible/lib/callback/profile_tasks.py deleted file mode 100644 index 89d956ac..00000000 --- a/tests/ansible/lib/callback/profile_tasks.py +++ /dev/null @@ -1,82 +0,0 @@ -# profile_tasks.py: an Ansible plugin for timing tasks - -# Copyright (C) 2014 Jharrod LaFon -# https://github.com/jlafon/ansible-profile/ -# Included with permission - - -# The MIT License (MIT) -# -# Copyright (c) 2014 Jharrod LaFon -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from ansible.plugins.callback import CallbackBase -import time - - -class CallbackModule(CallbackBase): - """ - A plugin for timing tasks - """ - def __init__(self): - super(CallbackModule, self).__init__() - self.stats = {} - self.current = None - - def playbook_on_task_start(self, name, is_conditional): - """ - Logs the start of each task - """ - if self.current is not None: - # Record the running time of the last executed task - self.stats[self.current] = time.time() - self.stats[self.current] - - # Record the start time of the current task - self.current = name - self.stats[self.current] = time.time() - - called = False - - def playbook_on_stats(self, stats): - """ - Prints the timings - """ - if CallbackModule.called: - return - CallbackModule.called = True - - # Record the timing of the very last task - if self.current is not None: - self.stats[self.current] = time.time() - self.stats[self.current] - - # Sort the tasks by their running time - results = sorted(self.stats.items(), - key=lambda value: value[1], reverse=True) - - # Just keep the top 10 - results = results[:10] - - # Print the timings - for name, elapsed in results: - print("{0:-<70}{1:->9}".format( - '{0} '.format(name), - ' {0:.02f}s'.format(elapsed))) - diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 65f660a8..c7a222e7 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I return details about the Python # interpreter I run within. diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py index 507e53dd..ae1b78cb 100644 --- a/tests/ansible/lib/modules/custom_python_external_module.py +++ b/tests/ansible/lib/modules/custom_python_external_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I expect the quote from modules2/module_utils/joker.py. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py index 95bd0c7b..be9acb24 100644 --- a/tests/ansible/lib/modules/custom_python_external_pkg.py +++ b/tests/ansible/lib/modules/custom_python_external_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.externalpkg import extmod diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ec..61640579 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py index 1d342329..255e3729 100644 --- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py +++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I leak state from each invocation # into a class variable and a global variable. diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 9767f855..51b74526 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I modify the process environment and # don't clean up after myself. diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index f9c176c1..1e7270cd 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I should receive an encoding string. import sys diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index 7fe3fd1b..c5e264ae 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,13 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. +import json import os -try: - import json -except ImportError: - import simplejson as json print(json.dumps({ 'cwd': os.getcwd() diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 7ce15208..0cf9774d 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index 4a6243d0..d6a839ae 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I run the script provided in the # parameter. diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 6b3a356b..03f3b6aa 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # issue #590: I am an Ansible new-style Python module that tries to use # ansible.module_utils.distro. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index d9ea7113..f5e33862 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,13 +1,9 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. +import json import sys -try: - import json -except ImportError: - import simplejson as json - WANT_JSON = 1 diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 289e9662..1b80a48b 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 37ab655c..1f71e879 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- # (c) 2012, Michael DeHaan diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 875d520e..0e599476 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -1,30 +1,16 @@ - import_playbook: issue_109__target_has_old_ansible_installed.yml - tags: regression - import_playbook: issue_113__duplicate_module_imports.yml - tags: regression - import_playbook: issue_118__script_not_marked_exec.yml - tags: regression - import_playbook: issue_122__environment_difference.yml - tags: regression - import_playbook: issue_140__thread_pileup.yml - tags: regression - import_playbook: issue_152__local_action_wrong_interpreter.yml - tags: regression - import_playbook: issue_152__virtualenv_python_fails.yml - tags: regression - import_playbook: issue_154__module_state_leaks.yml - tags: regression - import_playbook: issue_177__copy_module_failing.yml - tags: regression - import_playbook: issue_332_ansiblemoduleerror_first_occurrence.yml - tags: regression - import_playbook: issue_558_unarchive_failed.yml - tags: regression - import_playbook: issue_590__sys_modules_crap.yml - tags: regression - import_playbook: issue_591__setuptools_cwd_crash.yml - tags: regression - import_playbook: issue_615__streaming_transfer.yml - tags: regression - import_playbook: issue_655__wait_for_connection_error.yml - tags: regression +- import_playbook: issue_776__load_plugins_called_twice.yml diff --git a/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg b/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg new file mode 100644 index 00000000..ae9cfa19 Binary files /dev/null and b/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg differ diff --git a/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg b/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg new file mode 100644 index 00000000..66f1a94b Binary files /dev/null and b/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg differ diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml index 1188b6ce..e6677e2d 100644 --- a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml +++ b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml @@ -3,15 +3,14 @@ - name: regression/issue_109__target_has_old_ansible_installed.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - meta: end_play when: - ansible_version.full is version('2.6', '<', strict=True) - # Copy the naughty 'ansible' into place. - - copy: + - name: Copy the naughty ansible into place + copy: dest: "{{ansible_user_dir}}/ansible.py" src: ansible.py @@ -21,15 +20,15 @@ - custom_python_detect_environment: register: env - # Verify interpreter config would actually trigger the bug. - - assert: + - name: Verify interpreter config would actually trigger the bug + assert: that: - env.cwd == ansible_user_dir - (not env.mitogen_loaded) or (env.python_path.count("") == 1) fail_msg: env={{env}} - # Run some new-style modules that 'from ansible.module_utils...' - - stat: + - name: Run some new-style from ansible.module_utils... modules + stat: path: / tags: - issue_109 diff --git a/tests/ansible/regression/issue_113__duplicate_module_imports.yml b/tests/ansible/regression/issue_113__duplicate_module_imports.yml index b479e025..4975a07c 100644 --- a/tests/ansible/regression/issue_113__duplicate_module_imports.yml +++ b/tests/ansible/regression/issue_113__duplicate_module_imports.yml @@ -2,7 +2,6 @@ # by exercisizng the uri package. - name: regression/issue_113__duplicate_module_imports.yml - any_errors_fatal: true hosts: test-targets tasks: diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index e6ea83ce..1f614634 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -4,7 +4,6 @@ - name: regression/issue_140__thread_pileup.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Create file tree @@ -18,7 +17,8 @@ - name: Delete remote file tree file: path=/tmp/filetree.out state=absent - - file: + - name: Recreate file tree + file: state: directory path: /tmp/filetree.out diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index 4da6f544..fe7f5f42 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -6,10 +6,10 @@ - name: regression/issue_152__local_action_wrong_interpreter.yml hosts: test-targets connection: local - any_errors_fatal: true tasks: - - copy: + - name: Create /tmp/issue_152_interpreter.sh + copy: dest: /tmp/issue_152_interpreter.sh mode: u+x content: | @@ -27,7 +27,8 @@ - out.env.CUSTOM_INTERPRETER == "1" fail_msg: out={{out}} - - file: + - name: Cleanup /tmp/issue_152_interpreter.sh + file: path: /tmp/issue_152_interpreter.sh state: absent tags: diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index 11f94edd..f4c47aba 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -1,5 +1,4 @@ - name: regression/issue_152__virtualenv_python_fails.yml - any_errors_fatal: true gather_facts: true hosts: test-targets tasks: @@ -8,11 +7,12 @@ # Can't use pip module because it can't create virtualenvs, must call it # directly. - - shell: virtualenv /tmp/issue_152_virtualenv + - name: Create /tmp/issue_152_virtualenv environment: https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" PATH: "{{ lookup('env', 'PATH') }}" + shell: virtualenv /tmp/issue_152_virtualenv when: - lout.python.version.full is version('2.7', '>=', strict=True) @@ -30,7 +30,8 @@ when: - lout.python.version.full is version('2.7', '>=', strict=True) - - file: + - name: Cleanup /tmp/issue_152_virtualenv + file: path: /tmp/issue_152_virtualenv state: absent when: diff --git a/tests/ansible/regression/issue_154__module_state_leaks.yml b/tests/ansible/regression/issue_154__module_state_leaks.yml index 59b970d9..3a7ee309 100644 --- a/tests/ansible/regression/issue_154__module_state_leaks.yml +++ b/tests/ansible/regression/issue_154__module_state_leaks.yml @@ -2,7 +2,6 @@ # must be reinitialized or cleared out somehow on each invocation. - name: regression/issue_154__module_state_leaks.yml - any_errors_fatal: true hosts: test-targets tasks: diff --git a/tests/ansible/regression/issue_177__copy_module_failing.yml b/tests/ansible/regression/issue_177__copy_module_failing.yml index 7da39962..c8dfc635 100644 --- a/tests/ansible/regression/issue_177__copy_module_failing.yml +++ b/tests/ansible/regression/issue_177__copy_module_failing.yml @@ -1,9 +1,9 @@ - name: regression/issue_177__copy_module_failing.yml - any_errors_fatal: true hosts: test-targets tasks: - - copy: + - name: Copy files + copy: src: /etc/{{item}} dest: /tmp/{{item}} mode: 0644 @@ -11,7 +11,8 @@ - passwd - hosts - - file: + - name: Cleanup files + file: path: /tmp/{{item}} state: absent with_items: diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml index 2433e566..be069b1e 100644 --- a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -4,7 +4,12 @@ - name: regression/issue_332_ansiblemoduleerror_first_occurrence.yml hosts: test-targets tasks: - - file: path=/usr/bin/does-not-exist mode='a-s' state=file follow=yes + - name: Attempt to modify /usr/bin/does-not-exist + file: + path: /usr/bin/does-not-exist + mode: a-s + state: file + follow: true ignore_errors: true register: out diff --git a/tests/ansible/regression/issue_558_unarchive_failed.yml b/tests/ansible/regression/issue_558_unarchive_failed.yml index 23e9caad..829101cb 100644 --- a/tests/ansible/regression/issue_558_unarchive_failed.yml +++ b/tests/ansible/regression/issue_558_unarchive_failed.yml @@ -4,9 +4,16 @@ - name: regression/issue_558_unarchive_failed.yml hosts: test-targets tasks: - - file: state=absent path=/tmp/foo - - file: state=directory path=/tmp/foo - - unarchive: + - name: Cleanup /tmp/foo + file: + path: /tmp/foo + state: absent + - name: Create /tmp/foo + file: + path: /tmp/foo + state: directory + - name: Unarchive unarchive_test.tar + unarchive: src: "{{git_basedir}}/tests/data/unarchive_test.tar" dest: /tmp/foo # garbage doesn't work with BSD tar diff --git a/tests/ansible/regression/issue_590__sys_modules_crap.yml b/tests/ansible/regression/issue_590__sys_modules_crap.yml index 81d9b4b2..476701e3 100644 --- a/tests/ansible/regression/issue_590__sys_modules_crap.yml +++ b/tests/ansible/regression/issue_590__sys_modules_crap.yml @@ -1,5 +1,5 @@ - -- hosts: test-targets +- name: regression/issue_590__sys_modules_crap.yml + hosts: test-targets tasks: - meta: end_play when: diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index 02676cb5..80680063 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -3,7 +3,8 @@ # to call getcwd() before AnsibleModule has had a chance to clean up the # process environment. -- hosts: test-targets +- name: regression/issue_591__setuptools_cwd_crash.yml + hosts: test-targets tasks: - meta: end_play when: not is_mitogen diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml index 3e174bed..8ec28499 100644 --- a/tests/ansible/regression/issue_615__streaming_transfer.yml +++ b/tests/ansible/regression/issue_615__streaming_transfer.yml @@ -1,7 +1,7 @@ # issue #615: 'fetch' with become: was internally using slurp. -- hosts: test-targets - any_errors_fatal: True +- name: regression/issue_615_streaming_transfer.yml + hosts: test-targets gather_facts: no # Without Mitogen this causes Ansible to use the slurp module, which is *slow* become: true @@ -9,21 +9,26 @@ mitogen_ssh_compression: false tasks: - block: - - shell: | + - name: Create /tmp/512mb.zero + shell: | dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; chmod go= /tmp/512mb.zero - - fetch: + - name: Fetch /tmp/512mb.zero + fetch: src: /tmp/512mb.zero dest: /tmp/fetch-out - - file: + - name: Cleanup /tmp/512mb.zero + file: path: /tmp/512mb.zero state: absent - - file: + - name: Cleanup fetched file + file: path: /tmp/fetch-out state: absent + become: false delegate_to: localhost run_once: true when: diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index cf0ff359..fbdb9f2b 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -4,9 +4,8 @@ # since things are ran on localhost; Azure DevOps loses connection and fails # TODO: do we want to install docker a different way to be able to do this for other tests too --- -# this should only run on our Mac hosts -- hosts: target - any_errors_fatal: True +- name: regression/issue_655_wait_for_connection_error.yml + hosts: localhost gather_facts: yes become: no tasks: @@ -84,5 +83,7 @@ loop: - cmd: podman stop testMitogen - cmd: podman machine stop + when: + - ansible_facts.pkg_mgr in ['homebrew'] tags: - issue_655 diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml new file mode 100755 index 00000000..ad5bbd69 --- /dev/null +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -0,0 +1,70 @@ +# https://github.com/mitogen-hq/mitogen/issues/776 +--- +- name: regression/issue_776__load_plugins_called_twice.yml + hosts: test-targets + become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" + gather_facts: yes + tags: + - issue_776 + vars: + ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" + package: rsync # Chosen to exist in all tested distros/package managers + tasks: + - name: Switch to centos-stream + command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos + when: + - ansible_facts.pkg_mgr in ["dnf"] + + - name: Switch to archive.debian.org + # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html + copy: + content: | + deb http://archive.debian.org/debian stretch main contrib non-free + dest: /etc/apt/sources.list + mode: u=rw,go=r + when: + - ansible_facts.distribution == "Debian" + - ansible_facts.distribution_major_version == "9" + + - name: Add signing keys + copy: + src: "{{ item.src }}" + dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}" + mode: u=rw,go=r + loop: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + when: + # Ideally this would check for Debian 11, but distribution_major_version + # is unpopulated sometimes. + - ansible_facts.distribution == "Debian" + + - name: Update package index + apt: + update_cache: true + when: + - ansible_facts.pkg_mgr in ["apt"] + + - name: Test package module 1st call + package: + name: "{{ package }}" + state: present + + - name: Test package module 2nd call + package: + name: "{{ package }}" + state: present + + - name: Test dnf module 2nd call + dnf: + name: "{{ package }}" + state: present + when: + - ansible_facts.pkg_mgr == 'dnf' + + - name: Test dnf module 2nd call + dnf: + name: "{{ package }}" + state: present + when: + - ansible_facts.pkg_mgr == 'dnf' diff --git a/tests/ansible/run_ansible_playbook.py b/tests/ansible/run_ansible_playbook.py index b2b619d2..04c0c9db 100755 --- a/tests/ansible/run_ansible_playbook.py +++ b/tests/ansible/run_ansible_playbook.py @@ -40,15 +40,6 @@ extra = { 'git_basedir': GIT_BASEDIR, } -if '-i' in sys.argv: - extra['MITOGEN_INVENTORY_FILE'] = ( - os.path.abspath(sys.argv[1 + sys.argv.index('-i')]) - ) -else: - extra['MITOGEN_INVENTORY_FILE'] = ( - os.path.join(GIT_BASEDIR, 'tests/ansible/hosts') - ) - if 'ANSIBLE_ARGV' in os.environ: args = eval(os.environ['ANSIBLE_ARGV']) else: diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml index 9077158f..450e4fb0 100644 --- a/tests/ansible/setup/report.yml +++ b/tests/ansible/setup/report.yml @@ -17,3 +17,4 @@ - debug: {var: ansible_run_tags} - debug: {var: ansible_skip_tags} - debug: {var: ansible_version.full} + - debug: {var: is_mitogen} diff --git a/tests/ansible/soak/_file_service_loop.yml b/tests/ansible/soak/_file_service_loop.yml index 96111b3c..474abd07 100644 --- a/tests/ansible/soak/_file_service_loop.yml +++ b/tests/ansible/soak/_file_service_loop.yml @@ -1,6 +1,8 @@ - - file: + - name: Delete existing /tmp/foo- + file: path: /tmp/foo-{{inventory_hostname}} state: absent - - copy: + - name: Create /tmp/foo- + copy: dest: /tmp/foo-{{inventory_hostname}} content: "{{content}}" diff --git a/tests/ansible/soak/file_service.yml b/tests/ansible/soak/file_service.yml index 65b10b2d..90462cbf 100644 --- a/tests/ansible/soak/file_service.yml +++ b/tests/ansible/soak/file_service.yml @@ -1,4 +1,5 @@ -- hosts: all +- name: soak/file_service.yml + hosts: all tasks: - set_fact: content: "{% for x in range(126977) %}x{% endfor %}" diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 0051a701..b968071b 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -205,7 +205,7 @@ class LinuxPolicyTest(testlib.TestCase): proc.wait() his_cpu = self._get_cpus(tf.name) - self.assertNotEquals(my_cpu, his_cpu) + self.assertNotEqual(my_cpu, his_cpu) self.policy._clear() finally: tf.close() diff --git a/tests/ansible/tests/connection_test.py b/tests/ansible/tests/connection_test.py index 8a831a5e..0ade6994 100644 --- a/tests/ansible/tests/connection_test.py +++ b/tests/ansible/tests/connection_test.py @@ -41,7 +41,7 @@ class ConnectionMixin(MuxProcessMixin): conn = self.klass(play_context, new_stdin=False) # conn functions don't fetch ActionModuleMixin objs from _get_task_vars() # through the usual walk-the-stack approach so we'll not run interpreter discovery here - conn._action = mock.MagicMock(_possible_python_interpreter='/usr/bin/python') + conn._action = mock.MagicMock(_possible_python_interpreter=testlib.base_executable()) conn.on_action_run( task_vars={}, delegate_to_hostname=None, diff --git a/tests/ansible/tests/env_file_watcher_test.py b/tests/ansible/tests/env_file_watcher_test.py index 6ca1b7e0..62d437cd 100644 --- a/tests/ansible/tests/env_file_watcher_test.py +++ b/tests/ansible/tests/env_file_watcher_test.py @@ -42,7 +42,7 @@ class WatcherTest(testlib.TestCase): self.tf.seek(0) self.tf.truncate(0) watcher.check() - self.assertTrue(b('SOMEKEY') not in environb) + self.assertNotIn(b('SOMEKEY'), environb) def test_key_added(self): watcher = klass(self.tf.name) diff --git a/tests/call_error_test.py b/tests/call_error_test.py index 1d098bab..9c1174b8 100644 --- a/tests/call_error_test.py +++ b/tests/call_error_test.py @@ -13,18 +13,18 @@ class ConstructorTest(testlib.TestCase): def test_string_noargs(self): e = self.klass('%s%s') self.assertEqual(e.args[0], '%s%s') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_string_args(self): e = self.klass('%s%s', 1, 1) self.assertEqual(e.args[0], '11') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_from_exc(self): ve = plain_old_module.MyError('eek') e = self.klass(ve) self.assertEqual(e.args[0], 'plain_old_module.MyError: eek') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_form_base_exc(self): ve = SystemExit('eek') @@ -33,7 +33,7 @@ class ConstructorTest(testlib.TestCase): self.assertEqual(e.args[0], # varies across 2/3. '%s.%s: eek' % (cls.__module__, cls.__name__)) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_from_exc_tb(self): try: @@ -43,18 +43,18 @@ class ConstructorTest(testlib.TestCase): e = self.klass(ve) self.assertTrue(e.args[0].startswith('plain_old_module.MyError: eek')) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) - self.assertTrue('test_from_exc_tb' in e.args[0]) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) + self.assertIn('test_from_exc_tb', e.args[0]) def test_bytestring_conversion(self): e = self.klass(mitogen.core.b('bytes')) self.assertEqual(u'bytes', e.args[0]) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_reduce(self): e = self.klass('eek') func, (arg,) = e.__reduce__() - self.assertTrue(func is mitogen.core._unpickle_call_error) + self.assertIs(func, mitogen.core._unpickle_call_error) self.assertEqual(arg, e.args[0]) @@ -105,4 +105,4 @@ class PickleTest(testlib.TestCase): e2 = pickle.loads(pickle.dumps(e)) self.assertTrue(e2.args[0].startswith('plain_old_module.MyError: eek')) - self.assertTrue('test_from_exc_tb' in e2.args[0]) + self.assertIn('test_from_exc_tb', e2.args[0]) diff --git a/tests/connection_test.py b/tests/connection_test.py index 5b54e090..5c3e678d 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -39,7 +39,7 @@ class ConnectionTest(testlib.RouterMixin, testlib.TestCase): th.join() exc, = result - self.assertTrue(isinstance(exc, mitogen.parent.CancelledError)) + self.assertIsInstance(exc, mitogen.parent.CancelledError) self.assertEqual(mitogen.parent.BROKER_SHUTDOWN_MSG, exc.args[0]) diff --git a/tests/create_child_test.py b/tests/create_child_test.py index d9bbd1a1..acf3ea66 100644 --- a/tests/create_child_test.py +++ b/tests/create_child_test.py @@ -101,6 +101,7 @@ class StdinSockMixin(object): self.assertTrue(flags & os.O_RDWR) self.assertTrue(info['buf'], 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class StdoutSockMixin(object): @@ -115,6 +116,7 @@ class StdoutSockMixin(object): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class CreateChildTest(StdinSockMixin, StdoutSockMixin, testlib.TestCase): @@ -126,6 +128,7 @@ class CreateChildTest(StdinSockMixin, StdoutSockMixin, testlib.TestCase): self.assertEqual(st.st_dev, info['st_dev']) self.assertEqual(st.st_mode, info['st_mode']) self.assertEqual(st.st_ino, info['st_ino']) + close_proc(proc) class CreateChildMergedTest(StdinSockMixin, StdoutSockMixin, @@ -144,6 +147,7 @@ class CreateChildMergedTest(StdinSockMixin, StdoutSockMixin, self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class CreateChildStderrPipeTest(StdinSockMixin, StdoutSockMixin, @@ -164,6 +168,7 @@ class CreateChildStderrPipeTest(StdinSockMixin, StdoutSockMixin, self.assertFalse(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_WRONLY) + close_proc(proc) class TtyCreateChildTest(testlib.TestCase): @@ -191,9 +196,9 @@ class TtyCreateChildTest(testlib.TestCase): self.assertEqual(proc.pid, waited_pid) self.assertEqual(0, status) self.assertEqual(mitogen.core.b(''), tf.read()) - proc.stdout.close() finally: tf.close() + close_proc(proc) def test_stdin(self): proc, info, _ = run_fd_check(self.func, 0, 'read', @@ -202,14 +207,14 @@ class TtyCreateChildTest(testlib.TestCase): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdin.fileno())) flags = fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) self.assertTrue(info['flags'] & os.O_RDWR) self.assertTrue(info['buf'], 'TEST') + close_proc(proc) def test_stdout(self): proc, info, buf = run_fd_check(self.func, 1, 'write', @@ -219,8 +224,7 @@ class TtyCreateChildTest(testlib.TestCase): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdout.fileno())) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) @@ -229,6 +233,7 @@ class TtyCreateChildTest(testlib.TestCase): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) def test_stderr(self): # proc.stderr is None in the parent since there is no separate stderr @@ -240,8 +245,7 @@ class TtyCreateChildTest(testlib.TestCase): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdout.fileno())) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) @@ -250,6 +254,7 @@ class TtyCreateChildTest(testlib.TestCase): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) def test_dev_tty_open_succeeds(self): # In the early days of UNIX, a process that lacked a controlling TTY @@ -274,6 +279,7 @@ class TtyCreateChildTest(testlib.TestCase): proc.stdout.close() finally: tf.close() + close_proc(proc) class StderrDiagTtyMixin(object): @@ -286,8 +292,7 @@ class StderrDiagTtyMixin(object): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stderr.fileno())) flags = fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) @@ -296,6 +301,7 @@ class StderrDiagTtyMixin(object): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) class HybridTtyCreateChildTest(StdinSockMixin, StdoutSockMixin, @@ -321,6 +327,7 @@ if 0: self.assertTrue(buf, 'TEST') self.assertFalse(info['flags'] & os.O_WRONLY) self.assertFalse(info['flags'] & os.O_RDWR) + close_proc(proc) def test_stdout(self): proc, info, buf = run_fd_check(self.func, 1, 'write', @@ -334,3 +341,4 @@ if 0: self.assertFalse(flags & os.O_RDWR) self.assertTrue(info['flags'] & os.O_WRONLY) self.assertTrue(buf, 'TEST') + close_proc(proc) diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json new file mode 100644 index 00000000..4bf0bd39 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py2x.json @@ -0,0 +1,186 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.utils", + "django.utils._os", + "django.utils.deprecation", + "django.utils.encoding", + "django.utils.functional", + "django.utils.inspect", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.six", + "django.utils.version" + ], + "django.db.models": [ + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.checks", + "django.core.checks.caches", + "django.core.checks.compatibility", + "django.core.checks.compatibility.django_1_10", + "django.core.checks.compatibility.django_1_8_0", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.urls", + "django.core.checks.utils", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.base", + "django.db.models.functions.datetime", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.decorators", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.html", + "django.utils.html_parser", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.safestring", + "django.utils.six", + "django.utils.text", + "django.utils.timezone", + "django.utils.translation", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json new file mode 100644 index 00000000..06e94e62 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-legacy.json @@ -0,0 +1,209 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json new file mode 100644 index 00000000..dcbcc785 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -0,0 +1,208 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" +] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} + \ No newline at end of file diff --git a/tests/data/minimize_samples/hashbang.py b/tests/data/minimize_samples/hashbang.py index 0245c48f..1d358671 100644 --- a/tests/data/minimize_samples/hashbang.py +++ b/tests/data/minimize_samples/hashbang.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 # comment diff --git a/tests/data/minimize_samples/hashbang_min.py b/tests/data/minimize_samples/hashbang_min.py index 5a74f481..19836fa5 100644 --- a/tests/data/minimize_samples/hashbang_min.py +++ b/tests/data/minimize_samples/hashbang_min.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 diff --git a/tests/doas_test.py b/tests/doas_test.py index 4e847942..b0035540 100644 --- a/tests/doas_test.py +++ b/tests/doas_test.py @@ -37,7 +37,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.doas(via=ssh) # ) -# self.assertTrue(mitogen.doas.password_required_msg in str(e)) +# self.assertIn(mitogen.doas.password_required_msg, str(e)) # def test_password_incorrect(self): # ssh = self.docker_ssh( @@ -47,7 +47,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.doas(via=ssh, password='x') # ) -# self.assertTrue(mitogen.doas.password_incorrect_msg in str(e)) +# self.assertIn(mitogen.doas.password_incorrect_msg, str(e)) # def test_password_okay(self): # ssh = self.docker_ssh( diff --git a/tests/error_test.py b/tests/error_test.py index a34d801e..3138d7cd 100644 --- a/tests/error_test.py +++ b/tests/error_test.py @@ -8,19 +8,19 @@ class ConstructorTest(testlib.TestCase): def test_literal_no_format(self): e = self.klass('error') self.assertEqual(e.args[0], 'error') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_literal_format_chars_present(self): e = self.klass('error%s') self.assertEqual(e.args[0], 'error%s') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_format(self): e = self.klass('error%s', 123) self.assertEqual(e.args[0], 'error123') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_bytes_to_unicode(self): e = self.klass(mitogen.core.b('error')) self.assertEqual(e.args[0], 'error') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) diff --git a/tests/file_service_test.py b/tests/file_service_test.py index 2a2008b3..b400ab3e 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -40,7 +40,7 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): pool.stop() expect = service.unregistered_msg % ('/etc/shadow',) - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) if sys.platform == 'darwin': ROOT_GROUP = 'wheel' @@ -48,13 +48,13 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): ROOT_GROUP = 'root' def _validate_response(self, resp): - self.assertTrue(isinstance(resp, dict)) + self.assertIsInstance(resp, dict) self.assertEqual('root', resp['owner']) self.assertEqual(self.ROOT_GROUP, resp['group']) - self.assertTrue(isinstance(resp['mode'], int)) - self.assertTrue(isinstance(resp['mtime'], float)) - self.assertTrue(isinstance(resp['atime'], float)) - self.assertTrue(isinstance(resp['size'], int)) + self.assertIsInstance(resp['mode'], int) + self.assertIsInstance(resp['mtime'], float) + self.assertIsInstance(resp['atime'], float) + self.assertIsInstance(resp['size'], int) def test_path_authorized(self): recv = mitogen.core.Receiver(self.router) @@ -117,7 +117,7 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): pool.stop() expect = service.unregistered_msg % (path,) - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) def test_prefix_authorized_abspath_good(self): l1 = self.router.local() @@ -144,4 +144,4 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): pool.stop() expect = service.unregistered_msg % (path,) - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) diff --git a/tests/fork_test.py b/tests/fork_test.py index 86e60cfc..35acfe13 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -1,5 +1,6 @@ import os import random +import subprocess import sys import unittest @@ -26,15 +27,28 @@ import plain_old_module def _find_ssl_linux(): - s = testlib.subprocess__check_output(['ldd', _ssl.__file__]) - for line in s.decode().splitlines(): + proc = subprocess.Popen( + ['ldd', _ssl.__file__], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + b_stdout, b_stderr = proc.communicate() + assert proc.returncode == 0 + assert b_stderr.decode() == '' + for line in b_stdout.decode().splitlines(): bits = line.split() if bits[0].startswith('libssl'): return bits[2] + def _find_ssl_darwin(): - s = testlib.subprocess__check_output(['otool', '-l', _ssl.__file__]) - for line in s.decode().splitlines(): + proc = subprocess.Popen( + ['otool', '-l', _ssl.__file__], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + b_stdout, b_stderr = proc.communicate() + assert proc.returncode == 0 + assert b_stderr.decode() == '' + for line in b_stdout.decode().splitlines(): bits = line.split() if bits[0] == 'name' and 'libssl' in bits[1]: return bits[1] diff --git a/tests/latch_test.py b/tests/latch_test.py index e7e2768d..1fac8219 100644 --- a/tests/latch_test.py +++ b/tests/latch_test.py @@ -198,7 +198,7 @@ class ThreadedCloseTest(testlib.TestCase): self.join() self.assertEqual(self.results, [None]) for exc in self.excs: - self.assertTrue(isinstance(exc, mitogen.core.LatchError)) + self.assertIsInstance(exc, mitogen.core.LatchError) def test_five_threads(self): latch = self.klass() @@ -208,4 +208,4 @@ class ThreadedCloseTest(testlib.TestCase): self.join() self.assertEqual(self.results, [None]*5) for exc in self.excs: - self.assertTrue(isinstance(exc, mitogen.core.LatchError)) + self.assertIsInstance(exc, mitogen.core.LatchError) diff --git a/tests/log_handler_test.py b/tests/log_handler_test.py index c270c20a..43b986cb 100644 --- a/tests/log_handler_test.py +++ b/tests/log_handler_test.py @@ -65,8 +65,8 @@ class StartupTest(testlib.RouterMixin, testlib.TestCase): c1.shutdown(wait=True) logs = log.stop() - self.assertTrue('Python version is' in logs) - self.assertTrue('Parent is context 0 (master)' in logs) + self.assertIn('Python version is', logs) + self.assertIn('Parent is context 0 (master)', logs) def test_earliest_messages_logged_via(self): c1 = self.router.local(name='c1') @@ -80,10 +80,10 @@ class StartupTest(testlib.RouterMixin, testlib.TestCase): c2.shutdown(wait=True) logs = log.stop() - self.assertTrue('Python version is' in logs) + self.assertIn('Python version is', logs) expect = 'Parent is context %s (%s)' % (c1.context_id, 'parent') - self.assertTrue(expect in logs) + self.assertIn(expect, logs) StartupTest = unittest.skipIf( condition=sys.version_info < (2, 7) or sys.version_info >= (3, 6), diff --git a/tests/lxc_test.py b/tests/lxc_test.py index 81d27318..b9ebfa53 100644 --- a/tests/lxc_test.py +++ b/tests/lxc_test.py @@ -26,7 +26,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) self.assertEqual(argv[0], self.lxc_attach_path) - self.assertTrue('--clear-env' in argv) + self.assertIn('--clear-env', argv) self.assertTrue(has_subseq(argv, ['--name', 'container_name'])) def test_eof(self): diff --git a/tests/message_test.py b/tests/message_test.py index b5f5cf08..2d2299d1 100644 --- a/tests/message_test.py +++ b/tests/message_test.py @@ -49,12 +49,12 @@ class ConstructorTest(testlib.TestCase): def test_data_default(self): m = self.klass() self.assertEqual(m.data, b('')) - self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + self.assertIsInstance(m.data, mitogen.core.BytesType) def test_data_explicit(self): m = self.klass(data=b('asdf')) self.assertEqual(m.data, b('asdf')) - self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + self.assertIsInstance(m.data, mitogen.core.BytesType) def test_data_hates_unicode(self): self.assertRaises(Exception, @@ -185,9 +185,9 @@ class PickledTest(testlib.TestCase): d = {1: 2, u'a': 3, b('b'): 4, 'c': {}} roundtrip = self.roundtrip(d) self.assertEqual(d, roundtrip) - self.assertTrue(isinstance(roundtrip, dict)) + self.assertIsInstance(roundtrip, dict) for k in d: - self.assertTrue(isinstance(roundtrip[k], type(d[k]))) + self.assertIsInstance(roundtrip[k], type(d[k])) def test_int(self): self.assertEqual(123, self.klass.pickled(123).unpickle()) @@ -195,10 +195,10 @@ class PickledTest(testlib.TestCase): def test_list(self): l = [1, u'b', b('c')] roundtrip = self.roundtrip(l) - self.assertTrue(isinstance(roundtrip, list)) + self.assertIsInstance(roundtrip, list) self.assertEqual(l, roundtrip) for k in range(len(l)): - self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + self.assertIsInstance(roundtrip[k], type(l[k])) @unittest.skipIf(condition=sys.version_info > (3, 0), reason='long missing in >3.x') @@ -206,21 +206,21 @@ class PickledTest(testlib.TestCase): l = long(0xffffffffffff) roundtrip = self.roundtrip(l) self.assertEqual(l, roundtrip) - self.assertTrue(isinstance(roundtrip, long)) + self.assertIsInstance(roundtrip, long) def test_tuple(self): l = (1, u'b', b('c')) roundtrip = self.roundtrip(l) self.assertEqual(l, roundtrip) - self.assertTrue(isinstance(roundtrip, tuple)) + self.assertIsInstance(roundtrip, tuple) for k in range(len(l)): - self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + self.assertIsInstance(roundtrip[k], type(l[k])) def test_unicode(self): u = u'abcd' roundtrip = self.roundtrip(u) self.assertEqual(u, roundtrip) - self.assertTrue(isinstance(roundtrip, mitogen.core.UnicodeType)) + self.assertIsInstance(roundtrip, mitogen.core.UnicodeType) #### custom types. see also: types_test.py, call_error_test.py @@ -231,25 +231,25 @@ class PickledTest(testlib.TestCase): def test_blob_nonempty(self): v = mitogen.core.Blob(b('dave')) roundtrip = self.roundtrip(v) - self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) + self.assertIsInstance(roundtrip, mitogen.core.Blob) self.assertEqual(b('dave'), roundtrip) def test_blob_empty(self): v = mitogen.core.Blob(b('')) roundtrip = self.roundtrip(v) - self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) + self.assertIsInstance(roundtrip, mitogen.core.Blob) self.assertEqual(b(''), v) def test_secret_nonempty(self): s = mitogen.core.Secret(u'dave') roundtrip = self.roundtrip(s) - self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) + self.assertIsInstance(roundtrip, mitogen.core.Secret) self.assertEqual(u'dave', roundtrip) def test_secret_empty(self): s = mitogen.core.Secret(u'') roundtrip = self.roundtrip(s) - self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) + self.assertIsInstance(roundtrip, mitogen.core.Secret) self.assertEqual(u'', roundtrip) def test_call_error(self): @@ -263,7 +263,7 @@ class PickledTest(testlib.TestCase): try: c = router.context_by_id(1234) roundtrip = self.roundtrip(c) - self.assertTrue(isinstance(roundtrip, mitogen.core.Context)) + self.assertIsInstance(roundtrip, mitogen.core.Context) self.assertEqual(c.context_id, 1234) finally: router.broker.shutdown() @@ -275,7 +275,7 @@ class PickledTest(testlib.TestCase): recv = mitogen.core.Receiver(router) sender = recv.to_sender() roundtrip = self.roundtrip(sender, router=router) - self.assertTrue(isinstance(roundtrip, mitogen.core.Sender)) + self.assertIsInstance(roundtrip, mitogen.core.Sender) self.assertEqual(roundtrip.context.context_id, mitogen.context_id) self.assertEqual(roundtrip.dst_handle, sender.dst_handle) finally: @@ -314,7 +314,7 @@ class ReplyTest(testlib.TestCase): my_reply = mitogen.core.Message.pickled(4444) msg.reply(my_reply, router=router) _, (reply,), _ = router.route.mock_calls[0] - self.assertTrue(my_reply is reply) + self.assertIs(my_reply, reply) self.assertEqual(reply.dst_id, 1234) self.assertEqual(reply.unpickle(), 4444) diff --git a/tests/mitogen_protocol_test.py b/tests/mitogen_protocol_test.py index 03e6517c..d6e3cc95 100644 --- a/tests/mitogen_protocol_test.py +++ b/tests/mitogen_protocol_test.py @@ -25,4 +25,4 @@ class ReceiveOneTest(testlib.TestCase): self.assertEqual(1, stream.on_disconnect.call_count) expect = self.klass.corrupt_msg % (stream.name, junk) - self.assertTrue(expect in capture.raw()) + self.assertIn(expect, capture.raw()) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 171c8b4c..02b8b886 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,4 +1,5 @@ import inspect +import json import os import sys import unittest @@ -66,7 +67,7 @@ class GetMainModuleDefectivePython3x(testlib.TestCase): import __main__ path, source, is_pkg = self.call('__main__') - self.assertTrue(path is not None) + self.assertIsNotNone(path) self.assertTrue(os.path.exists(path)) self.assertEqual(path, __main__.__file__) fp = open(path, 'rb') @@ -296,199 +297,65 @@ class FindRelatedTest(testlib.TestCase): self.assertEqual(set(related), self.SIMPLE_EXPECT) -if sys.version_info > (2, 6): - class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') - - # TODO: rip out Django and replace with a static tree of weird imports - # that don't depend on .. Django! The hack below is because the version - # of Django we need to test against 2.6 doesn't actually run on 3.6. - # But we don't care, we just need to be able to import it. - # - # File "django/utils/html_parser.py", line 12, in - # AttributeError: module 'html.parser' has no attribute - # 'HTMLParseError' - # - from django.utils.six.moves import html_parser as _html_parser - _html_parser.HTMLParseError = Exception - - @classmethod - def setUpClass(cls): - super(DjangoMixin, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoMixin, cls).tearDownClass() - - - class FindRelatedImportsTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related_imports(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - 'django', - 'django.core', - 'django.core.signals', - 'django.db.utils', - 'django.utils.functional', - ]) - - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.maxDiff=None - self.assertEqual(related, [ - u'django', - u'django.core.exceptions', - u'django.db', - u'django.db.models', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.query', - u'django.db.models.signals', - ]) - - - class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - maxDiff = None - - def call(self, fullname): - return self.klass().find_related(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.signals', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.utils', - u'django.utils._os', - u'django.utils.encoding', - u'django.utils.functional', - u'django.utils.importlib', - u'django.utils.module_loading', - u'django.utils.six', - ]) - - @unittest.skipIf( - condition=(sys.version_info >= (3, 0)), - reason='broken due to ancient vendored six.py' - ) - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.files', - u'django.core.files.base', - u'django.core.files.images', - u'django.core.files.locks', - u'django.core.files.move', - u'django.core.files.storage', - u'django.core.files.utils', - u'django.core.signals', - u'django.core.validators', - u'django.db', - u'django.db.backends', - u'django.db.backends.signals', - u'django.db.backends.util', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.constants', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.proxy', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.options', - u'django.db.models.query', - u'django.db.models.query_utils', - u'django.db.models.related', - u'django.db.models.signals', - u'django.db.models.sql', - u'django.db.models.sql.aggregates', - u'django.db.models.sql.constants', - u'django.db.models.sql.datastructures', - u'django.db.models.sql.expressions', - u'django.db.models.sql.query', - u'django.db.models.sql.subqueries', - u'django.db.models.sql.where', - u'django.db.transaction', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.forms', - u'django.forms.fields', - u'django.forms.forms', - u'django.forms.formsets', - u'django.forms.models', - u'django.forms.util', - u'django.forms.widgets', - u'django.utils', - u'django.utils._os', - u'django.utils.crypto', - u'django.utils.datastructures', - u'django.utils.dateformat', - u'django.utils.dateparse', - u'django.utils.dates', - u'django.utils.datetime_safe', - u'django.utils.decorators', - u'django.utils.deprecation', - u'django.utils.encoding', - u'django.utils.formats', - u'django.utils.functional', - u'django.utils.html', - u'django.utils.html_parser', - u'django.utils.importlib', - u'django.utils.ipv6', - u'django.utils.itercompat', - u'django.utils.module_loading', - u'django.utils.numberformat', - u'django.utils.safestring', - u'django.utils.six', - u'django.utils.text', - u'django.utils.timezone', - u'django.utils.translation', - u'django.utils.tree', - u'django.utils.tzinfo', - u'pytz', - u'pytz.exceptions', - u'pytz.lazy', - u'pytz.tzfile', - u'pytz.tzinfo', - ]) +class DjangoMixin(object): + WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') + + @classmethod + def modules_expected_path(cls): + if sys.version_info[0:2] < (3, 0): + modules_expected_filename = 'modules_expected_py2x.json' + elif sys.version_info[0:2] <= (3, 6): + modules_expected_filename = 'modules_expected_py3x-legacy.json' + elif sys.version_info[0:2] >= (3, 10): + modules_expected_filename = 'modules_expected_py3x-new.json' + return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) + + @classmethod + def setUpClass(cls): + super(DjangoMixin, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + with open(cls.modules_expected_path(), 'rb') as f: + cls.MODULES_EXPECTED = json.load(f) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoMixin, cls).tearDownClass() + + +class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db') + expected = self.MODULES_EXPECTED['find_related']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db.models') + expected = self.MODULES_EXPECTED['find_related']['django.db.models'] + self.assertEqual(related, expected) + + +class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db.models') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] + self.assertEqual(related, expected) diff --git a/tests/parent_test.py b/tests/parent_test.py index b291f656..fa7e4fb7 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -132,7 +132,7 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): ) ) expect = mitogen.parent.Connection.eof_error_msg - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) def test_direct_enoent(self): e = self.assertRaises(mitogen.core.StreamError, @@ -154,7 +154,7 @@ class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): ) ) s = 'Child start failed: [Errno 2] No such file or directory' - self.assertTrue(s in e.args[0]) + self.assertIn(s, e.args[0]) class ContextTest(testlib.RouterMixin, testlib.TestCase): @@ -173,9 +173,9 @@ class OpenPtyTest(testlib.TestCase): master_fp, slave_fp = self.func() try: self.assertTrue(master_fp.isatty()) - self.assertTrue(isinstance(master_fp, file)) + self.assertIsInstance(master_fp, file) self.assertTrue(slave_fp.isatty()) - self.assertTrue(isinstance(slave_fp, file)) + self.assertIsInstance(slave_fp, file) finally: master_fp.close() slave_fp.close() diff --git a/tests/poller_test.py b/tests/poller_test.py index 4b6bdc0e..f915df0a 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -163,12 +163,12 @@ class PollMixin(PollerMixin): def test_empty_zero_timeout(self): t0 = mitogen.core.now() self.assertEqual([], list(self.p.poll(0))) - self.assertTrue((mitogen.core.now() - t0) < .1) # vaguely reasonable + self.assertLess((mitogen.core.now() - t0), .1) # vaguely reasonable def test_empty_small_timeout(self): t0 = mitogen.core.now() self.assertEqual([], list(self.p.poll(.2))) - self.assertTrue((mitogen.core.now() - t0) >= .2) + self.assertGreaterEqual((mitogen.core.now() - t0), .2) class ReadableMixin(PollerMixin, SockMixin): @@ -394,21 +394,16 @@ class AllMixin(ReceiveStateMixin, """ -class SelectTest(AllMixin, testlib.TestCase): +class CorePollerTest(AllMixin, testlib.TestCase): klass = mitogen.core.Poller -SelectTest = unittest.skipIf( - condition=(not SelectTest.klass.SUPPORTED), - reason='select.select() not supported' -)(SelectTest) - class PollTest(AllMixin, testlib.TestCase): klass = mitogen.parent.PollPoller PollTest = unittest.skipIf( condition=(not PollTest.klass.SUPPORTED), - reason='select.poll() not supported' + reason='select.poll() not available', )(PollTest) @@ -417,7 +412,7 @@ class KqueueTest(AllMixin, testlib.TestCase): KqueueTest = unittest.skipIf( condition=(not KqueueTest.klass.SUPPORTED), - reason='select.kqueue() not supported' + reason='select.kqueue() not available', )(KqueueTest) @@ -426,5 +421,5 @@ class EpollTest(AllMixin, testlib.TestCase): EpollTest = unittest.skipIf( condition=(not EpollTest.klass.SUPPORTED), - reason='select.epoll() not supported' + reason='select.epoll() not available', )(EpollTest) diff --git a/tests/polyfill_functions_test.py b/tests/polyfill_functions_test.py index 222da7b5..c3441a63 100644 --- a/tests/polyfill_functions_test.py +++ b/tests/polyfill_functions_test.py @@ -9,27 +9,27 @@ class BytesPartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(b('dave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) self.assertEqual(left, b('dave')) self.assertEqual(sep, b('')) self.assertEqual(right, b('')) def test_one_sep(self): left, sep, right = self.func(b('davexdave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) self.assertEqual(left, b('dave')) self.assertEqual(sep, b('x')) self.assertEqual(right, b('dave')) def test_two_seps(self): left, sep, right = self.func(b('davexdavexdave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) self.assertEqual(left, b('dave')) self.assertEqual(sep, b('x')) self.assertEqual(right, b('davexdave')) @@ -40,27 +40,27 @@ class StrPartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(u'dave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'dave') self.assertEqual(sep, u'') self.assertEqual(right, u'') def test_one_sep(self): left, sep, right = self.func(u'davexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'dave') self.assertEqual(sep, u'x') self.assertEqual(right, u'dave') def test_two_seps(self): left, sep, right = self.func(u'davexdavexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'dave') self.assertEqual(sep, u'x') self.assertEqual(right, u'davexdave') @@ -71,27 +71,27 @@ class StrRpartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(u'dave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'') self.assertEqual(sep, u'') self.assertEqual(right, u'dave') def test_one_sep(self): left, sep, right = self.func(u'davexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'dave') self.assertEqual(sep, u'x') self.assertEqual(right, u'dave') def test_two_seps(self): left, sep, right = self.func(u'davexdavexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) self.assertEqual(left, u'davexdave') self.assertEqual(sep, u'x') self.assertEqual(right, u'dave') diff --git a/tests/receiver_test.py b/tests/receiver_test.py index aaf15b2b..0cb9b15d 100644 --- a/tests/receiver_test.py +++ b/tests/receiver_test.py @@ -16,8 +16,8 @@ def yield_stuff_then_die(sender): class ConstructorTest(testlib.RouterMixin, testlib.TestCase): def test_handle(self): recv = mitogen.core.Receiver(self.router) - self.assertTrue(isinstance(recv.handle, int)) - self.assertTrue(recv.handle > 100) + self.assertIsInstance(recv.handle, int) + self.assertGreater(recv.handle, 100) self.router.route( mitogen.core.Message.pickled( 'hi', diff --git a/tests/requirements.txt b/tests/requirements.txt index 319e0f51..1e5d2a1d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,13 +1,13 @@ -psutil==5.4.8 -coverage==4.5.1 -Django==1.6.11 # Last version supporting 2.6. -faulthandler==3.2; python_version < '3.3' +cffi==1.15.1 +coverage==5.5; python_version < '3.7' +coverage==6.4.4; python_version >= '3.7' +Django==1.11.29; python_version < '3.0' +Django==3.2.20; python_version >= '3.6' mock==2.0.0 -pytz==2018.5 -cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect -pycparser==2.18 # Last version supporting 2.6. +psutil==5.9.5 pytest-catchlog==1.2.2 pytest==3.1.2 +subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings diff --git a/tests/responder_test.py b/tests/responder_test.py index 3ab60be3..1131b421 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -25,7 +25,7 @@ class NeutralizeMainTest(testlib.RouterMixin, testlib.TestCase): _, stderr = proc.communicate() self.assertEqual(1, proc.returncode) expect = self.klass.main_guard_msg % (path,) - self.assertTrue(expect in stderr.decode()) + self.assertIn(expect, stderr.decode()) HAS_MITOGEN_MAIN = mitogen.core.b( textwrap.dedent(""" @@ -93,8 +93,10 @@ class GoodModulesTest(testlib.RouterMixin, testlib.TestCase): # Ensure a program composed of a single script can be imported # successfully. args = [sys.executable, testlib.data_path('self_contained_program.py')] - output = testlib.subprocess__check_output(args).decode() - self.assertEqual(output, "['__main__', 50]\n") + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + b_stdout, _ = proc.communicate() + self.assertEqual(proc.returncode, 0) + self.assertEqual(b_stdout.decode(), "['__main__', 50]\n") class BrokenModulesTest(testlib.TestCase): @@ -179,7 +181,7 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase): ) ) s = capture.stop() - self.assertTrue('dropping forward of' in s) + self.assertIn('dropping forward of', s) def test_stats(self): # Forwarding stats broken because forwarding is broken. See #469. diff --git a/tests/router_test.py b/tests/router_test.py index 656ca734..27668154 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -71,7 +71,7 @@ class SourceVerifyTest(testlib.RouterMixin, testlib.TestCase): # Ensure error was logged. expect = 'bad auth_id: got %r via' % (self.child2_msg.auth_id,) - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) def test_parent_unaware_of_disconnect(self): # Parent -> Child A -> Child B. B disconnects concurrent to Parent @@ -125,7 +125,7 @@ class SourceVerifyTest(testlib.RouterMixin, testlib.TestCase): # Ensure error was lgoged. expect = 'bad src_id: got %d via' % (self.child1_msg.src_id,) - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) class PolicyTest(testlib.RouterMixin, testlib.TestCase): @@ -165,7 +165,7 @@ class PolicyTest(testlib.RouterMixin, testlib.TestCase): self.sync_with_broker() # Verify log. - self.assertTrue(self.router.refused_msg in log.stop()) + self.assertIn(self.router.refused_msg, log.stop()) # Verify message was not delivered. self.assertTrue(recv.empty()) @@ -202,7 +202,7 @@ class CrashTest(testlib.BrokerMixin, testlib.TestCase): # Ensure it was logged. expect = 'broker crashed' - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) self.broker.join() @@ -264,7 +264,7 @@ class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase): router.broker.defer_sync(lambda: None) expect = 'message too large (max 4096 bytes)' - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) def test_local_dead_message(self): # Local router should generate dead message when reply_to is set. @@ -282,7 +282,7 @@ class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase): lambda: child.call(zlib.crc32, ' '*8192)) self.assertEqual(e.args[0], expect) - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) def test_remote_dead_message(self): # Router should send dead message to original recipient when reply_to @@ -325,7 +325,7 @@ class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase): remote.call(send_n_sized_reply, recv.to_sender(), 128*1024) expect = 'message too large (max %d bytes)' % (64*1024,) - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) class NoRouteTest(testlib.RouterMixin, testlib.TestCase): diff --git a/tests/service_test.py b/tests/service_test.py index 5d08bb47..d22558f1 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -83,14 +83,14 @@ class ActivationTest(testlib.RouterMixin, testlib.TestCase): l1 = self.router.local() counter, id_ = l1.call_service(MyService, 'get_id') self.assertEqual(1, counter) - self.assertTrue(isinstance(id_, int)) + self.assertIsInstance(id_, int) def test_sibling_cannot_activate_framework(self): l1 = self.router.local(name='l1') l2 = self.router.local(name='l2') exc = self.assertRaises(mitogen.core.CallError, lambda: l2.call(call_service_in, l1, MyService2.name(), 'get_id')) - self.assertTrue(mitogen.core.Router.refused_msg in exc.args[0]) + self.assertIn(mitogen.core.Router.refused_msg, exc.args[0]) def test_sibling_cannot_activate_service(self): l1 = self.router.local() @@ -104,7 +104,7 @@ class ActivationTest(testlib.RouterMixin, testlib.TestCase): finally: capture.stop() msg = mitogen.service.Activator.not_active_msg % (MyService2.name(),) - self.assertTrue(msg in exc.args[0]) + self.assertIn(msg, exc.args[0]) def test_activates_only_once(self): l1 = self.router.local() @@ -138,7 +138,7 @@ class PermissionTest(testlib.RouterMixin, testlib.TestCase): u'privileged_op', MyService.name(), ) - self.assertTrue(msg in exc.args[0]) + self.assertIn(msg, exc.args[0]) class CloseTest(testlib.RouterMixin, testlib.TestCase): diff --git a/tests/ssh_test.py b/tests/ssh_test.py index c1a0b9eb..3149fcbc 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -52,7 +52,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): s = capture.stop() expect = "%s: debug1: Reading configuration data" % (context.name,) - self.assertTrue(expect in s) + self.assertIn(expect, s) def test_bash_permission_denied(self): # issue #271: only match Permission Denied at start of line. @@ -134,12 +134,13 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_enforce_unknown_host_key(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: e = self.assertRaises(mitogen.ssh.HostKeyError, lambda: self.docker_ssh( username='mitogen__has_sudo_pubkey', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) ) @@ -149,11 +150,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_accept_enforce_host_keys(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='accept', ) context.shutdown(wait=True) @@ -161,12 +163,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): fp.seek(0) # Lame test, but we're about to use enforce mode anyway, which # verifies the file contents. - self.assertTrue(len(fp.read()) > 0) + self.assertGreater(len(fp.read()), 0) context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) context.shutdown(wait=True) diff --git a/tests/su_test.py b/tests/su_test.py index d8952b53..234c509b 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -45,7 +45,7 @@ class SuTest(testlib.DockerMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.su(via=ssh) ) - self.assertTrue(mitogen.su.password_required_msg in str(e)) + self.assertIn(mitogen.su.password_required_msg, str(e)) def test_password_incorrect(self): ssh = self.docker_ssh( @@ -55,7 +55,7 @@ class SuTest(testlib.DockerMixin, testlib.TestCase): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.su(via=ssh, password='x') ) - self.assertTrue(mitogen.su.password_incorrect_msg in str(e)) + self.assertIn(mitogen.su.password_incorrect_msg, str(e)) def test_password_okay(self): ssh = self.docker_ssh( diff --git a/tests/sudo_test.py b/tests/sudo_test.py index 2df05ec7..a8ec1d05 100644 --- a/tests/sudo_test.py +++ b/tests/sudo_test.py @@ -73,7 +73,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh) # ) -# self.assertTrue(mitogen.sudo.password_required_msg in str(e)) +# self.assertIn(mitogen.sudo.password_required_msg, str(e)) # def test_password_incorrect(self): # ssh = self.docker_ssh( @@ -85,7 +85,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh, password='x') # ) -# self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) +# self.assertIn(mitogen.sudo.password_incorrect_msg, str(e)) # def test_password_okay(self): # ssh = self.docker_ssh( @@ -97,4 +97,4 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh, password='rootpassword') # ) -# self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) +# self.assertIn(mitogen.sudo.password_incorrect_msg, str(e)) diff --git a/tests/testlib.py b/tests/testlib.py index 8ab895c4..ec0a7443 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,18 +1,27 @@ import errno +import io import logging import os import random import re -import signal import socket -import subprocess +import stat import sys import threading import time import traceback import unittest +try: + import configparser +except ImportError: + import ConfigParser as configparser + import psutil +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess import mitogen.core import mitogen.fork @@ -63,36 +72,73 @@ if faulthandler is not None: mitogen.core.LOG.propagate = True -def data_path(suffix): - path = os.path.join(DATA_DIR, suffix) - if path.endswith('.key'): - # SSH is funny about private key permissions. - os.chmod(path, int('0600', 8)) - return path +def base_executable(executable=None): + '''Return the path of the Python executable used to create the virtualenv. + ''' + # https://docs.python.org/3/library/venv.html + # https://github.com/pypa/virtualenv/blob/main/src/virtualenv/discovery/py_info.py + # https://virtualenv.pypa.io/en/16.7.9/reference.html#compatibility-with-the-stdlib-venv-module + if executable is None: + executable = sys.executable + if not executable: + raise ValueError -def subprocess__check_output(*popenargs, **kwargs): - # Missing from 2.6. - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, _ = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output + try: + base_executable = sys._base_executable + except AttributeError: + base_executable = None + if base_executable and base_executable != executable: + return base_executable -def Popen__terminate(proc): - os.kill(proc.pid, signal.SIGTERM) + # Python 2.x only has sys.base_prefix if running outside a virtualenv. + try: + sys.base_prefix + except AttributeError: + # Python 2.x outside a virtualenv + return executable + + # Python 3.3+ has sys.base_prefix. In a virtualenv it differs to sys.prefix. + if sys.base_prefix == sys.prefix: + return executable + + while executable.startswith(sys.prefix) and stat.S_ISLNK(os.lstat(executable).st_mode): + dirname = os.path.dirname(executable) + target = os.path.join(dirname, os.readlink(executable)) + executable = os.path.abspath(os.path.normpath(target)) + print(executable) + + if executable.startswith(sys.base_prefix): + return executable + + # Virtualenvs record details in pyvenv.cfg + parser = configparser.RawConfigParser() + with io.open(os.path.join(sys.prefix, 'pyvenv.cfg'), encoding='utf-8') as f: + content = u'[virtualenv]\n' + f.read() + try: + parser.read_string(content) + except AttributeError: + parser.readfp(io.StringIO(content)) + # virtualenv style pyvenv.cfg includes the base executable. + # venv style pyvenv.cfg doesn't. + try: + return parser.get(u'virtualenv', u'base-executable') + except configparser.NoOptionError: + pass -if hasattr(subprocess, 'check_output'): - subprocess__check_output = subprocess.check_output + basename = os.path.basename(executable) + home = parser.get(u'virtualenv', u'home') + return os.path.join(home, basename) -if hasattr(subprocess.Popen, 'terminate'): - Popen__terminate = subprocess.Popen.terminate + +def data_path(suffix): + path = os.path.join(DATA_DIR, suffix) + if path.endswith('.key'): + # SSH is funny about private key permissions. + os.chmod(path, int('0600', 8)) + return path def threading__thread_is_alive(thread): @@ -456,23 +502,22 @@ def get_docker_host(): class DockerizedSshDaemon(object): - def _get_container_port(self): - s = subprocess__check_output(['docker', 'port', self.container_name]) - for line in s.decode().splitlines(): - m = self.PORT_RE.match(line) - if not m: - continue - dport, proto, _, bport = m.groups() - if dport == '22' and proto == 'tcp': - self.port = int(bport) + PORT_RE = re.compile( + # e.g. 0.0.0.0:32771, :::32771, [::]:32771' + r'(?P[0-9.]+|::|\[[a-f0-9:.]+\]):(?P[0-9]+)', + ) - self.host = self.get_host() - if self.port is None: + @classmethod + def get_port(cls, container): + s = subprocess.check_output(['docker', 'port', container, '22/tcp']) + m = cls.PORT_RE.search(s.decode()) + if not m: raise ValueError('could not find SSH port in: %r' % (s,)) + return int(m.group('port')) def start_container(self): try: - subprocess__check_output(['docker', '--version']) + subprocess.check_output(['docker', '--version']) except Exception: raise unittest.SkipTest('Docker binary is unavailable') @@ -486,8 +531,7 @@ class DockerizedSshDaemon(object): '--name', self.container_name, self.image, ] - subprocess__check_output(args) - self._get_container_port() + subprocess.check_output(args) def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): if '-' in mitogen_test_distro: @@ -502,12 +546,9 @@ class DockerizedSshDaemon(object): self.python_path = '/usr/bin/python' self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - - # 22/tcp -> 0.0.0.0:32771 - self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') - self.port = None - self.start_container() + self.host = self.get_host() + self.port = self.get_port(self.container_name) def get_host(self): return get_docker_host() @@ -518,7 +559,7 @@ class DockerizedSshDaemon(object): def check_processes(self): args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm='] counts = {} - for comm in subprocess__check_output(args).decode().splitlines(): + for comm in subprocess.check_output(args).decode().splitlines(): comm = comm.strip() counts[comm] = counts.get(comm, 0) + 1 @@ -533,7 +574,7 @@ class DockerizedSshDaemon(object): def close(self): args = ['docker', 'rm', '-f', self.container_name] - subprocess__check_output(args) + subprocess.check_output(args) class BrokerMixin(object): @@ -588,12 +629,33 @@ class DockerMixin(RouterMixin): cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() + @property + def docker_ssh_default_kwargs(self): + return { + 'hostname': self.dockerized_ssh.host, + 'port': self.dockerized_ssh.port, + 'check_host_keys': 'ignore', + 'ssh_debug_level': 3, + # https://www.openssh.com/legacy.html + # ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. + # Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). + # Duplicated cases in + # - tests/ansible/ansible.cfg + # - tests/ansible/integration/connection_delegation/delegate_to_template.yml + # - tests/ansible/integration/connection_delegation/stack_construction.yml + # - tests/ansible/integration/process/unix_socket_cleanup.yml + # - tests/ansible/integration/ssh/variables.yml + # - tests/testlib.py + 'ssh_args': [ + '-o', 'HostKeyAlgorithms +ssh-rsa', + '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa', + ], + 'python_path': self.dockerized_ssh.python_path, + } + def docker_ssh(self, **kwargs): - kwargs.setdefault('hostname', self.dockerized_ssh.host) - kwargs.setdefault('port', self.dockerized_ssh.port) - kwargs.setdefault('check_host_keys', 'ignore') - kwargs.setdefault('ssh_debug_level', 3) - kwargs.setdefault('python_path', self.dockerized_ssh.python_path) + for k, v in self.docker_ssh_default_kwargs.items(): + kwargs.setdefault(k, v) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): diff --git a/tests/timer_test.py b/tests/timer_test.py index ff30c905..dac4d7df 100644 --- a/tests/timer_test.py +++ b/tests/timer_test.py @@ -87,7 +87,7 @@ class ScheduleTest(TimerListMixin, testlib.TestCase): timer2 = self.list.schedule(31, lambda: None) self.assertEqual(31, timer.when) self.assertEqual(31, timer2.when) - self.assertTrue(timer is not timer2) + self.assertIsNot(timer, timer2) self.assertEqual(1, self.list.get_timeout()) diff --git a/tests/types_test.py b/tests/types_test.py index 7c212b70..0229b94c 100644 --- a/tests/types_test.py +++ b/tests/types_test.py @@ -90,8 +90,8 @@ class KwargsTest(testlib.TestCase): self.assertEqual({}, kw) self.assertEqual('Kwargs({})', repr(kw)) klass, (dct,) = kw.__reduce__() - self.assertTrue(klass is self.klass) - self.assertTrue(type(dct) is dict) + self.assertIs(klass, self.klass) + self.assertIs(type(dct), dict) self.assertEqual({}, dct) @unittest.skipIf(condition=(sys.version_info >= (2, 6)), @@ -108,11 +108,11 @@ class KwargsTest(testlib.TestCase): self.assertEqual({u'key': 123}, kw) self.assertEqual("Kwargs({'key': 123})", repr(kw)) klass, (dct,) = kw.__reduce__() - self.assertTrue(klass is self.klass) - self.assertTrue(type(dct) is dict) + self.assertIs(klass, self.klass) + self.assertIs(type(dct), dict) self.assertEqual({u'key': 123}, dct) key, = dct - self.assertTrue(type(key) is mitogen.core.UnicodeType) + self.assertIs(type(key), mitogen.core.UnicodeType) class AdornedUnicode(mitogen.core.UnicodeType): diff --git a/tests/unix_test.py b/tests/unix_test.py index aecc62ba..14fc54ae 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -130,7 +130,8 @@ class ClientTest(testlib.TestCase): def test_simple(self): path = mitogen.unix.make_socket_path() proc = subprocess.Popen( - [sys.executable, __file__, 'ClientTest_server', path] + [sys.executable, __file__, 'ClientTest_server', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) try: self._test_simple_client(path) @@ -139,7 +140,9 @@ class ClientTest(testlib.TestCase): mitogen.context_id = 0 mitogen.parent_id = None mitogen.parent_ids = [] - proc.wait() + b_stdout, _ = proc.communicate() + self.assertEqual(proc.returncode, 0) + self.assertEqual(b_stdout.decode(), '') if __name__ == '__main__': diff --git a/tox.ini b/tox.ini index 6b2addc7..e4c26c39 100644 --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,20 @@ -# This file is a local convenience. It is not a substitute for the full CI -# suite, and does not cover the full range of Python versions for Mitogen. - -# I use this on Ubuntu 20.04, with the following additions +# This configuration drives both CI and local development. +# I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox -# Last version to support each python version -# -# Python tox virt'env pip A cntllr A target coverage -# ========== ======== ======== ======== ======== ======== ======== -# python2.4 1.4 1.8 1.1 2.3? -# python2.5 1.6.1 1.9.1 1.3.1 ??? -# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 2.13 4.5.4 -# python2.7 20.3 2.11 -# python3.5 2.11 -# python3.6 2.11 -# python3.7 2.11 +# Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv +# ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== +# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 +# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 +# 2.6 <= 2.6.20 <= 2.13 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? +# 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 +# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 +# 3.7 <= 2.12 <= 3.2.20 +# 3.8 <= 2.12 # Ansible Dependency # ================== ====================== @@ -26,6 +23,7 @@ # ansible == 3.* ansible-base ~= 2.10.0 # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 +# ansible == 6.* ansible-core ~= 2.13.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -36,13 +34,11 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{310}-mode_ansible-ansible{2.10,3,4,5}, - py{27,36,310}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,310}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{311}-mode_ansible-ansible{2.10,3,4,5,6}, + py{27,36,311}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,311}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, -requires = - tox-factor [testenv] basepython = @@ -54,19 +50,15 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt - ansible2.3: ansible==2.3.3.0 - ansible2.4: ansible==2.4.6.0 - ansible2.8: ansible==2.8.3 - ansible2.9: ansible==2.9.6 - ansible2.10: ansible-base<2.10.14 - ansible2.10: ansible==2.10.0 - ansible3: ansible-base<2.10.14 + ansible2.10: ansible==2.10.7 ansible3: ansible==3.4.0 - ansible4: ansible==4.8.0 - ansible5: ansible==5.0.1 + ansible4: ansible==4.10.0 + ansible5: ansible==5.8.0 + ansible6: ansible==6.0.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -86,6 +78,7 @@ passenv = AWS_SECRET_ACCESS_KEY HOME setenv = + # See also azure-pipelines.yml ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 @@ -102,6 +95,8 @@ setenv = distro_ubuntu1804: DISTRO=ubuntu1804 distro_ubuntu2004: DISTRO=ubuntu2004 # Note the plural, only applicable to MODE=ansible + # Ansible 6 (ansible-core 2.13) requires Python >= 2.7 or >= 3.5 on targets + ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6 @@ -144,7 +139,9 @@ whitelist_externals = echo [testenv:docs] -basepython = python3 +basepython = python3.8 changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +deps = + -r docs/requirements.txt