From 4506c528cadfee206d808b1b982b5291e5591fbf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 10 Feb 2019 22:32:57 +0000 Subject: [PATCH 01/54] issue #481: add test. --- tests/data/stubs/stub-sudo.py | 13 +++++++++---- tests/sudo_test.py | 9 +++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/data/stubs/stub-sudo.py b/tests/data/stubs/stub-sudo.py index a7f2704f..71364df7 100755 --- a/tests/data/stubs/stub-sudo.py +++ b/tests/data/stubs/stub-sudo.py @@ -8,7 +8,12 @@ import sys os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) os.environ['THIS_IS_STUB_SUDO'] = '1' -# This must be a child process and not exec() since Mitogen replaces its stderr -# descriptor, causing the last user of the slave PTY to close it, resulting in -# the master side indicating EIO. -subprocess.check_call(sys.argv[sys.argv.index('--') + 1:]) +if os.environ.get('PREHISTORIC_SUDO'): + # issue #481: old versions of sudo did in fact use execve, thus we must + # have TTY handle preservation in core.py. + os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) +else: + # This must be a child process and not exec() since Mitogen replaces its + # stderr descriptor, causing the last user of the slave PTY to close it, + # resulting in the master side indicating EIO. + subprocess.check_call(sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/sudo_test.py b/tests/sudo_test.py index 5bf9f4de..1d10ba9a 100644 --- a/tests/sudo_test.py +++ b/tests/sudo_test.py @@ -55,6 +55,15 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): '--' ]) + def test_tty_preserved(self): + # issue #481 + os.environ['PREHISTORIC_SUDO'] = '1' + try: + context, argv = self.run_sudo() + self.assertEquals('1', context.call(os.getenv, 'PREHISTORIC_SUDO')) + finally: + del os.environ['PREHISTORIC_SUDO'] + class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase): # Only mitogen/debian-test has a properly configured sudo. From 8592765732a0c8d6af99537a953cdfa8779398ff Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 10 Feb 2019 22:38:26 +0000 Subject: [PATCH 02/54] stable: fix preamble_size on stable docs. --- preamble_size.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/preamble_size.py b/preamble_size.py index ead5af85..f5f1adc1 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -8,6 +8,7 @@ import sys import zlib import mitogen.fakessh +import mitogen.fork import mitogen.master import mitogen.minify import mitogen.parent @@ -16,9 +17,6 @@ import mitogen.service import mitogen.ssh import mitogen.sudo -import ansible_mitogen.runner -import ansible_mitogen.target - router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) stream = mitogen.ssh.Stream(router, 0, max_message_size=0, hostname='foo') @@ -46,8 +44,6 @@ print( for mod in ( mitogen.parent, mitogen.fork, - ansible_mitogen.target, - ansible_mitogen.runner, mitogen.ssh, mitogen.sudo, mitogen.select, From 81e386586f8bb0e96abf0ce4fd740e2978aaedbc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:04:30 +0000 Subject: [PATCH 03/54] tests: allow running Ansible tests locally without -udmw again. --- tests/ansible/hosts/default.hosts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 02f3c614..d40c3dd0 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,8 +1,9 @@ # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the -# local machine. -target ansible_host=localhost +# local machine. The ansible_user override is necessary since some tests want a +# fixed ansible.cfg remote_user setting to test against. +target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" [test-targets] target From 86f9572ef7a5547299d67e345c17bf6560af848e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:06:22 +0000 Subject: [PATCH 04/54] issue #536: stop defining explicit localhost in inventory. This was needed at some point in the past, but the tests don't seem to care about it any more. We'll fix any CI breakage by changing the tests, since verifying implicit localhost behaviour is important. --- tests/ansible/hosts/localhost.hosts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts index 89bf7b38..41af412e 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts @@ -1,9 +1,8 @@ # vim: syntax=dosini -# This must be defined explicitly, otherwise _create_implicit_localhost() -# generates its own copy, which includes an ansible_python_interpreter that -# varies according to host machine. -localhost +# issue #511, #536: we must not define an explicit localhost, as some +# transport_config/python_path.yml needs to test the implicit localhost +# behaviour. # This is only used for manual testing. [localhost-x10] From 01faed708d82dcb6ac1667df38a92ee32d771415 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:07:22 +0000 Subject: [PATCH 05/54] issue #536: add tests for each ansible_python_interpreter case. --- tests/ansible/hosts/transport_config.hosts | 10 ++++ tests/ansible/integration/all.yml | 1 + .../integration/transport_config/README.md | 7 +++ .../integration/transport_config/all.yml | 2 + .../transport_config/python_path.yml | 51 +++++++++++++++++++ 5 files changed, 71 insertions(+) create mode 100644 tests/ansible/hosts/transport_config.hosts create mode 100644 tests/ansible/integration/transport_config/README.md create mode 100644 tests/ansible/integration/transport_config/all.yml create mode 100644 tests/ansible/integration/transport_config/python_path.yml diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts new file mode 100644 index 00000000..d9685811 --- /dev/null +++ b/tests/ansible/hosts/transport_config.hosts @@ -0,0 +1,10 @@ +# integration/transport_config +# Hosts with twiddled configs that need to be checked somehow. + +tc-python-path-unset +tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python + +# local connections get virtualenv python path +tc-python-path-local-unset ansible_connection=local +tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c + diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index bd68b4ab..5898b9cd 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -19,3 +19,4 @@ - include: ssh/all.yml - include: strategy/all.yml - include: stub_connections/all.yml +- include: transport_config/all.yml diff --git a/tests/ansible/integration/transport_config/README.md b/tests/ansible/integration/transport_config/README.md new file mode 100644 index 00000000..bec55d04 --- /dev/null +++ b/tests/ansible/integration/transport_config/README.md @@ -0,0 +1,7 @@ + +# Tests for correct selection of connection variables. + +This directory is a placeholder for a work-in-progress test set that tries +every combination of the variables extracted via `transport_config.py`. + +In the meantime, it has ad-hoc scripts for bugs already encountered. diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml new file mode 100644 index 00000000..704760d1 --- /dev/null +++ b/tests/ansible/integration/transport_config/all.yml @@ -0,0 +1,2 @@ + +- include: python_path.yml diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml new file mode 100644 index 00000000..52546903 --- /dev/null +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -0,0 +1,51 @@ +# related: issue #511, #536 + +- name: integration/transport_config/python_path.yml + hosts: tc-python-path-unset + + # When no ansible_python_interpreter is set, executor/module_common.py + # chooses "/usr/bin/python". + tasks: + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + + +- hosts: tc-python-path-hostvar + tasks: + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: [/hostvar/path/to/python] + + +# Implicit localhost gets ansible_python_interpreter=virtualenv interpreter +- hosts: localhost + tasks: + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + + +# explicit local connections get the same treatment as everything else. +- hosts: tc-python-path-local-unset + tasks: + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + + +- hosts: tc-python-path-local-explicit + tasks: + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/a/b/c"] From 0dfcf5560b4392db72877319aeb4fa985f0ddc73 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:08:02 +0000 Subject: [PATCH 06/54] tests: define MITOGEN_INVENTORY_FILE even if -i unspecified. To fix running tests locally. --- tests/ansible/run_ansible_playbook.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ansible/run_ansible_playbook.py b/tests/ansible/run_ansible_playbook.py index 51f864f4..b5b459a1 100755 --- a/tests/ansible/run_ansible_playbook.py +++ b/tests/ansible/run_ansible_playbook.py @@ -46,6 +46,10 @@ if '-i' in sys.argv: extra['MITOGEN_INVENTORY_FILE'] = ( os.path.abspath(sys.argv[1 + sys.argv.index('-i')]) ) +else: + extra['MITOGEN_INVENTORY_FILE'] = ( + os.path.join(GIT_BASEDIR, 'tests/ansible/hosts') + ) args = ['ansible-playbook'] args += ['-e', json.dumps(extra)] From 1d43e187e8dfa2313e7a865c1ebcc2f033eb6d8b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:09:16 +0000 Subject: [PATCH 07/54] issue #536: connection_delegation/ tests were erroneously broken While fixing delegate_to, this un-hardwiring of /usr/bin/python happened. It was always incorrect. --- .../delegate_to_template.yml | 4 +-- .../connection_delegation/local_action.yml | 4 +-- .../osa_delegate_to_self.yml | 2 +- .../stack_construction.yml | 26 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index a5c0216c..6e18ab6d 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -39,7 +39,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -66,7 +66,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', diff --git a/tests/ansible/integration/connection_delegation/local_action.yml b/tests/ansible/integration/connection_delegation/local_action.yml index d166c0d9..9d2cb65c 100644 --- a/tests/ansible/integration/connection_delegation/local_action.yml +++ b/tests/ansible/integration/connection_delegation/local_action.yml @@ -15,7 +15,7 @@ right: [ { 'kwargs': { - 'python_path': null + 'python_path': ["{{ansible_playbook_python}}"], }, 'method': 'local', }, @@ -23,7 +23,7 @@ 'enable_lru': true, 'kwargs': { 'connect_timeout': 10, - 'python_path': null, + 'python_path': ["{{ansible_playbook_python}}"], 'password': null, 'username': 'root', 'sudo_path': null, diff --git a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml index a761c432..4a1fa681 100644 --- a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml +++ b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml @@ -24,7 +24,7 @@ 'lxc_info_path': null, 'lxc_path': null, 'machinectl_path': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'ansible-cfg-remote-user', }, 'method': 'setns', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 0c48be3f..1b1f249d 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -43,7 +43,7 @@ "connect_timeout": 10, "doas_path": null, "password": null, - "python_path": null, + "python_path": ["/usr/bin/python"], "username": "normal-user", }, "method": "doas", @@ -72,7 +72,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -112,7 +112,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -147,7 +147,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -162,7 +162,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -202,7 +202,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -229,7 +229,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -264,7 +264,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -279,7 +279,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -320,7 +320,7 @@ 'identity_file': null, 'password': null, 'port': null, - 'python_path': null, + "python_path": ["/usr/bin/python"], 'ssh_args': [ '-o', 'UserKnownHostsFile=/dev/null', @@ -352,7 +352,7 @@ right: [ { 'kwargs': { - 'python_path': null + "python_path": ["{{ansible_playbook_python}}"], }, 'method': 'local', }, @@ -374,7 +374,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'normal-user', }, 'method': 'doas', @@ -384,7 +384,7 @@ 'connect_timeout': 10, 'doas_path': null, 'password': null, - 'python_path': null, + 'python_path': ["/usr/bin/python"], 'username': 'newuser-doas-normal-user', }, 'method': 'doas', From 001e3fee86fad6ca122db7961d285c343b895cd4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:11:43 +0000 Subject: [PATCH 08/54] issue #536: restore correct Python interpreter selection behaviour. --- ansible_mitogen/transport_config.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 290c12d5..3401b035 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -329,9 +329,11 @@ class PlayContextSpec(Spec): return self._play_context.port def python_path(self): - return parse_python_path( - self._connection.get_task_var('ansible_python_interpreter') - ) + s = self._connection.get_task_var('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') def private_key_file(self): return self._play_context.private_key_file @@ -487,11 +489,12 @@ class MitogenViaSpec(Spec): ) def python_path(self): - return parse_python_path( - self._host_vars.get('ansible_python_interpreter') - # This variable has no default for remote hosts. For local hosts it - # is sys.executable. - ) + s = self._host_vars.get('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') + def private_key_file(self): # TODO: must come from PlayContext too. From b3f20f54e6edf0dc832ad7cfdb2a3054b02cb9e1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 00:11:43 +0000 Subject: [PATCH 09/54] issue #536: disable transport_config tests on vanilla --- .../transport_config/python_path.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index 52546903..d0c16788 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -1,11 +1,13 @@ # related: issue #511, #536 + +# When no ansible_python_interpreter is set, executor/module_common.py chooses +# "/usr/bin/python". - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset - - # When no ansible_python_interpreter is set, executor/module_common.py - # chooses "/usr/bin/python". tasks: + - meta: end_play + when: not is_mitogen - mitogen_get_stack: register: out - assert_equal: @@ -13,8 +15,11 @@ right: ["/usr/bin/python"] +# Non-localhost with explicit ansible_python_interpreter - hosts: tc-python-path-hostvar tasks: + - meta: end_play + when: not is_mitogen - mitogen_get_stack: register: out - assert_equal: @@ -25,6 +30,8 @@ # Implicit localhost gets ansible_python_interpreter=virtualenv interpreter - hosts: localhost tasks: + - meta: end_play + when: not is_mitogen - mitogen_get_stack: register: out - assert_equal: @@ -35,6 +42,8 @@ # explicit local connections get the same treatment as everything else. - hosts: tc-python-path-local-unset tasks: + - meta: end_play + when: not is_mitogen - mitogen_get_stack: register: out - assert_equal: @@ -44,6 +53,8 @@ - hosts: tc-python-path-local-explicit tasks: + - meta: end_play + when: not is_mitogen - mitogen_get_stack: register: out - assert_equal: From f514dbeba9b9708a5ae1349ce39767a6d6d29c5f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 10 Feb 2019 22:10:40 +0000 Subject: [PATCH 10/54] docs: update Changelog release date. --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bcca6088..9d1d9079 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,7 +125,7 @@ Core Library series. -v0.2.4 (2018-??-??) +v0.2.4 (2019-02-10) ------------------- Mitogen for Ansible From 90401833facfe8f321fff6989bced795444eaf0b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 01:22:52 +0000 Subject: [PATCH 11/54] docs: update Changelog; closes #511, closes #536. --- docs/changelog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d1d9079..635115e7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,6 +125,23 @@ Core Library series. +v0.2.5 (2019-02-1?) +------------------- + +* `#511 `_, + `#536 `_: changes in 0.2.4 to + repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` + handling. Test coverage was added. + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +bug reports, testing, features and fixes in this release contributed by +`Guy Knights `_, and +`Josh Smift `_, + + v0.2.4 (2019-02-10) ------------------- From 2f29c76eecc8475b445aa5e2e4eadb82d0deff3c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 02:06:44 +0000 Subject: [PATCH 12/54] tests: for 2.3 compatibility, disable gcloud.py for now --- tests/ansible/ansible.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index a968f84a..bec749f7 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,5 +1,5 @@ [defaults] -inventory = hosts,lib/inventory +inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy action_plugins = lib/action From 604b418412eff09a0f0ac8147a0fd7459e5a5b65 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 02:10:32 +0000 Subject: [PATCH 13/54] ansible: fix a crash on 2.3 when mitogen_via= host is missing. --- ansible_mitogen/connection.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c7e70c43..b00430ca 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -547,7 +547,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): def connected(self): return self.context is not None - def _spec_from_via(self, via_spec): + def _spec_from_via(self, proxied_inventory_name, via_spec): """ Produce a dict connection specifiction given a string `via_spec`, of the form `[[become_method:]become_user@]inventory_hostname`. @@ -555,15 +555,17 @@ class Connection(ansible.plugins.connection.ConnectionBase): become_user, _, inventory_name = via_spec.rpartition('@') become_method, _, become_user = become_user.rpartition(':') - via_vars = self.host_vars[inventory_name] - if isinstance(via_vars, jinja2.runtime.Undefined): + # must use __contains__ to avoid a TypeError for a missing host on + # Ansible 2.3. + if self.host_vars is None or inventory_name not in self.host_vars: raise ansible.errors.AnsibleConnectionFailure( self.unknown_via_msg % ( via_spec, - inventory_name, + proxied_inventory_name, ) ) + via_vars = self.host_vars[inventory_name] return ansible_mitogen.transport_config.MitogenViaSpec( inventory_name=inventory_name, host_vars=dict(via_vars), # TODO: make it lazy @@ -615,7 +617,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): if spec.mitogen_via(): stack = self._stack_from_spec( - self._spec_from_via(spec.mitogen_via()), + self._spec_from_via(spec.inventory_name(), spec.mitogen_via()), stack=stack, seen_names=seen_names + (spec.inventory_name(),), ) From e1df98168c0e056f27f8de70a4080773955ea8a7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 02:12:39 +0000 Subject: [PATCH 14/54] issue #536: add mitogen_via= tests too. --- ansible_mitogen/transport_config.py | 1 - tests/ansible/integration/_mitogen_only.yml | 4 + .../transport_config/python_path.yml | 82 ++++++++++++++++--- 3 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 tests/ansible/integration/_mitogen_only.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 3401b035..511cba5f 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -495,7 +495,6 @@ class MitogenViaSpec(Spec): # interpreter is specified. return parse_python_path(s or '/usr/bin/python') - def private_key_file(self): # TODO: must come from PlayContext too. return ( diff --git a/tests/ansible/integration/_mitogen_only.yml b/tests/ansible/integration/_mitogen_only.yml new file mode 100644 index 00000000..85ef378e --- /dev/null +++ b/tests/ansible/integration/_mitogen_only.yml @@ -0,0 +1,4 @@ +# Include me for plays that can't run on vanilla. +# +- meta: end_play + when: not is_mitogen diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index d0c16788..db6c873b 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -1,4 +1,5 @@ # related: issue #511, #536 +# Each case is followed by mitogen_via= case to test hostvars method. # When no ansible_python_interpreter is set, executor/module_common.py chooses @@ -6,57 +7,118 @@ - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset tasks: - - meta: end_play - when: not is_mitogen + - include: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] +- hosts: tc-python-path-hostvar + vars: {mitogen_via: tc-python-path-unset} + tasks: + - include: ../_mitogen_only.yml + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/hostvar/path/to/python"] + # Non-localhost with explicit ansible_python_interpreter - hosts: tc-python-path-hostvar tasks: - - meta: end_play - when: not is_mitogen + - include: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: left: out.result[0].kwargs.python_path right: [/hostvar/path/to/python] +- hosts: tc-python-path-unset + vars: {mitogen_via: tc-python-path-hostvar} + tasks: + - include: ../_mitogen_only.yml + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/hostvar/path/to/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/usr/bin/python"] + # Implicit localhost gets ansible_python_interpreter=virtualenv interpreter - hosts: localhost tasks: - - meta: end_play - when: not is_mitogen + - include: ../_mitogen_only.yml + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + +- hosts: tc-python-path-unset + vars: {mitogen_via: localhost} + tasks: + - include: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: left: out.result[0].kwargs.python_path right: ["{{ansible_playbook_python}}"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["/usr/bin/python"] # explicit local connections get the same treatment as everything else. - hosts: tc-python-path-local-unset tasks: - - meta: end_play - when: not is_mitogen + - include: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] +- hosts: localhost + vars: {mitogen_via: tc-python-path-local-unset} + tasks: + - include: ../_mitogen_only.yml + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/usr/bin/python"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["{{ansible_playbook_python}}"] + +# explicit local connection with explicit interpreter - hosts: tc-python-path-local-explicit tasks: - - meta: end_play - when: not is_mitogen + - include: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: left: out.result[0].kwargs.python_path right: ["/a/b/c"] + +- hosts: localhost + vars: {mitogen_via: tc-python-path-local-explicit} + tasks: + - include: ../_mitogen_only.yml + - mitogen_get_stack: + register: out + - assert_equal: + left: out.result[0].kwargs.python_path + right: ["/a/b/c"] + - assert_equal: + left: out.result[1].kwargs.python_path + right: ["{{ansible_playbook_python}}"] From eae7b216732addfa42f6778cad815fd299252250 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 03:23:51 +0000 Subject: [PATCH 15/54] docs: sans-serif fonts, reduce shouty headings --- docs/_static/style.css | 51 ++++++++++++++++++++++++++++++++++++++++-- docs/ansible.rst | 2 +- docs/index.rst | 2 +- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/docs/_static/style.css b/docs/_static/style.css index ec25901f..0738fdb1 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -1,4 +1,51 @@ +body { + font-size: 100%; +} + +.sphinxsidebar { + font-size: 80% !important; +} + +.sphinxsidebar h3 { + font-size: 130% !important; +} + +.body h1 { font-size: 200% !important; } +.body h2 { font-size: 175% !important; } +.body h3 { font-size: 150% !important; } +.body h4 { font-size: 125% !important; } +.body h5 { font-size: 125% !important; } + +.body h1, +.body h2, +.body h3, +.body h4, +.body h5 { + margin-top: 20px !important; +} + +.body h1 { + margin-top: 0 !important; +} + +body, +.sphinxsidebar, +.sphinxsidebar h1, +.sphinxsidebar h2, +.sphinxsidebar h3, +.sphinxsidebar h4, +.sphinxsidebar h5, +.body h1, +.body h2, +.body h3, +.body h4, +.body h5 { + /*font-family: sans-serif !important;*/ + font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol !important; +} + + .document { width: 1000px !important; } @@ -38,10 +85,10 @@ div.body p, div.body dd, div.body li, div.body blockquote { width: 150px; } -.mitogen-right-200 { +.mitogen-right-180 { float: right; padding-left: 8px; - width: 200px; + width: 180px; } .mitogen-right-225 { diff --git a/docs/ansible.rst b/docs/ansible.rst index 17354755..e6e237d9 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -3,7 +3,7 @@ Mitogen for Ansible =================== .. image:: images/ansible/ansible_mitogen.svg - :class: mitogen-right-200 mitogen-logo-wrap + :class: mitogen-right-180 mitogen-logo-wrap An extension to `Ansible`_ is included that implements connections over Mitogen, replacing embedded shell invocations with pure-Python equivalents diff --git a/docs/index.rst b/docs/index.rst index 066d6716..6b5deb71 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,7 @@ Mitogen .. image:: images/mitogen.svg - :class: mitogen-right-200 mitogen-logo-wrap + :class: mitogen-right-180 mitogen-logo-wrap Mitogen is a Python library for writing distributed self-replicating programs. From d060f73f46e2c460ebcaf3fd3d233a87cec04563 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 03:28:20 +0000 Subject: [PATCH 16/54] docs: enable fixed_sidebar --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index abb6e97e..189a07c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,6 +17,7 @@ html_theme = 'alabaster' html_theme_options = { 'font_family': "Georgia, serif", 'head_font_family': "Georgia, serif", + 'fixed_sidebar': True, } htmlhelp_basename = 'mitogendoc' intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} From 5df8991744f56b0a8f2fe044d5b7bcb7d1542255 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 03:34:34 +0000 Subject: [PATCH 17/54] docs: tidy up footer and GitHub link --- docs/_templates/github.html | 6 +----- docs/conf.py | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/_templates/github.html b/docs/_templates/github.html index 12533b52..bb2b5ee5 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,8 +1,4 @@


-Star -

- -

-GitHub Repository +Star

diff --git a/docs/conf.py b/docs/conf.py index 189a07c4..f65f12dd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ html_theme_options = { 'font_family': "Georgia, serif", 'head_font_family': "Georgia, serif", 'fixed_sidebar': True, + 'show_powered_by': False, } htmlhelp_basename = 'mitogendoc' intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} From 45c3871d6fde22197fb675f877f461c75575f49f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 03:45:19 +0000 Subject: [PATCH 18/54] docs: tighter

margins, even less shouting, red headings --- docs/_static/style.css | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/_static/style.css b/docs/_static/style.css index 0738fdb1..667d7cf2 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -11,9 +11,19 @@ body { font-size: 130% !important; } +img + p, +h1 + p, +h2 + p, +h3 + p, +h4 + p, +h5 + p +{ + margin-top: 0; +} + .body h1 { font-size: 200% !important; } -.body h2 { font-size: 175% !important; } -.body h3 { font-size: 150% !important; } +.body h2 { font-size: 165% !important; } +.body h3 { font-size: 145% !important; } .body h4 { font-size: 125% !important; } .body h5 { font-size: 125% !important; } @@ -22,7 +32,8 @@ body { .body h3, .body h4, .body h5 { - margin-top: 20px !important; + margin-top: 25px !important; + color: #7f0000; } .body h1 { From 676f756153b0687cfca126ab121739ca9b0582dc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 04:10:34 +0000 Subject: [PATCH 19/54] docs: more margin tweaks for changelog --- docs/_static/style.css | 12 ++++++++---- docs/changelog.rst | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/_static/style.css b/docs/_static/style.css index 667d7cf2..98dd4bac 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -21,18 +21,22 @@ h5 + p margin-top: 0; } +.section > h3:first-child { + margin-top: 15px !important; +} + .body h1 { font-size: 200% !important; } .body h2 { font-size: 165% !important; } -.body h3 { font-size: 145% !important; } -.body h4 { font-size: 125% !important; } -.body h5 { font-size: 125% !important; } +.body h3 { font-size: 125% !important; } +.body h4 { font-size: 110% !important; font-weight: bold; } +.body h5 { font-size: 100% !important; font-weight: bold; } .body h1, .body h2, .body h3, .body h4, .body h5 { - margin-top: 25px !important; + margin-top: 30px !important; color: #7f0000; } diff --git a/docs/changelog.rst b/docs/changelog.rst index 635115e7..422e3eca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Release Notes From 1292b55fec85e6fad78f6c15bbadd2d6e450ac72 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 04:12:15 +0000 Subject: [PATCH 20/54] docs: add 'Fixes' heading --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 422e3eca..c3c3e8a4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -128,6 +128,9 @@ Core Library v0.2.5 (2019-02-1?) ------------------- +Fixes +~~~~~ + * `#511 `_, `#536 `_: changes in 0.2.4 to repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` From 10dcbaa4935481958db5f4d9e39edc7b6a481a9c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 04:17:02 +0000 Subject: [PATCH 21/54] docs: lighter pink --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index f65f12dd..3708a943 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,6 +19,8 @@ html_theme_options = { 'head_font_family': "Georgia, serif", 'fixed_sidebar': True, 'show_powered_by': False, + 'pink_2': 'fffafaf', + 'pink_1': '#fff0f0', } htmlhelp_basename = 'mitogendoc' intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} From 8fd641c4420bb3e4c3e5086998e9d03af2d1e44d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 22/54] tests/ansible: Spec.transport() test. --- tests/ansible/hosts/transport_config.hosts | 9 ++-- .../integration/transport_config/all.yml | 2 +- .../transport_config/python_path.yml | 30 ++++-------- .../transport_config/transport.yml | 48 +++++++++++++++++++ 4 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 tests/ansible/integration/transport_config/transport.yml diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index d9685811..ac046dd5 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,10 +1,13 @@ # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. + +# tansport() +tc-transport-unset +tc-transport-local ansible_connection=local + +# python_path() tc-python-path-unset tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python - -# local connections get virtualenv python path tc-python-path-local-unset ansible_connection=local tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c - diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 704760d1..cd3f7ac2 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,2 +1,2 @@ - - include: python_path.yml +- include: transport.yml diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index db6c873b..c5359e93 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -8,8 +8,7 @@ hosts: tc-python-path-unset tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] @@ -18,8 +17,7 @@ vars: {mitogen_via: tc-python-path-unset} tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] @@ -32,8 +30,7 @@ - hosts: tc-python-path-hostvar tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: [/hostvar/path/to/python] @@ -42,8 +39,7 @@ vars: {mitogen_via: tc-python-path-hostvar} tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/hostvar/path/to/python"] @@ -56,8 +52,7 @@ - hosts: localhost tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["{{ansible_playbook_python}}"] @@ -66,8 +61,7 @@ vars: {mitogen_via: localhost} tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["{{ansible_playbook_python}}"] @@ -80,8 +74,7 @@ - hosts: tc-python-path-local-unset tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] @@ -90,8 +83,7 @@ vars: {mitogen_via: tc-python-path-local-unset} tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/usr/bin/python"] @@ -104,8 +96,7 @@ - hosts: tc-python-path-local-explicit tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/a/b/c"] @@ -114,8 +105,7 @@ vars: {mitogen_via: tc-python-path-local-explicit} tasks: - include: ../_mitogen_only.yml - - mitogen_get_stack: - register: out + - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/a/b/c"] diff --git a/tests/ansible/integration/transport_config/transport.yml b/tests/ansible/integration/transport_config/transport.yml new file mode 100644 index 00000000..efedc8d4 --- /dev/null +++ b/tests/ansible/integration/transport_config/transport.yml @@ -0,0 +1,48 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_connection= is set, transport comes via ansible.cfg ("smart" +# is parsed away to either paramiko or ssh). +- name: integration/transport_config/transport.yml + hosts: tc-transport-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "ssh" + +- hosts: tc-transport-local + vars: {mitogen_via: tc-transport-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "ssh" + - assert_equal: + left: out.result[1].method + right: "local" + + +# ansible_connection=local + +- hosts: tc-transport-local + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "local" + +- hosts: tc-transport-unset + vars: {mitogen_via: tc-transport-local} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].method + right: "local" + - assert_equal: + left: out.result[1].method + right: "ssh" From 748f5f675de946e0ddc0f679ecebb21e50f4735d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 23/54] tests/ansible: Spec.remote_addr() test & mitogen_via= fix. ansible_ssh_host was not respected. --- ansible_mitogen/transport_config.py | 2 + tests/ansible/hosts/transport_config.hosts | 10 +- .../integration/transport_config/all.yml | 1 + .../transport_config/remote_addr.yml | 95 +++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/integration/transport_config/remote_addr.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 511cba5f..c9aa9860 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -447,7 +447,9 @@ class MitogenViaSpec(Spec): return self._inventory_name def remote_addr(self): + # play_context.py::MAGIC_VARIABLE_MAPPING return ( + self._host_vars.get('ansible_ssh_host') or self._host_vars.get('ansible_host') or self._inventory_name ) diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index ac046dd5..18cac116 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -3,7 +3,7 @@ # tansport() -tc-transport-unset +tc-transport-unset tc-transport-local ansible_connection=local # python_path() @@ -11,3 +11,11 @@ tc-python-path-unset tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python tc-python-path-local-unset ansible_connection=local tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c + +# remote_addr() +tc-remote-addr-unset # defaults to inventory_hostname +tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host +tc-remote-addr-explicit-host ansible_host=ansi.host +tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d + + diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index cd3f7ac2..ae86bb18 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,2 +1,3 @@ - include: python_path.yml +- include: remote_addr.yml - include: transport.yml diff --git a/tests/ansible/integration/transport_config/remote_addr.yml b/tests/ansible/integration/transport_config/remote_addr.yml new file mode 100644 index 00000000..b9887202 --- /dev/null +++ b/tests/ansible/integration/transport_config/remote_addr.yml @@ -0,0 +1,95 @@ + +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_host/ansible_ssh_host= is set, hostname is same as inventory +# name. +- name: integration/transport_config/remote_addr.yml + hosts: tc-remote-addr-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + +- hosts: tc-remote-addr-unset + vars: {mitogen_via: tc-remote-addr-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.ssh.host" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "tc-remote-addr-unset" + + +# ansible_ssh_host= + +- hosts: tc-remote-addr-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.ssh.host" + +- hosts: tc-remote-addr-explicit-ssh + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "ansi.ssh.host" + + +# ansible_host= + +- hosts: tc-remote-addr-explicit-host + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "ansi.host" + +- hosts: tc-remote-addr-explicit-host + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "ansi.host" + + +# both; ansible_ssh_host= takes precedence according to play_context.py. + +- hosts: tc-remote-addr-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "a.b.c" + +- hosts: tc-remote-addr-explicit-both + vars: {mitogen_via: tc-remote-addr-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.hostname + right: "tc-remote-addr-unset" + - assert_equal: + left: out.result[1].kwargs.hostname + right: "a.b.c" From 21ad299d7bf6d273a8f50d19bff2445775429bc6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 24/54] tests/ansible: Spec.remote_user() test & mitogen_via= fix. ansible_ssh_user precedence was incorrect. --- ansible_mitogen/transport_config.py | 2 +- tests/ansible/hosts/transport_config.hosts | 5 + .../integration/transport_config/all.yml | 1 + .../transport_config/remote_user.yml | 96 +++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/integration/transport_config/remote_user.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index c9aa9860..7af4dc55 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -456,8 +456,8 @@ class MitogenViaSpec(Spec): def remote_user(self): return ( - self._host_vars.get('ansible_user') or self._host_vars.get('ansible_ssh_user') or + self._host_vars.get('ansible_user') or C.DEFAULT_REMOTE_USER ) diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 18cac116..ab83c65d 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -18,4 +18,9 @@ tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host tc-remote-addr-explicit-host ansible_host=ansi.host tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d +# remote_addr() +tc-remote-user-unset # defaults to C.DEFAULT_REMOTE_USER +tc-remote-user-explicit-ssh ansible_ssh_user=ansi-ssh-user +tc-remote-user-explicit-user ansible_user=ansi-user +tc-remote-user-explicit-both ansible_user=a.b.c ansible_ssh_user=c.b.a diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index ae86bb18..618e72e3 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,3 +1,4 @@ - include: python_path.yml - include: remote_addr.yml +- include: remote_user.yml - include: transport.yml diff --git a/tests/ansible/integration/transport_config/remote_user.yml b/tests/ansible/integration/transport_config/remote_user.yml new file mode 100644 index 00000000..b873fcbe --- /dev/null +++ b/tests/ansible/integration/transport_config/remote_user.yml @@ -0,0 +1,96 @@ + +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_user/ansible_ssh_user= is set, username is +# C.DEFAULT_REMOTE_USER. +- name: integration/transport_config/remote_user.yml + hosts: tc-remote-user-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + # We set DEFAULT_REMOTE_USER in our ansible.cfg + right: "ansible-cfg-remote-user" + +- hosts: tc-remote-user-unset + vars: {mitogen_via: tc-remote-user-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-ssh-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansible-cfg-remote-user" + + +# ansible_ssh_user= + +- hosts: tc-remote-user-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-ssh-user" + +- hosts: tc-remote-user-explicit-ssh + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansi-ssh-user" + + +# ansible_user= + +- hosts: tc-remote-user-explicit-user + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansi-user" + +- hosts: tc-remote-user-explicit-host + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "ansi-user" + + +# both; ansible_ssh_user= takes precedence according to play_context.py. + +- hosts: tc-remote-user-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "c.b.a" + +- hosts: tc-remote-user-explicit-both + vars: {mitogen_via: tc-remote-user-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.username + right: "ansible-cfg-remote-user" + - assert_equal: + left: out.result[1].kwargs.username + right: "c.b.a" From d1cadf8ac8f2c95eeaf0943b7c7716ed1a951f9b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 25/54] tests/ansible: Spec.password() test, document interactive pw limitation. --- ansible_mitogen/connection.py | 1 + ansible_mitogen/transport_config.py | 26 ++++- docs/ansible.rst | 5 + docs/changelog.rst | 8 ++ tests/ansible/hosts/transport_config.hosts | 11 +-- .../integration/transport_config/all.yml | 1 + .../integration/transport_config/password.yml | 94 +++++++++++++++++++ 7 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 tests/ansible/integration/transport_config/password.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index b00430ca..7bae8c25 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -568,6 +568,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): via_vars = self.host_vars[inventory_name] return ansible_mitogen.transport_config.MitogenViaSpec( inventory_name=inventory_name, + play_context=self._play_context, host_vars=dict(via_vars), # TODO: make it lazy become_method=become_method or None, become_user=become_user or None, diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 7af4dc55..15694523 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -430,12 +430,33 @@ class MitogenViaSpec(Spec): having a configruation problem with connection delegation, the answer to your problem lies in the method implementations below! """ - def __init__(self, inventory_name, host_vars, - become_method, become_user): + def __init__(self, inventory_name, host_vars, become_method, become_user, + play_context): + """ + :param str inventory_name: + The inventory name of the intermediary machine, i.e. not the target + machine. + :param dict host_vars: + The HostVars magic dictionary provided by Ansible in task_vars. + :param str become_method: + If the mitogen_via= spec included a become method, the method it + specifies. + :param str become_user: + If the mitogen_via= spec included a become user, the user it + specifies. + :param PlayContext play_context: + For some global values **only**, the PlayContext used to describe + the real target machine. Values from this object are **strictly + restricted** to values that are Ansible-global, e.g. the passwords + specified interactively. + """ self._inventory_name = inventory_name self._host_vars = host_vars self._become_method = become_method self._become_user = become_user + # Dangerous! You may find a variable you want in this object, but it's + # almost certainly for the wrong machine! + self._dangerous_play_context = play_context def transport(self): return ( @@ -479,7 +500,6 @@ class MitogenViaSpec(Spec): def password(self): return optional_secret( - # TODO: Might have to come from PlayContext. self._host_vars.get('ansible_ssh_pass') or self._host_vars.get('ansible_password') ) diff --git a/docs/ansible.rst b/docs/ansible.rst index e6e237d9..f7788011 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -246,6 +246,11 @@ container. as duplicate connections between hops, due to not perfectly replicating the configuration Ansible would normally use for the intermediary. + * Intermediary machines cannot use login and become passwords that were + supplied to Ansible interactively. If an intermediary requires a + password, it must be supplied via ``ansible_ssh_pass``, + ``ansible_password``, or ``ansible_become_pass`` inventory variables. + * Automatic tunnelling of SSH-dependent actions, such as the ``synchronize`` module, is not yet supported. This will be added in the 0.3 series. diff --git a/docs/changelog.rst b/docs/changelog.rst index c3c3e8a4..245e87e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,6 +136,14 @@ Fixes repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. +* `748f5f67 `_: the + ``ansible_ssh_host`` parameter is respected when ``mitogen_via=`` is active. + +* `21ad299d `_: the + precedence of ``ansible_ssh_user`` and ``ansible_user`` was corrected when + when ``mitogen_via=`` is active. + + Thanks! ~~~~~~~ diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index ab83c65d..cb64d87b 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -18,9 +18,8 @@ tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host tc-remote-addr-explicit-host ansible_host=ansi.host tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d -# remote_addr() -tc-remote-user-unset # defaults to C.DEFAULT_REMOTE_USER -tc-remote-user-explicit-ssh ansible_ssh_user=ansi-ssh-user -tc-remote-user-explicit-user ansible_user=ansi-user -tc-remote-user-explicit-both ansible_user=a.b.c ansible_ssh_user=c.b.a - +# password() +tc-password-unset +tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass +tc-password-explicit-user ansible_password=ansi-pass +tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 618e72e3..a9880ffc 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,3 +1,4 @@ +- include: password.yml - include: python_path.yml - include: remote_addr.yml - include: remote_user.yml diff --git a/tests/ansible/integration/transport_config/password.yml b/tests/ansible/integration/transport_config/password.yml new file mode 100644 index 00000000..ac236d66 --- /dev/null +++ b/tests/ansible/integration/transport_config/password.yml @@ -0,0 +1,94 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# When no ansible_ssh_pass/ansible_password= is set, password comes via +# interactive input. +- name: integration/transport_config/password.yml + hosts: tc-password-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" # actually null, but assert_equal limitation + +- hosts: tc-password-unset + vars: {mitogen_via: tc-password-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-ssh-pass" + - assert_equal: + left: out.result[1].kwargs.password + right: "" + + +# ansible_ssh_user= + +- hosts: tc-password-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-ssh-pass" + +- hosts: tc-password-explicit-ssh + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "ansi-ssh-pass" + + +# ansible_user= + +- hosts: tc-password-explicit-pass + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "ansi-pass" + +- hosts: tc-password-explicit-pass + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "ansi-pass" + + +# both; ansible_ssh_user= takes precedence according to play_context.py. + +- hosts: tc-password-explicit-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "c.b.a" + +- hosts: tc-password-explicit-both + vars: {mitogen_via: tc-password-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert_equal: + left: out.result[0].kwargs.password + right: "" + - assert_equal: + left: out.result[1].kwargs.password + right: "c.b.a" From 8ba75d82ec96e4b3e2f15682cc2062f35d0c241a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 26/54] tests/ansible: Spec.become() test --- tests/ansible/hosts/transport_config.hosts | 4 ++ .../integration/transport_config/all.yml | 1 + .../integration/transport_config/become.yml | 68 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 tests/ansible/integration/transport_config/become.yml diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index cb64d87b..1546b26f 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -23,3 +23,7 @@ tc-password-unset tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass tc-password-explicit-user ansible_password=ansi-pass tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a + +# become() +tc-become-unset +tc-become-set diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index a9880ffc..f259f0a0 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,3 +1,4 @@ +- include: become.yml - include: password.yml - include: python_path.yml - include: remote_addr.yml diff --git a/tests/ansible/integration/transport_config/become.yml b/tests/ansible/integration/transport_config/become.yml new file mode 100644 index 00000000..baa2085e --- /dev/null +++ b/tests/ansible/integration/transport_config/become.yml @@ -0,0 +1,68 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# No become set. +- name: integration/transport_config/become.yml + hosts: tc-become-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + +- hosts: tc-become-unset + vars: {mitogen_via: becomeuser@tc-become-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "becomeuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-unset" + + +# Become set. +- name: integration/transport_config/become.yml + hosts: tc-become-set + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "becomeuser" + +- hosts: tc-become-set + vars: {mitogen_via: tc-become-unset} + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-unset" + - out.result[0].kwargs.username == "ansible-cfg-remote-user" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-set" + + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "becomeuser" From 4d33598fa069bd8d670b6b798eedd05c7309abe3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 16:15:52 +0000 Subject: [PATCH 27/54] setup.py: include LICENSE; closes #538. --- MANIFEST.in | 1 + docs/changelog.rst | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..1aba38f6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE diff --git a/docs/changelog.rst b/docs/changelog.rst index 245e87e6..7b1c7584 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,6 +136,9 @@ Fixes repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. +* `#538 `_: the Mitogen source + distribution now includes a requisite ``LICENSE`` file. + * `748f5f67 `_: the ``ansible_ssh_host`` parameter is respected when ``mitogen_via=`` is active. @@ -149,6 +152,7 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for bug reports, testing, features and fixes in this release contributed by +`Carl George `_, `Guy Knights `_, and `Josh Smift `_, From 8ae6ca1d5b83de17a83e943aa4028ed2f8f30bdd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 14:33:36 +0000 Subject: [PATCH 28/54] tests/ansible: Spec.become_method() test & mitogen_via= fix. ansible_become_method hostvar was not taken into account. --- ansible_mitogen/transport_config.py | 6 +- docs/changelog.rst | 4 +- tests/ansible/hosts/transport_config.hosts | 4 + .../integration/transport_config/all.yml | 1 + .../transport_config/become_method.yml | 83 +++++++++++++++++++ 5 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 tests/ansible/integration/transport_config/become_method.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 15694523..edc9ce1e 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -486,7 +486,11 @@ class MitogenViaSpec(Spec): return bool(self._become_user) def become_method(self): - return self._become_method or C.DEFAULT_BECOME_METHOD + return ( + self._become_method or + self._host_vars.get('ansible_become_method') or + C.DEFAULT_BECOME_METHOD + ) def become_user(self): return self._become_user diff --git a/docs/changelog.rst b/docs/changelog.rst index 7b1c7584..9ce16e57 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -143,8 +143,8 @@ Fixes ``ansible_ssh_host`` parameter is respected when ``mitogen_via=`` is active. * `21ad299d `_: the - precedence of ``ansible_ssh_user`` and ``ansible_user`` was corrected when - when ``mitogen_via=`` is active. + precedence of ``ansible_ssh_user`` and ``ansible_user`` was corrected when + when ``mitogen_via=`` is active. Thanks! diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 1546b26f..c98d4f41 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -27,3 +27,7 @@ tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a # become() tc-become-unset tc-become-set + +# become_method() +tc-become-method-unset +tc-become-method-su ansible_become_method=su diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index f259f0a0..908f6735 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,4 +1,5 @@ - include: become.yml +- include: become_method.yml - include: password.yml - include: python_path.yml - include: remote_addr.yml diff --git a/tests/ansible/integration/transport_config/become_method.yml b/tests/ansible/integration/transport_config/become_method.yml new file mode 100644 index 00000000..5129e5b8 --- /dev/null +++ b/tests/ansible/integration/transport_config/become_method.yml @@ -0,0 +1,83 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + + +# No become-method set. +- name: integration/transport_config/become-method.yml + hosts: tc-become-method-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + +- hosts: tc-become-method-unset + vars: {mitogen_via: becomeuser@tc-become-method-su} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "su" + - out.result[1].kwargs.username == "becomeuser" + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-method-unset" + + +# ansible_become_method=su +- hosts: tc-become-method-su + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "su" + - out.result[1].kwargs.username == "becomeuser" + +- hosts: tc-become-method-su + vars: {mitogen_via: tc-become-method-unset} + become: true + become_user: becomeuser + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-method-unset" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-method-su" + + - out.result[2].method == "su" + - out.result[2].kwargs.username == "becomeuser" + + + +# mitogen_via used to specify explicit become method +- hosts: tc-become-method-unset + vars: {mitogen_via: "doas:doasuser@tc-become-method-su"} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-method-su" + + - out.result[1].method == "doas" + - out.result[1].kwargs.username == "doasuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-method-unset" From 24c48c165d3de7bf94b4243cd04780799e341083 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 16:33:53 +0000 Subject: [PATCH 29/54] docs: update Changelog. --- docs/changelog.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ce16e57..f57ba5c4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -137,14 +137,18 @@ Fixes handling. Test coverage was added. * `#538 `_: the Mitogen source - distribution now includes a requisite ``LICENSE`` file. + distribution includes a requisite ``LICENSE`` file. * `748f5f67 `_: the - ``ansible_ssh_host`` parameter is respected when ``mitogen_via=`` is active. + ``ansible_ssh_host`` variable is respected when ``mitogen_via=`` is active. * `21ad299d `_: the - precedence of ``ansible_ssh_user`` and ``ansible_user`` was corrected when - when ``mitogen_via=`` is active. + precedence of ``ansible_ssh_user`` and ``ansible_user`` variables were + corrected when when ``mitogen_via=`` is active. + +* `8ae6ca1d `_: the + ``ansible_become_method`` variable is respected when ``mitogen_via=`` is + active. Thanks! From d45797b37020bd3e788ef5532d95eef0f8f49e49 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 12 Feb 2019 16:34:20 +0000 Subject: [PATCH 30/54] docs: duplicate word --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f57ba5c4..6e9db711 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -144,7 +144,7 @@ Fixes * `21ad299d `_: the precedence of ``ansible_ssh_user`` and ``ansible_user`` variables were - corrected when when ``mitogen_via=`` is active. + corrected when ``mitogen_via=`` is active. * `8ae6ca1d `_: the ``ansible_become_method`` variable is respected when ``mitogen_via=`` is From 9a69feb0c838ebc9ee15a385a68a1fc828418583 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 01:51:56 +0000 Subject: [PATCH 31/54] .github: add some more questions to issue template --- .github/ISSUE_TEMPLATE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index f1dc425c..c25697c6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,6 +4,10 @@ Please drag-drop large logs as text file attachments. Feel free to write an issue in your preferred format, however if in doubt, use the following checklist as a guide for what to include. +* Which version of Ansible are you running? +* Is your version of Ansible patched in any way? +* Are you running with any custom modules, or `module_utils` loaded? + * Have you tried the latest master version from Git? * Do you have some idea of what the underlying problem may be? https://mitogen.rtfd.io/en/stable/ansible.html#common-problems has From 7ff4e6694cefcfae3a3577915e7058d072a86609 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 02:16:49 +0000 Subject: [PATCH 32/54] issue #536: rework how 2.3-compatible simplejson is served Regardless of the version of simplejson loaded in the master, load up the ModuleResponder cache with our 2.4-compatible version. To cope with simplejson being loaded due to modules like ec2_group that try to import it before importing 'json', also update target.py to remove it from the whitelist if a local 'json' module import succeeds. --- ansible_mitogen/process.py | 39 +++++++++++++++++++++++++++++++++++--- ansible_mitogen/target.py | 5 +++++ mitogen/core.py | 5 +++++ mitogen/master.py | 13 +++++++++---- mitogen/parent.py | 4 ++-- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 8137af9c..32f4a012 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -236,6 +236,41 @@ class MuxProcess(object): if secs: mitogen.debug.dump_to_logger(secs=secs) + def _setup_simplejson(self, responder): + """ + We support serving simplejson for Python 2.4 targets on Ansible 2.3, at + least so the package's own CI Docker scripts can run without external + help, however newer versions of simplejson no longer support Python + 2.4. Therefore override any installed/loaded version with a + 2.4-compatible version we ship in the compat/ directory. + """ + responder.whitelist_prefix('simplejson') + + # issue #536: must be at end of sys.path, in case existing newer + # version is already loaded. + compat_path = os.path.join(os.path.dirname(__file__), 'compat') + sys.path.append(compat_path) + + for fullname, is_pkg, suffix in ( + (u'simplejson', True, '__init__.py'), + (u'simplejson.decoder', False, 'decoder.py'), + (u'simplejson.encoder', False, 'encoder.py'), + (u'simplejson.scanner', False, 'scanner.py'), + ): + path = os.path.join(compat_path, 'simplejson', suffix) + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + responder.add_source_override( + fullname=fullname, + path=path, + source=source, + is_pkg=is_pkg, + ) + def _setup_responder(self, responder): """ Configure :class:`mitogen.master.ModuleResponder` to only permit @@ -243,9 +278,7 @@ class MuxProcess(object): """ responder.whitelist_prefix('ansible') responder.whitelist_prefix('ansible_mitogen') - responder.whitelist_prefix('simplejson') - simplejson_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.insert(0, simplejson_path) + self._setup_simplejson(responder) # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 01877e34..58b715ce 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -377,6 +377,11 @@ def init_child(econtext, log_level, candidate_temp_dirs): LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) + # issue #536: if the json module is available, remove simplejson from the + # importer whitelist to avoid confusing certain Ansible modules. + if json.__name__ == 'json': + econtext.importer.whitelist.remove('simplejson') + global _fork_parent if FORK_SUPPORTED: mitogen.parent.upgrade_router(econtext) diff --git a/mitogen/core.py b/mitogen/core.py index a48e13ed..9f329d88 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1125,6 +1125,11 @@ class Importer(object): self.whitelist = list(whitelist) or [''] self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST + # Preserve copies of the original server-supplied whitelist/blacklist + # for later use by children. + self.master_whitelist = self.whitelist[:] + self.master_blacklist = self.blacklist[:] + # Presence of an entry in this map indicates in-flight GET_MODULE. self._callbacks = {} self._cache = {} diff --git a/mitogen/master.py b/mitogen/master.py index 257fb81b..18323844 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -485,8 +485,10 @@ class ModuleFinder(object): return path, source, is_pkg def _get_module_via_sys_modules(self, fullname): - """Attempt to fetch source code via sys.modules. This is specifically - to support __main__, but it may catch a few more cases.""" + """ + Attempt to fetch source code via sys.modules. This is specifically to + support __main__, but it may catch a few more cases. + """ module = sys.modules.get(fullname) LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if not isinstance(module, types.ModuleType): @@ -883,10 +885,13 @@ class ModuleResponder(object): if msg.is_dead: return - LOG.debug('%r._on_get_module(%r)', self, msg.data) - self.get_module_count += 1 stream = self._router.stream_by_id(msg.src_id) + if stream is None: + return + fullname = msg.data.decode() + LOG.debug('%s requested module %s', stream.name, fullname) + self.get_module_count += 1 if fullname in stream.sent_modules: LOG.warning('_on_get_module(): dup request for %r from %r', fullname, stream) diff --git a/mitogen/parent.py b/mitogen/parent.py index 91a4e5eb..814d20e3 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -2054,12 +2054,12 @@ class Router(mitogen.core.Router): def get_module_blacklist(self): if mitogen.context_id == 0: return self.responder.blacklist - return self.importer.blacklist + return self.importer.master_blacklist def get_module_whitelist(self): if mitogen.context_id == 0: return self.responder.whitelist - return self.importer.whitelist + return self.importer.master_whitelist def allocate_id(self): return self.id_allocator.allocate() From 4be4b085e3f00888780c5d1efacd024ac931aec6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 02:36:39 +0000 Subject: [PATCH 33/54] docs: update Changelog. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e9db711..b037ace1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,6 +136,11 @@ Fixes repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. +* `#536 `_: changes in 0.2.4 to + support Python 2.4 interacted poorly with modules that imported + ``simplejson`` from a controller that also loaded an incompatible newer + version of ``simplejson``. + * `#538 `_: the Mitogen source distribution includes a requisite ``LICENSE`` file. From 1c955a98768c07de68bb6705f8f4dc1c5afeea86 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 03:36:39 +0000 Subject: [PATCH 34/54] ansible: capture stderr stream of async tasks. Closes #540. --- ansible_mitogen/target.py | 2 +- docs/changelog.rst | 3 +++ tests/ansible/integration/async/result_shell_echo_hi.yml | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 58b715ce..e891730e 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -502,7 +502,7 @@ class AsyncRunner(object): ) result = json.loads(filtered) result.setdefault('warnings', []).extend(warnings) - result['stderr'] = dct['stderr'] + result['stderr'] = dct['stderr'] or result.get('stderr', '') self._update(result) def _run(self): diff --git a/docs/changelog.rst b/docs/changelog.rst index b037ace1..e7ce61c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -144,6 +144,9 @@ Fixes * `#538 `_: the Mitogen source distribution includes a requisite ``LICENSE`` file. +* `#540 `_: the ``stderr`` stream of + async module invocations was previously discarded. + * `748f5f67 `_: the ``ansible_ssh_host`` variable is respected when ``mitogen_via=`` is active. diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index c2d2dc42..e1068587 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -5,7 +5,7 @@ any_errors_fatal: true tasks: - - shell: echo hi + - shell: echo hi; echo there >&2 async: 100 poll: 0 register: job @@ -21,10 +21,10 @@ - assert: that: - async_out.changed == True - - async_out.cmd == "echo hi" + - async_out.cmd == "echo hi; echo there >&2" - 'async_out.delta.startswith("0:00:")' - async_out.end.startswith("20") - - async_out.invocation.module_args._raw_params == "echo hi" + - async_out.invocation.module_args._raw_params == "echo hi; echo there >&2" - async_out.invocation.module_args._uses_shell == True - async_out.invocation.module_args.chdir == None - async_out.invocation.module_args.creates == None @@ -33,7 +33,7 @@ - async_out.invocation.module_args.warn == True - async_out.rc == 0 - async_out.start.startswith("20") - - async_out.stderr == "" + - async_out.stderr == "there" - async_out.stdout == "hi" vars: async_out: "{{result.content|b64decode|from_json}}" From ae5a471e312b9a71a847c699f70a176174994244 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 03:48:56 +0000 Subject: [PATCH 35/54] issue #539: disable logger propagation. --- ansible_mitogen/logging.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 97832938..e358bb53 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -75,25 +75,28 @@ class Handler(logging.Handler): def setup(): """ - Install a handler for Mitogen's logger to redirect it into the Ansible - display framework, and prevent propagation to the root logger. + Install handlers for Mitogen loggers to redirect them into the Ansible + display framework. Ansible installs its own logging framework handlers when + C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers. """ - logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)] - mitogen.core.LOG.handlers = [Handler(display.vvv)] - mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] - mitogen.core.IOLOG.propagate = False + l_mitogen = logging.getLogger('mitogen') + l_mitogen_io = logging.getLogger('mitogen.io') + l_ansible_mitogen = logging.getLogger('ansible_mitogen') + + for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen: + logger.handlers = [Handler(display.vvv)] + logger.propagate = False if display.verbosity > 2: - mitogen.core.LOG.setLevel(logging.DEBUG) - logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) + l_ansible_mitogen.setLevel(logging.DEBUG) + l_mitogen.setLevel(logging.DEBUG) else: # Mitogen copies the active log level into new children, allowing them # to filter tiny messages before they hit the network, and therefore # before they wake the IO loop. Explicitly setting INFO saves ~4% # running against just the local machine. - mitogen.core.LOG.setLevel(logging.ERROR) - logging.getLogger('ansible_mitogen').setLevel(logging.ERROR) + l_mitogen.setLevel(logging.ERROR) + l_ansible_mitogen.setLevel(logging.ERROR) if display.verbosity > 3: - mitogen.core.IOLOG.setLevel(logging.DEBUG) - logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) + l_mitogen_io.setLevel(logging.DEBUG) From 968b8268001334d70a9e9b0598e3b486083437d5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 03:49:59 +0000 Subject: [PATCH 36/54] docs: update Changelog; closes #539. --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e7ce61c3..decff2de 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -144,6 +144,9 @@ Fixes * `#538 `_: the Mitogen source distribution includes a requisite ``LICENSE`` file. +* `#539 `_: log output is no longer + duplicated when the Ansible ``log_path`` setting is enabled. + * `#540 `_: the ``stderr`` stream of async module invocations was previously discarded. From 7b3ed52fd82ecc1fb65143484eb5d5467eb6dd8e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 04:04:54 +0000 Subject: [PATCH 37/54] tests/ansible: Spec.become_user() test. --- tests/ansible/hosts/transport_config.hosts | 4 + .../integration/transport_config/all.yml | 1 + .../transport_config/become_user.yml | 106 ++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 tests/ansible/integration/transport_config/become_user.yml diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index c98d4f41..76998e26 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -31,3 +31,7 @@ tc-become-set # become_method() tc-become-method-unset tc-become-method-su ansible_become_method=su + +# become_user() +tc-become-user-unset +tc-become-user-set ansible_become_user=ansi-become-user diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 908f6735..47409b19 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,5 +1,6 @@ - include: become.yml - include: become_method.yml +- include: become_user.yml - include: password.yml - include: python_path.yml - include: remote_addr.yml diff --git a/tests/ansible/integration/transport_config/become_user.yml b/tests/ansible/integration/transport_config/become_user.yml new file mode 100644 index 00000000..43cbca2a --- /dev/null +++ b/tests/ansible/integration/transport_config/become_user.yml @@ -0,0 +1,106 @@ +# Each case is followed by mitogen_via= case to test hostvars user. + + +# No become-user set, defaults to "root" +- name: integration/transport_config/become-user.yml + hosts: tc-become-user-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "root" + +# Not set, unbecoming mitogen_via= +- hosts: tc-become-user-unset + become: true + vars: {mitogen_via: tc-become-user-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "ssh" + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "root" + +# Not set, becoming mitogen_via= +- hosts: tc-become-user-unset + become: true + vars: {mitogen_via: viauser@tc-become-user-set} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "viauser" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.username == "root" + + +# ansible_become_user= set. +- hosts: tc-become-user-set + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.username == "ansi-become-user" + + +# ansible_become_user=, unbecoming via= +- hosts: tc-become-user-set + vars: {mitogen_via: tc-become-user-unset} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-user-unset" + + - out.result[1].method == "ssh" + - out.result[1].kwargs.hostname == "tc-become-user-set" + + - out.result[2].method == "sudo" + - out.result[2].kwargs.username == "ansi-become-user" + + +# ansible_become_user=, becoming via= +- hosts: tc-become-user-set + vars: {mitogen_via: "doas:doasuser@tc-become-user-unset"} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[0].kwargs.hostname == "tc-become-user-unset" + + - out.result[1].method == "doas" + - out.result[1].kwargs.username == "doasuser" + + - out.result[2].method == "ssh" + - out.result[2].kwargs.hostname == "tc-become-user-set" + + - out.result[3].method == "sudo" + - out.result[3].kwargs.username == "ansi-become-user" + From 2d536b49eb99901311bf1aa13ed12a488d4bab61 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 16:02:31 +0000 Subject: [PATCH 38/54] docs: remove a little more top margin wastage --- docs/_static/style.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/style.css b/docs/_static/style.css index 98dd4bac..456473a9 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -3,6 +3,10 @@ body { font-size: 100%; } +.sphinxsidebarwrapper { + padding-top: 0 !important; +} + .sphinxsidebar { font-size: 80% !important; } From 023b44b1292485f24c8e55148162eeef02f5a5b3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 16:03:08 +0000 Subject: [PATCH 39/54] docs: remove top "Table of Contents" link --- docs/_templates/globaltoc.html | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/_templates/globaltoc.html diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html new file mode 100644 index 00000000..76accef7 --- /dev/null +++ b/docs/_templates/globaltoc.html @@ -0,0 +1 @@ +{{ toctree() }} From b5b23e8f3d53c8361053152ae49bc8adcd8d5296 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 04:04:54 +0000 Subject: [PATCH 40/54] tests/ansible: Spec.become_pass() test. --- ansible_mitogen/transport_config.py | 1 - tests/ansible/hosts/transport_config.hosts | 6 + .../integration/transport_config/all.yml | 3 +- .../transport_config/become_pass.yml | 142 ++++++++++++++++++ 4 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 tests/ansible/integration/transport_config/become_pass.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index edc9ce1e..3f827744 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -497,7 +497,6 @@ class MitogenViaSpec(Spec): def become_pass(self): return optional_secret( - # TODO: Might have to come from PlayContext. self._host_vars.get('ansible_become_password') or self._host_vars.get('ansible_become_pass') ) diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 76998e26..709d26f7 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -35,3 +35,9 @@ tc-become-method-su ansible_become_method=su # become_user() tc-become-user-unset tc-become-user-set ansible_become_user=ansi-become-user + +# become_pass() +tc-become-pass-unset +tc-become-pass-password ansible_become_password=apassword +tc-become-pass-pass ansible_become_pass=apass +tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 47409b19..23423a9d 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,6 +1,7 @@ -- include: become.yml - include: become_method.yml +- include: become_pass.yml - include: become_user.yml +- include: become.yml - include: password.yml - include: python_path.yml - include: remote_addr.yml diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml new file mode 100644 index 00000000..02c6528d --- /dev/null +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -0,0 +1,142 @@ +# Each case is followed by mitogen_via= case to test hostvars pass. + + +# No become-pass set, defaults to "root" +- name: integration/transport_config/become-pass.yml + hosts: tc-become-pass-unset + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == None + +# Not set, unbecoming mitogen_via= +- hosts: tc-become-pass-unset + become: true + vars: {mitogen_via: tc-become-pass-password} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "ssh" + - out.result[2].method == "sudo" + - out.result[2].kwargs.password == None + +# Not set, becoming mitogen_via= +- hosts: tc-become-pass-unset + become: true + vars: {mitogen_via: viapass@tc-become-pass-password} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == None + + +# ansible_become_password= set. +- hosts: tc-become-pass-password + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + + +# ansible_become_password=, via= +- hosts: tc-become-pass-password + vars: {mitogen_via: root@tc-become-pass-pass} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apass" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == "apassword" + + +# ansible_become_pass= +- hosts: tc-become-pass-pass + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apass" + + +# ansible_become_pass=, via= +- hosts: tc-become-pass-pass + vars: {mitogen_via: root@tc-become-pass-password} + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 4 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "apassword" + - out.result[2].method == "ssh" + - out.result[3].method == "sudo" + - out.result[3].kwargs.password == "apass" + + + +# ansible_become_pass & ansible_become_password set, password takes precedence +- hosts: tc-become-pass-both + become: true + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "a.b.c" + + +# both, mitogen_via +- hosts: tc-become-pass-unset + vars: {mitogen_via: root@tc-become-pass-both} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 3 + - out.result[0].method == "ssh" + - out.result[1].method == "sudo" + - out.result[1].kwargs.password == "a.b.c" + - out.result[2].method == "ssh" From 1f77d24becf2ccd20b062ecc6027e682cd28e63c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 16:16:49 +0000 Subject: [PATCH 41/54] Update copyright year everywhere. --- LICENSE | 2 +- ansible_mitogen/affinity.py | 2 +- ansible_mitogen/connection.py | 2 +- ansible_mitogen/loaders.py | 2 +- ansible_mitogen/logging.py | 2 +- ansible_mitogen/mixins.py | 2 +- ansible_mitogen/module_finder.py | 2 +- ansible_mitogen/parsing.py | 2 +- ansible_mitogen/planner.py | 2 +- ansible_mitogen/plugins/action/mitogen_get_stack.py | 2 +- ansible_mitogen/plugins/connection/mitogen_doas.py | 2 +- ansible_mitogen/plugins/connection/mitogen_docker.py | 2 +- ansible_mitogen/plugins/connection/mitogen_jail.py | 2 +- ansible_mitogen/plugins/connection/mitogen_local.py | 2 +- ansible_mitogen/plugins/connection/mitogen_lxc.py | 2 +- ansible_mitogen/plugins/connection/mitogen_lxd.py | 2 +- ansible_mitogen/plugins/connection/mitogen_machinectl.py | 2 +- ansible_mitogen/plugins/connection/mitogen_setns.py | 2 +- ansible_mitogen/plugins/connection/mitogen_ssh.py | 2 +- ansible_mitogen/plugins/connection/mitogen_su.py | 2 +- ansible_mitogen/plugins/connection/mitogen_sudo.py | 2 +- ansible_mitogen/plugins/strategy/mitogen.py | 2 +- ansible_mitogen/plugins/strategy/mitogen_free.py | 2 +- ansible_mitogen/plugins/strategy/mitogen_host_pinned.py | 2 +- ansible_mitogen/plugins/strategy/mitogen_linear.py | 2 +- ansible_mitogen/process.py | 2 +- ansible_mitogen/runner.py | 2 +- ansible_mitogen/services.py | 2 +- ansible_mitogen/strategy.py | 2 +- ansible_mitogen/target.py | 2 +- ansible_mitogen/transport_config.py | 2 +- mitogen/__init__.py | 2 +- mitogen/core.py | 2 +- mitogen/debug.py | 2 +- mitogen/doas.py | 2 +- mitogen/docker.py | 2 +- mitogen/fakessh.py | 2 +- mitogen/fork.py | 2 +- mitogen/jail.py | 2 +- mitogen/lxc.py | 2 +- mitogen/lxd.py | 2 +- mitogen/master.py | 2 +- mitogen/parent.py | 2 +- mitogen/profiler.py | 2 +- mitogen/select.py | 2 +- mitogen/service.py | 2 +- mitogen/setns.py | 2 +- mitogen/ssh.py | 2 +- mitogen/su.py | 2 +- mitogen/sudo.py | 2 +- mitogen/unix.py | 2 +- mitogen/utils.py | 2 +- setup.py | 2 +- 53 files changed, 53 insertions(+), 53 deletions(-) diff --git a/LICENSE b/LICENSE index 61d21fee..70e43a94 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2017, David Wilson +Copyright 2019, David Wilson Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index d7ae45a6..bcab89af 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 7bae8c25..254a4286 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 08c59278..ff06c0c5 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index e358bb53..e2035254 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 7a180952..5f51cc6f 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 56e8b82e..633e3cad 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/parsing.py b/ansible_mitogen/parsing.py index fa79282a..525e60cf 100644 --- a/ansible_mitogen/parsing.py +++ b/ansible_mitogen/parsing.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index f3e4500e..3c5bd64f 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/action/mitogen_get_stack.py b/ansible_mitogen/plugins/action/mitogen_get_stack.py index ed7520cf..12afbfba 100644 --- a/ansible_mitogen/plugins/action/mitogen_get_stack.py +++ b/ansible_mitogen/plugins/action/mitogen_get_stack.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_doas.py b/ansible_mitogen/plugins/connection/mitogen_doas.py index 873b0d9d..1113d7c6 100644 --- a/ansible_mitogen/plugins/connection/mitogen_doas.py +++ b/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index 5904c83e..b71ef5f1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index fb7bce54..c7475fb1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index fcd9c030..24b84a03 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index ce394102..696c9abd 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 77efe6c1..95e692a0 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index 9b332a3f..0f5a0d28 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 23f62135..20c6f137 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index dbaba407..df0e87cb 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_su.py b/ansible_mitogen/plugins/connection/mitogen_su.py index 104a7190..4ab2711e 100644 --- a/ansible_mitogen/plugins/connection/mitogen_su.py +++ b/ansible_mitogen/plugins/connection/mitogen_su.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py index 367dd61b..130f5445 100644 --- a/ansible_mitogen/plugins/connection/mitogen_sudo.py +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index f8608745..66872663 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index d3b1cdc6..ffe2fbd9 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py index 175e1f8b..23eccd36 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py +++ b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index 51b03096..1b198e61 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 32f4a012..d7f36496 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 768cc57c..04c70e78 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 61286382..a7c0e46f 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 4d1636e2..50486841 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index e891730e..809165da 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 3f827744..22e82cc6 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 26e48aff..060430c6 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/core.py b/mitogen/core.py index 9f329d88..920a94b6 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/debug.py b/mitogen/debug.py index 8f290c4d..3d13347f 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/doas.py b/mitogen/doas.py index 250b6faf..1b687fb2 100644 --- a/mitogen/doas.py +++ b/mitogen/doas.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/docker.py b/mitogen/docker.py index 074f0e90..0c0d40e7 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index 2f2726eb..d39a710d 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/fork.py b/mitogen/fork.py index 081f7e3d..c78558b8 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/jail.py b/mitogen/jail.py index fade8cbb..6e0ac68b 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 6d4acba6..879d19a1 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/lxd.py b/mitogen/lxd.py index 7de4903a..faea2561 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/master.py b/mitogen/master.py index 18323844..1396f4e1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/parent.py b/mitogen/parent.py index 814d20e3..7e567aaa 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/profiler.py b/mitogen/profiler.py index 10ec6086..74bbdb23 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/select.py b/mitogen/select.py index 6b87e671..fd2cbe9a 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/service.py b/mitogen/service.py index c67b35e8..0fba59c9 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/setns.py b/mitogen/setns.py index d38aa092..b1d69783 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 47c90fff..11b74c1b 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/su.py b/mitogen/su.py index 7eff60a6..5ff9e177 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 05a04989..868d4d76 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/unix.py b/mitogen/unix.py index 3e315d6f..66141eec 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/mitogen/utils.py b/mitogen/utils.py index 6c56d6d5..94a171fb 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -1,4 +1,4 @@ -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: diff --git a/setup.py b/setup.py index 6f31133d..c3257996 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright 2017, David Wilson +# Copyright 2019, David Wilson # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: From 7fd0d349105e9bcefeed530265661ffeb682e97b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 04:04:54 +0000 Subject: [PATCH 42/54] tests/ansible: Spec.port() test & mitogen_via= fix. ansible_ssh_port was not respected. --- ansible_mitogen/transport_config.py | 1 + tests/ansible/hosts/transport_config.hosts | 6 ++ .../integration/transport_config/all.yml | 1 + .../integration/transport_config/port.yml | 101 ++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 tests/ansible/integration/transport_config/port.yml diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 22e82cc6..8ef12165 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -509,6 +509,7 @@ class MitogenViaSpec(Spec): def port(self): return ( + self._host_vars.get('ansible_ssh_port') or self._host_vars.get('ansible_port') or C.DEFAULT_REMOTE_PORT ) diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 709d26f7..d68b2d84 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -41,3 +41,9 @@ tc-become-pass-unset tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a + +# port() +tc-port-unset +tc-port-explicit-port ansible_port=1234 +tc-port-explicit-ssh ansible_ssh_port=4321 +tc-port-both ansible_port=1717 ansible_ssh_port=1532 diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 23423a9d..64199314 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -3,6 +3,7 @@ - include: become_user.yml - include: become.yml - include: password.yml +- include: port.yml - include: python_path.yml - include: remote_addr.yml - include: remote_user.yml diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml new file mode 100644 index 00000000..2781081a --- /dev/null +++ b/tests/ansible/integration/transport_config/port.yml @@ -0,0 +1,101 @@ +# Each case is followed by mitogen_via= case to test hostvars pass. + + +# No port set +- name: integration/transport_config/port.yml + hosts: tc-port-unset + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == None + +# Not set, mitogen_via= +- hosts: tc-port-explicit-ssh + vars: {mitogen_via: tc-port-unset} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == None + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == 4321 + +# ansible_ssh_port= +- hosts: tc-port-explicit-ssh + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 4321 + +- hosts: tc-port-explicit-unset + vars: {mitogen_via: tc-port-explicit-ssh} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[1].kwargs.port == 4321 + - out.result[1].method == "ssh" + - out.result[0].kwargs.port == None + +# ansible_port= +- hosts: tc-port-explicit-port + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1234 + +- hosts: tc-port-unset + vars: {mitogen_via: tc-port-explicit-port} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1234 + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == None + + +# both, ssh takes precedence +- hosts: tc-port-both + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1532 + +- hosts: tc-port-unset + vars: {mitogen_via: tc-port-both} + tasks: + - include: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result|length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.port == 1532 + - out.result[1].method == "ssh" + - out.result[1].kwargs.port == None From c39ee9b7fe84248cf2a20842f4ef63a309b2b2cb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 16:33:22 +0000 Subject: [PATCH 43/54] docs: update Changelog. --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index decff2de..84162bf8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -161,6 +161,9 @@ Fixes ``ansible_become_method`` variable is respected when ``mitogen_via=`` is active. +* `7fd0d349 `_: the + ``ansible_ssh_port`` variable is respected when ``mitogen_via=`` is active. + Thanks! ~~~~~~~ From 0f30808234c4974cb13e8eab718a9dec8b40605a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 17:28:01 +0000 Subject: [PATCH 44/54] ansible: quiesce boto logger; closes #541. --- ansible_mitogen/logging.py | 3 ++- docs/changelog.rst | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index e2035254..1c439be8 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -54,7 +54,8 @@ class Handler(logging.Handler): #: may simply be to bury all target logs in DEBUG output, but not by #: overriding their log level as done here. NOISY_LOGGERS = frozenset([ - 'dnf', # issue #272; warns when a package is already installed. + 'dnf', # issue #272; warns when a package is already installed. + 'boto', # issue #541; normal boto retry logic can cause ERROR logs. ]) def emit(self, record): diff --git a/docs/changelog.rst b/docs/changelog.rst index 84162bf8..8d246e98 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -150,6 +150,11 @@ Fixes * `#540 `_: the ``stderr`` stream of async module invocations was previously discarded. +* `#541 `_: Python error logs + originating from the ``boto`` package are quiesced, and appear only in + ``-vvv`` output. This is since many EC2 modules triggers errors during normal + operation, in the process of retrying transiently failing requests. + * `748f5f67 `_: the ``ansible_ssh_host`` variable is respected when ``mitogen_via=`` is active. From b317b66317b6f01447897cebcc54018c530b4dab Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 17:29:47 +0000 Subject: [PATCH 45/54] issue #541: changelog typos. --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d246e98..800ac164 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -152,8 +152,8 @@ Fixes * `#541 `_: Python error logs originating from the ``boto`` package are quiesced, and appear only in - ``-vvv`` output. This is since many EC2 modules triggers errors during normal - operation, in the process of retrying transiently failing requests. + ``-vvv`` output. This is since EC2 modules may trigger errors during normal + operation, when retrying transiently failing requests. * `748f5f67 `_: the ``ansible_ssh_host`` variable is respected when ``mitogen_via=`` is active. From 1b4eb06f72d26bf6604ce17d1109c589840c3a8c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 17:35:42 +0000 Subject: [PATCH 46/54] docs: more concise Changelog. --- docs/changelog.rst | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 800ac164..81d2a654 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -155,19 +155,13 @@ Fixes ``-vvv`` output. This is since EC2 modules may trigger errors during normal operation, when retrying transiently failing requests. -* `748f5f67 `_: the - ``ansible_ssh_host`` variable is respected when ``mitogen_via=`` is active. - -* `21ad299d `_: the - precedence of ``ansible_ssh_user`` and ``ansible_user`` variables were - corrected when ``mitogen_via=`` is active. - -* `8ae6ca1d `_: the - ``ansible_become_method`` variable is respected when ``mitogen_via=`` is - active. - -* `7fd0d349 `_: the - ``ansible_ssh_port`` variable is respected when ``mitogen_via=`` is active. +* `748f5f67 `_, + `21ad299d `_, + `8ae6ca1d `_, + `7fd0d349 `_: + the ``ansible_ssh_host``, ``ansible_ssh_user``, ``ansible_user``, + ``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly + match typical behaviour when ``mitogen_via=`` is active. Thanks! From d4c0250083be59176ee06c060bf395702200109c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 21:25:45 +0000 Subject: [PATCH 47/54] issue #532: PushFileService race. There has always been a race in PushFileService since given a parent asked to forward modules to two children via some intermediary: interm = router.local() c1 = router.local(via=interm) c2 = router.local(via=interm) service.propagate_to(c1, 'foo/bar.py') service.propagate_to(c2, 'foo/bar.py') Two calls will be emitted to 'interm': PushFileService.store_and_forward(c1, 'foo/bar.py', [blob]) PushFileService.store(c2, 'foo/bar.py') Which will be processed in-order up to the point where service pool threads in 'interm' are woken to process the message. While it is guaranteed store_and_forward() will be processed first, no guarantee existed that its assigned pool thread would wake and take _lock first, thus it was possible for forward() to win the race, and for a request to arrive to forward a file that had not been placed in local cache yet. Here we get rid of SerializedInvoker entirely, as it is partially to blame for hiding the race: SerializedInvoker can only ensure no two messages are processed simultaneously, it cannot ensure the messages are processed in their intended order. Instead, teach forward() that it may be called before store_and_forward(), and if that is the case, to place the forward request on to _waiters alongside any local threads blocked in get(). --- mitogen/service.py | 60 +++++++++++++++++++-------------- tests/push_file_service_test.py | 56 ++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 tests/push_file_service_test.py diff --git a/mitogen/service.py b/mitogen/service.py index 0fba59c9..3254e69a 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -603,8 +603,6 @@ class PushFileService(Service): This service will eventually be merged into FileService. """ - invoker_class = SerializedInvoker - def __init__(self, **kwargs): super(PushFileService, self).__init__(**kwargs) self._lock = threading.Lock() @@ -613,13 +611,16 @@ class PushFileService(Service): self._sent_by_stream = {} def get(self, path): + """ + Fetch a file from the cache. + """ assert isinstance(path, mitogen.core.UnicodeType) self._lock.acquire() try: if path in self._cache: return self._cache[path] - waiters = self._waiters.setdefault(path, []) latch = mitogen.core.Latch() + waiters = self._waiters.setdefault(path, []) waiters.append(lambda: latch.put(None)) finally: self._lock.release() @@ -633,14 +634,15 @@ class PushFileService(Service): stream = self.router.stream_by_id(context.context_id) child = mitogen.core.Context(self.router, stream.remote_id) sent = self._sent_by_stream.setdefault(stream, set()) - if path in sent and child.context_id != context.context_id: - child.call_service_async( - service_name=self.name(), - method_name='forward', - path=path, - context=context - ).close() - elif path not in sent: + if path in sent: + if child.context_id != context.context_id: + child.call_service_async( + service_name=self.name(), + method_name='forward', + path=path, + context=context + ).close() + else: child.call_service_async( service_name=self.name(), method_name='store_and_forward', @@ -680,14 +682,6 @@ class PushFileService(Service): fp.close() self._forward(context, path) - def _store(self, path, data): - self._lock.acquire() - try: - self._cache[path] = data - return self._waiters.pop(path, []) - finally: - self._lock.release() - @expose(policy=AllowParents()) @no_reply() @arg_spec({ @@ -696,9 +690,16 @@ class PushFileService(Service): 'context': mitogen.core.Context, }) def store_and_forward(self, path, data, context): - LOG.debug('%r.store_and_forward(%r, %r, %r)', - self, path, data, context) - waiters = self._store(path, data) + LOG.debug('%r.store_and_forward(%r, %r, %r) %r', + self, path, data, context, + threading.currentThread().getName()) + self._lock.acquire() + try: + self._cache[path] = data + waiters = self._waiters.pop(path, []) + finally: + self._lock.release() + if context.context_id != mitogen.context_id: self._forward(context, path) for callback in waiters: @@ -712,10 +713,17 @@ class PushFileService(Service): }) def forward(self, path, context): LOG.debug('%r.forward(%r, %r)', self, path, context) - if path not in self._cache: - LOG.error('%r: %r is not in local cache', self, path) - return - self._forward(context, path) + func = lambda: self._forward(context, path) + + self._lock.acquire() + try: + if path in self._cache: + func() + else: + LOG.debug('%r: %r not cached yet, queueing', self, path) + self._waiters.setdefault(path, []).append(func) + finally: + self._lock.release() class FileService(Service): diff --git a/tests/push_file_service_test.py b/tests/push_file_service_test.py new file mode 100644 index 00000000..1dfff241 --- /dev/null +++ b/tests/push_file_service_test.py @@ -0,0 +1,56 @@ + +import os +import tempfile +import unittest2 + +import mitogen.core +import mitogen.service +import testlib +from mitogen.core import b + + +def prepare(): + # ensure module loading delay is complete before loading PushFileService. + pass + + +@mitogen.core.takes_router +def wait_for_file(path, router): + pool = mitogen.service.get_or_create_pool(router=router) + service = pool.get_service(u'mitogen.service.PushFileService') + return service.get(path) + + +class PropagateToTest(testlib.RouterMixin, testlib.TestCase): + klass = mitogen.service.PushFileService + + def test_two_grandchild_one_intermediary(self): + tf = tempfile.NamedTemporaryFile() + path = mitogen.core.to_text(tf.name) + + try: + tf.write(b('test')) + tf.flush() + + interm = self.router.local(name='interm') + c1 = self.router.local(via=interm, name='c1') + c2 = self.router.local(via=interm) + + c1.call(prepare) + c2.call(prepare) + + service = self.klass(router=self.router) + service.propagate_to(context=c1, path=path) + service.propagate_to(context=c2, path=path) + + s = c1.call(wait_for_file, path=path) + self.assertEquals(b('test'), s) + + s = c2.call(wait_for_file, path=path) + self.assertEquals(b('test'), s) + finally: + tf.close() + + +if __name__ == '__main__': + unittest2.main() From cf8ecf19b7b07f0c8f74447e1041934eabd7f62c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 21:32:47 +0000 Subject: [PATCH 48/54] docs: update Changelog; closes #532. --- docs/changelog.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81d2a654..c30c8f3c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,13 +136,17 @@ Fixes repair ``delegate_to`` handling broke default ``ansible_python_interpreter`` handling. Test coverage was added. +* `#532 `_: fix a race in the service + used to propagate Ansible modules, that could easily manifest when starting + asynchronous tasks in a loop. + * `#536 `_: changes in 0.2.4 to support Python 2.4 interacted poorly with modules that imported ``simplejson`` from a controller that also loaded an incompatible newer version of ``simplejson``. -* `#538 `_: the Mitogen source - distribution includes a requisite ``LICENSE`` file. +* `#538 `_: the source distribution + includes a ``LICENSE`` file. * `#539 `_: log output is no longer duplicated when the Ansible ``log_path`` setting is enabled. @@ -151,7 +155,7 @@ Fixes async module invocations was previously discarded. * `#541 `_: Python error logs - originating from the ``boto`` package are quiesced, and appear only in + originating from the ``boto`` package are quiesced, and only appear in ``-vvv`` output. This is since EC2 modules may trigger errors during normal operation, when retrying transiently failing requests. From 2a8567b4327bd7d67eb835e71a7d7af2ec564cc3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 22:10:09 +0000 Subject: [PATCH 49/54] core: serialize calls to _service_stub_main(). See comment. --- mitogen/core.py | 17 ++++++++++++++--- mitogen/fork.py | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 920a94b6..470b00ca 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3136,10 +3136,21 @@ class ExternalContext(object): if not self.config['profiling']: os.kill(os.getpid(), signal.SIGTERM) + #: On Python >3.4, the global importer lock has been sharded into a + #: per-module lock, meaning there is no guarantee the import statement in + #: service_stub_main will be truly complete before a second thread + #: attempting the same import will see a partially initialized module. + #: Sigh. Therefore serialize execution of the stub itself. + service_stub_lock = threading.Lock() + def _service_stub_main(self, msg): - import mitogen.service - pool = mitogen.service.get_or_create_pool(router=self.router) - pool._receiver._on_receive(msg) + self.service_stub_lock.acquire() + try: + import mitogen.service + pool = mitogen.service.get_or_create_pool(router=self.router) + pool._receiver._on_receive(msg) + finally: + self.service_stub_lock.release() def _on_call_service_msg(self, msg): """ diff --git a/mitogen/fork.py b/mitogen/fork.py index c78558b8..d6685d70 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -98,6 +98,7 @@ def on_fork(): fixup_prngs() mitogen.core.Latch._on_fork() mitogen.core.Side._on_fork() + mitogen.core.ExternalContext.service_stub_lock = threading.Lock() mitogen__service = sys.modules.get('mitogen.service') if mitogen__service: From 6d7bd7e3b939b5eda7ca51c7f43c5109dfcec62d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 22:11:26 +0000 Subject: [PATCH 50/54] docs: update Changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c30c8f3c..180fe9e0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -167,6 +167,10 @@ Fixes ``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly match typical behaviour when ``mitogen_via=`` is active. +* `2a8567b4 `_: fix a race + initializing a child's service thread pool on Python 3.4+, due to a change in + locking scheme used by the Python import mechanism. + Thanks! ~~~~~~~ From e010667230e7fd67e5a0c0ce743968c43599af69 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 23:10:30 +0000 Subject: [PATCH 51/54] Bump version for release. --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 180fe9e0..3bdc248f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,7 +125,7 @@ Core Library series. -v0.2.5 (2019-02-1?) +v0.2.5 (2019-02-13) ------------------- Fixes @@ -179,7 +179,7 @@ Mitogen would not be possible without the support of users. A huge thanks for bug reports, testing, features and fixes in this release contributed by `Carl George `_, `Guy Knights `_, and -`Josh Smift `_, +`Josh Smift `_. v0.2.4 (2019-02-10) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 060430c6..08f875d4 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 4) +__version__ = (0, 2, 5) #: This is :data:`False` in slave contexts. Previously it was used to prevent From 8f9c67daf1eed865b55b4119fd65bb8df89ca49f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 00:35:16 +0000 Subject: [PATCH 52/54] ansible: refactor affinity class and add abstract tests. --- ansible_mitogen/affinity.py | 52 ++++++--- tests/ansible/tests/affinity_test.py | 157 ++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 19 deletions(-) diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index bcab89af..57926516 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -156,11 +156,11 @@ class Policy(object): Assign the helper subprocess policy to this process. """ - -class LinuxPolicy(Policy): +class FixedPolicy(Policy): """ - :class:`Policy` for Linux machines. The scheme here was tested on an - otherwise idle 16 thread machine. + :class:`Policy` for machines where the only control method available is + fixed CPU placement. The scheme here was tested on an otherwise idle 16 + thread machine. - The connection multiplexer is pinned to CPU 0. - The Ansible top-level (strategy) is pinned to CPU 1. @@ -180,26 +180,35 @@ class LinuxPolicy(Policy): CPU-intensive children like SSH are not forced to share the same core as the (otherwise potentially very busy) parent. """ - def __init__(self): + def __init__(self, cpu_count=None): + #: For tests. + self.cpu_count = cpu_count or multiprocessing.cpu_count() self.mem = mmap.mmap(-1, 4096) self.state = State.from_buffer(self.mem) self.state.lock.init() - if self._cpu_count() < 4: - self._reserve_mask = 3 - self._reserve_shift = 2 - self._reserve_controller = True - else: + + if self.cpu_count < 2: + # uniprocessor + self._reserve_mux = False + self._reserve_controller = False + self._reserve_mask = 0 + self._reserve_shift = 0 + elif self.cpu_count < 4: + # small SMP + self._reserve_mux = True + self._reserve_controller = False self._reserve_mask = 1 self._reserve_shift = 1 - self._reserve_controller = False + else: + # big SMP + self._reserve_mux = True + self._reserve_controller = True + self._reserve_mask = 3 + self._reserve_shift = 2 def _set_affinity(self, mask): mitogen.parent._preexec_hook = self._clear - s = struct.pack('L', mask) - _sched_setaffinity(os.getpid(), len(s), s) - - def _cpu_count(self): - return multiprocessing.cpu_count() + self._set_cpu_mask(mask) def _balance(self): self.state.lock.acquire() @@ -210,14 +219,15 @@ class LinuxPolicy(Policy): self.state.lock.release() self._set_cpu(self._reserve_shift + ( - (n % max(1, (self._cpu_count() - self._reserve_shift))) + (n % (self.cpu_count - self._reserve_shift)) )) def _set_cpu(self, cpu): self._set_affinity(1 << cpu) def _clear(self): - self._set_affinity(0xffffffff & ~self._reserve_mask) + all_cpus = (1 << self.cpu_count) - 1 + self._set_affinity(all_cpus & ~self._reserve_mask) def assign_controller(self): if self._reserve_controller: @@ -235,6 +245,12 @@ class LinuxPolicy(Policy): self._clear() +class LinuxPolicy(FixedPolicy): + def _set_cpu_mask(self, mask): + s = struct.pack('L', mask) + _sched_setaffinity(os.getpid(), len(s), s) + + if _sched_setaffinity is not None: policy = LinuxPolicy() else: diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index d898c782..ceb53513 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -11,11 +11,156 @@ import mitogen.parent import ansible_mitogen.affinity + +class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy): + def _set_cpu_mask(self, mask): + self.mask = mask + + +class FixedPolicyTest(testlib.TestCase): + klass = NullFixedPolicy + + def test_assign_controller_1core(self): + # Uniprocessor . + policy = self.klass(cpu_count=1) + policy.assign_controller() + self.assertEquals(0x1, policy.mask) + + def test_assign_controller_2core(self): + # Small SMP gets 1.. % cpu_count + policy = self.klass(cpu_count=2) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + + def test_assign_controller_3core(self): + # Small SMP gets 1.. % cpu_count + policy = self.klass(cpu_count=3) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x4, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x4, policy.mask) + policy.assign_controller() + + def test_assign_controller_4core(self): + # Big SMP gets a dedicated core. + policy = self.klass(cpu_count=4) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + policy.assign_controller() + self.assertEquals(0x2, policy.mask) + + def test_assign_muxprocess_1core(self): + # Uniprocessor . + policy = self.klass(cpu_count=1) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_muxprocess_2core(self): + # Small SMP gets dedicated core. + policy = self.klass(cpu_count=2) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + + def test_assign_muxprocess_3core(self): + # Small SMP gets a dedicated core. + policy = self.klass(cpu_count=3) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_muxprocess_4core(self): + # Big SMP gets a dedicated core. + policy = self.klass(cpu_count=4) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_muxprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_worker_1core(self): + # Balance n % 1 + policy = self.klass(cpu_count=1) + policy.assign_worker() + self.assertEquals(0x1, policy.mask) + policy.assign_worker() + self.assertEquals(0x1, policy.mask) + + def test_assign_worker_2core(self): + # Balance n % 1 + policy = self.klass(cpu_count=2) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + + def test_assign_worker_3core(self): + # Balance n % 1 + policy = self.klass(cpu_count=3) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + policy.assign_worker() + self.assertEquals(0x4, policy.mask) + policy.assign_worker() + self.assertEquals(0x2, policy.mask) + + def test_assign_worker_4core(self): + # Balance n % 1 + policy = self.klass(cpu_count=4) + policy.assign_worker() + self.assertEquals(4, policy.mask) + policy.assign_worker() + self.assertEquals(8, policy.mask) + policy.assign_worker() + self.assertEquals(4, policy.mask) + + def test_assign_subprocess_1core(self): + # allow all except reserved. + policy = self.klass(cpu_count=1) + policy.assign_subprocess() + self.assertEquals(0x1, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x1, policy.mask) + + def test_assign_subprocess_2core(self): + # allow all except reserved. + policy = self.klass(cpu_count=2) + policy.assign_subprocess() + self.assertEquals(0x2, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x2, policy.mask) + + def test_assign_subprocess_3core(self): + # allow all except reserved. + policy = self.klass(cpu_count=3) + policy.assign_subprocess() + self.assertEquals(0x2 + 0x4, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x2 + 0x4, policy.mask) + + def test_assign_subprocess_4core(self): + # allow all except reserved. + policy = self.klass(cpu_count=4) + policy.assign_subprocess() + self.assertEquals(0x4 + 0x8, policy.mask) + policy.assign_subprocess() + self.assertEquals(0x4 + 0x8, policy.mask) + + @unittest2.skipIf( reason='Linux/SMP only', condition=(not ( os.uname()[0] == 'Linux' and - multiprocessing.cpu_count() >= 4 + multiprocessing.cpu_count() > 1 )) ) class LinuxPolicyTest(testlib.TestCase): @@ -33,6 +178,16 @@ class LinuxPolicyTest(testlib.TestCase): finally: fp.close() + def test_set_cpu_mask(self): + self.policy._set_cpu_mask(0x1) + self.assertEquals(0x1, self._get_cpus()) + + self.policy._set_cpu_mask(0x2) + self.assertEquals(0x2, self._get_cpus()) + + self.policy._set_cpu_mask(0x3) + self.assertEquals(0x3, self._get_cpus()) + def test_set_clear(self): before = self._get_cpus() self.policy._set_cpu(3) From 45f915f3923280b54947eeebeca8149fcd2baa97 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 00:37:53 +0000 Subject: [PATCH 53/54] docs: update Changelog; closes #537. --- docs/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3bdc248f..f7ecadbd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,7 +125,7 @@ Core Library series. -v0.2.5 (2019-02-13) +v0.2.5 (2019-02-14) ------------------- Fixes @@ -145,6 +145,10 @@ Fixes ``simplejson`` from a controller that also loaded an incompatible newer version of ``simplejson``. +* `#537 `_: a swapped operator in the + CPU affinity logic meant 2 cores were reserved on 1`_: the source distribution includes a ``LICENSE`` file. From 5ed445c4aaa79a860c8dae3b6f189d42be200859 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 00:57:49 +0000 Subject: [PATCH 54/54] issue #537: disable just the trivial LinuxPolicyTest on Travis. --- tests/ansible/tests/affinity_test.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index ceb53513..8fa8cdb6 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -160,7 +160,7 @@ class FixedPolicyTest(testlib.TestCase): reason='Linux/SMP only', condition=(not ( os.uname()[0] == 'Linux' and - multiprocessing.cpu_count() > 1 + multiprocessing.cpu_count() > 2 )) ) class LinuxPolicyTest(testlib.TestCase): @@ -188,13 +188,6 @@ class LinuxPolicyTest(testlib.TestCase): self.policy._set_cpu_mask(0x3) self.assertEquals(0x3, self._get_cpus()) - def test_set_clear(self): - before = self._get_cpus() - self.policy._set_cpu(3) - self.assertEquals(self._get_cpus(), 1 << 3) - self.policy._clear() - self.assertEquals(self._get_cpus(), before) - def test_clear_on_popen(self): tf = tempfile.NamedTemporaryFile() try: