From 06690901e498b6a254f9c33977e8d3e3ffa57d07 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 15:55:50 +0100 Subject: [PATCH 01/17] issue #589: split services example out and make it run. --- docs/services.rst | 50 +--------------------------- examples/service/self_contained.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 49 deletions(-) create mode 100644 examples/service/self_contained.py diff --git a/docs/services.rst b/docs/services.rst index ef402214..e5f0571e 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -61,55 +61,7 @@ Pool Example ------- -.. code-block:: python - - import mitogen - import mitogen.service - - - class FileService(mitogen.service.Service): - """ - Simple file server, for demonstration purposes only! Use of this in - real code would be a security vulnerability as it would permit children - to read arbitrary files from the master's disk. - """ - handle = 500 - required_args = { - 'path': str - } - - def dispatch(self, args, msg): - with open(args['path'], 'r') as fp: - return fp.read() - - - def download_file(context, path): - s = mitogen.service.call(context, FileService.handle, { - 'path': path - }) - - with open(path, 'w') as fp: - fp.write(s) - - - @mitogen.core.takes_econtext - def download_some_files(paths, econtext): - for path in paths: - download_file(econtext.master, path) - - - @mitogen.main() - def main(router): - pool = mitogen.service.Pool(router, size=1, services=[ - FileService(router), - ]) - - remote = router.ssh(hostname='k3') - remote.call(download_some_files, [ - '/etc/passwd', - '/etc/hosts', - ]) - pool.stop() +.. literalinclude:: ../examples/service/self_contained.py Reference diff --git a/examples/service/self_contained.py b/examples/service/self_contained.py new file mode 100644 index 00000000..332aa24e --- /dev/null +++ b/examples/service/self_contained.py @@ -0,0 +1,52 @@ +import mitogen +import mitogen.service + + +class FileService(mitogen.service.Service): + """ + Simple file server, for demonstration purposes only! Use of this in + real code would be a security vulnerability as it would permit children + to read any file from the master's disk. + """ + + @mitogen.service.expose(policy=mitogen.service.AllowAny()) + @mitogen.service.arg_spec(spec={ + 'path': str + }) + def read_file(self, path): + with open(path, 'rb') as fp: + return fp.read() + + +def download_file(source_context, path): + s = source_context.call_service( + service_name=FileService, # may also be string 'pkg.mod.FileService' + method_name='read_file', + path=path, + ) + + with open(path, 'w') as fp: + fp.write(s) + + +def download_some_files(source_context, paths): + for path in paths: + download_file(source_context, path) + + +@mitogen.main() +def main(router): + pool = mitogen.service.Pool(router, services=[ + FileService(router), + ]) + + remote = router.ssh(hostname='k3') + remote.call(download_some_files, + source_context=router.myself(), + paths=[ + '/etc/passwd', + '/etc/hosts', + ] + ) + pool.stop() + From 2d083d19df8e08f5ff274d37fe8200272c47ae55 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 15:55:50 +0100 Subject: [PATCH 02/17] issue #589: remove outdated/incomplete examples --- docs/changelog.rst | 5 +++-- examples/service/client.py | 15 --------------- examples/service/server.py | 20 -------------------- 3 files changed, 3 insertions(+), 37 deletions(-) delete mode 100644 examples/service/client.py delete mode 100644 examples/service/server.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 9f5ed993..6f1b24f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -59,8 +59,9 @@ bug reports, testing, features and fixes in this release contributed by `Anton Markelov `_, `Nigel Metheringham `_, `Orion Poplawski `_, -`Ulrich Schreiner `_, and -`Yuki Nishida `_. +`Ulrich Schreiner `_, +`Yuki Nishida `_, and +`@ghp-rr `_. v0.2.7 (2019-05-19) diff --git a/examples/service/client.py b/examples/service/client.py deleted file mode 100644 index fc2d8427..00000000 --- a/examples/service/client.py +++ /dev/null @@ -1,15 +0,0 @@ - -import mitogen.master -import mitogen.unix -import mitogen.service -import mitogen.utils - - -PING = 500 - - -mitogen.utils.log_to_file() - -router, parent = mitogen.unix.connect('/tmp/mitosock') -with router: - print(mitogen.service.call(parent, CONNECT_BY_ID, {})) diff --git a/examples/service/server.py b/examples/service/server.py deleted file mode 100644 index 1f8c1475..00000000 --- a/examples/service/server.py +++ /dev/null @@ -1,20 +0,0 @@ - -# The service framework will fundamentally change (i.e. become much nicer, and -# hopefully lose those hard-coded magic numbers somehow), but meanwhile this is -# a taster of how it looks today. - -import mitogen -import mitogen.service -import mitogen.unix - - -class PingService(mitogen.service.Service): - def dispatch(self, dct, msg): - return 'Hello, world' - - -@mitogen.main() -def main(router): - listener = mitogen.unix.Listener(router, path='/tmp/mitosock') - service = PingService(router) - service.run() From 687d4033d5edc32235141d7829a8aacc1a5bb6c2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 27 May 2019 23:11:17 +0100 Subject: [PATCH 03/17] docs: add new contributor entry --- docs/contributors.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/contributors.rst b/docs/contributors.rst index dcfb50fa..584c4cd4 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -88,6 +88,9 @@ sponsorship and outstanding future-thinking of its early adopters.

Private Sponsors

    +
  • SkunkWerks — + Mitogen on FreeBSD runs like a kid in a candy store: fast & + sweet.
  • Donald Clark Jackson — Mitogen is an exciting project, and I am happy to support its development.
    • From 874e75276f6275f037599bb02ae6042d278b15a7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 16:17:50 +0100 Subject: [PATCH 04/17] issue #589: ensure real FileService/PushFileService are in the docs --- docs/services.rst | 5 +++++ mitogen/service.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/services.rst b/docs/services.rst index e5f0571e..bcf89740 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -83,6 +83,11 @@ Reference .. autoclass:: mitogen.service.Service :members: +.. autoclass:: mitogen.service.FileService + :members: +.. autoclass:: mitogen.service.PushFileService + :members: + .. autoclass:: mitogen.service.Pool :members: diff --git a/mitogen/service.py b/mitogen/service.py index 302e81ab..942ed4f7 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -625,7 +625,7 @@ class PushFileService(Service): """ Push-based file service. Files are delivered and cached in RAM, sent recursively from parent to child. A child that requests a file via - :meth:`get` will block until it has ben delivered by a parent. + :meth:`get` will block until it has been delivered by a parent. This service will eventually be merged into FileService. """ From 8fc491ac4309b6e18b5bf68e6569b18cbd37e040 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 16:17:50 +0100 Subject: [PATCH 05/17] issue #589: ensure real FileService/PushFileService are in the docs --- docs/services.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/services.rst b/docs/services.rst index bcf89740..85b08e6d 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -83,11 +83,15 @@ Reference .. autoclass:: mitogen.service.Service :members: -.. autoclass:: mitogen.service.FileService - :members: -.. autoclass:: mitogen.service.PushFileService +.. autoclass:: mitogen.service.Pool :members: -.. autoclass:: mitogen.service.Pool + +Built-in Services +----------------- + +.. autoclass:: mitogen.service.FileService :members: +.. autoclass:: mitogen.service.PushFileService + :members: From 95fd9b815c67300188001c82fb68ed22af6ac94b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 18:05:40 +0100 Subject: [PATCH 06/17] travis: exclude docs-master from CI --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 921ad12b..75b0752d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ notifications: language: python +branches: + except: + - docs-master + cache: - pip - directories: From 1a32a79fa6343a5c655a0e0cd9d84c79aa11f359 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 20:42:59 +0100 Subject: [PATCH 07/17] issue #578: update Changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f1b24f5..9135c48f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,6 +34,9 @@ Enhancements Fixes ^^^^^ +* `#578 `_: the extension could crash + while rendering an error message, due to an incorrect format string. + * `#590 `_: the importer can handle modules that replace themselves in :mod:`sys.modules` during import. @@ -59,6 +62,7 @@ bug reports, testing, features and fixes in this release contributed by `Anton Markelov `_, `Nigel Metheringham `_, `Orion Poplawski `_, +`Szabó Dániel Ernő `_, `Ulrich Schreiner `_, `Yuki Nishida `_, and `@ghp-rr `_. From 418fc15e8017ad2a2a7a3c209c3cdbfb27b9c440 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 20:45:48 +0100 Subject: [PATCH 08/17] tests: allow running without hdrhistograms library. --- tests/ansible/lib/callback/fork_histogram.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/ansible/lib/callback/fork_histogram.py b/tests/ansible/lib/callback/fork_histogram.py index 9ce50e13..15260cb5 100644 --- a/tests/ansible/lib/callback/fork_histogram.py +++ b/tests/ansible/lib/callback/fork_histogram.py @@ -10,7 +10,11 @@ import sys import time import ansible.plugins.callback -import hdrh.histogram + +try: + import hdrh.histogram +except ImportError: + hdrh = None def get_fault_count(who=resource.RUSAGE_CHILDREN): @@ -25,9 +29,9 @@ class CallbackModule(ansible.plugins.callback.CallbackBase): if self.hist is not None: return - self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3) - self.fork_latency_sum_usec = 0.0 - if 'FORK_HISTOGRAM' in os.environ: + if hdrh and 'FORK_HISTOGRAM' in os.environ: + self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3) + self.fork_latency_sum_usec = 0.0 self.install() def install(self): @@ -54,7 +58,7 @@ class CallbackModule(ansible.plugins.callback.CallbackBase): self.hist.record_value(latency_usec) def playbook_on_stats(self, stats): - if 'FORK_HISTOGRAM' not in os.environ: + if hdrh is None or 'FORK_HISTOGRAM' not in os.environ: return self_faults = get_fault_count(resource.RUSAGE_SELF) - self.faults_at_start From a766fd3be5f1e58cfc4a6f7486baf9fe75e09e39 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 20:48:46 +0100 Subject: [PATCH 09/17] add .*.pid to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cf9c084d..be62d308 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ htmlcov/ *.egg-info __pycache__/ extra +tests/ansible/.*.pid From 73a87d425d6712053f81ee0416129f5e6d7ab55a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 20:49:27 +0100 Subject: [PATCH 10/17] ci: try bumping more Travis jobs to Ansible 2.8. --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75b0752d..0f6e7d1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ matrix: env: MODE=debops_common VER=2.4.6.0 # 2.5.7; 3.6 -> 2.7 - python: "3.6" - env: MODE=debops_common VER=2.6.2 + env: MODE=debops_common VER=2.8.0 # ansible_mitogen tests. @@ -61,14 +61,14 @@ matrix: - python: "2.6" env: MODE=ansible VER=2.4.6.0 - python: "2.6" - env: MODE=ansible VER=2.6.2 + env: MODE=ansible VER=2.8.0 # 3.6 -> {debian, centos6, centos7} - python: "3.6" env: MODE=ansible VER=2.4.6.0 - python: "3.6" - env: MODE=ansible VER=2.6.2 + env: MODE=ansible VER=2.8.0 # Sanity check against vanilla Ansible. One job suffices. - python: "2.7" - env: MODE=ansible VER=2.6.2 DISTROS=debian STRATEGY=linear + env: MODE=ansible VER=2.8.0 DISTROS=debian STRATEGY=linear From 2f05b93a087f1f1a5d3117fbe6ddfaba4b540f71 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 23:42:25 +0100 Subject: [PATCH 11/17] update gitignore again --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index be62d308..55f37f29 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ htmlcov/ *.egg-info __pycache__/ extra -tests/ansible/.*.pid +**/.*.pid From 7ae926b325fa1de35552bde506420a624b295ff7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 3 Jun 2019 23:43:15 +0100 Subject: [PATCH 12/17] ansible: prevent tempfile.mkstemp() leaks. This avoids a leak present in Ansible 2.7.0..current HEAD, and all similar leaks. See ansible/ansible#57327. --- ansible_mitogen/runner.py | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 05bc55c0..843ffe19 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -112,6 +112,45 @@ else: for token in shlex.split(str(s), comments=comments)] +class TempFileWatcher(object): + """ + Since Ansible 2.7.0, lineinfile leaks file descriptors returned by + :func:`tempfile.mkstemp` (ansible/ansible#57327). Handle this and all + similar cases by recording descriptors produced by mkstemp during module + execution, and cleaning up any leaked descriptors on completion. + """ + def __init__(self): + self._real_mkstemp = tempfile.mkstemp + # (fd, st.st_dev, st.st_ino) + self._fd_dev_inode = [] + tempfile.mkstemp = self._wrap_mkstemp + + def _wrap_mkstemp(self, *args, **kwargs): + fd, path = self._real_mkstemp(*args, **kwargs) + st = os.fstat(fd) + self._fd_dev_inode.append((fd, st.st_dev, st.st_ino)) + return fd, path + + def revert(self): + tempfile.mkstemp = self._real_mkstemp + for tup in self._fd_dev_inode: + self._revert_one(*tup) + + def _revert_one(self, fd, st_dev, st_ino): + try: + st = os.fstat(fd) + except OSError: + # FD no longer exists. + return + + if not (st.st_dev == st_dev and st.st_ino == st_ino): + # FD reused. + return + + LOG.info("a tempfile.mkstemp() FD was leaked during the last task") + os.close(fd) + + class EnvironmentFileWatcher(object): """ Usually Ansible edits to /etc/environment and ~/.pam_environment are @@ -803,6 +842,7 @@ class NewStyleRunner(ScriptRunner): # module, but this has never been a bug report. Instead act like an # interpreter that had its script piped on stdin. self._argv = TemporaryArgv(['']) + self._temp_watcher = TempFileWatcher() self._importer = ModuleUtilsImporter( context=self.service_context, module_utils=self.module_map['custom'], @@ -818,6 +858,7 @@ class NewStyleRunner(ScriptRunner): def revert(self): self.atexit_wrapper.revert() + self._temp_watcher.revert() self._argv.revert() self._stdio.revert() self._revert_excepthook() From 08e7fe4f80e31f529034c6d9453e6ad3acdcc4fc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 4 Jun 2019 00:47:15 +0100 Subject: [PATCH 13/17] tests: add 2.8 format async error timeout message --- .../integration/async/runner_with_polling_and_timeout.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml index 6d87fe6c..dcfa186f 100644 --- a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml +++ b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml @@ -20,5 +20,6 @@ - job1.failed == True - | job1.msg == "async task did not complete within the requested time" or + job1.msg == "async task did not complete within the requested time - 1s" or job1.msg == "Job reached maximum time limit of 1 seconds." From ab9a80cfd43dc22fba1f950387db2fb58f0287be Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 4 Jun 2019 00:50:06 +0100 Subject: [PATCH 14/17] ci: Ansible 2.8 requires Python 2.7. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f6e7d1c..206499c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: # 2.4.6.0; 2.7 -> 2.7 - python: "2.7" env: MODE=debops_common VER=2.4.6.0 - # 2.5.7; 3.6 -> 2.7 + # 2.8.0; 3.6 -> 2.7 - python: "3.6" env: MODE=debops_common VER=2.8.0 @@ -60,7 +60,8 @@ matrix: # 2.6 -> {debian, centos6, centos7} - python: "2.6" env: MODE=ansible VER=2.4.6.0 - - python: "2.6" + # 2.7 -> {debian, centos6, centos7} + - python: "2.7" env: MODE=ansible VER=2.8.0 # 3.6 -> {debian, centos6, centos7} From ad5a80f20097449ea56e39d75d9b7c1609d6ab2a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 4 Jun 2019 13:23:16 +0100 Subject: [PATCH 15/17] Use virtualenv Python for stub connections to workaround problem ../data/stubs/stub-kubectl.py exec -it localhost -- /usr/bin/python -c "...": Traceback (most recent call last): File "", line 1, in LookupError: unknown encoding: base64 It's not clear why this is happening. "stub-kubectl.py" is executed with the 2.7 virtualenv, while the exec() that happens inside stub-kubectl was for "/usr/bin/python". That second Python can't find chunks of its stdlib: stat("/usr/lib/python2.7/encodings/base64", 0x7ffde8744c60) = -1 ENOENT (No such file or directory) open("/usr/lib/python2.7/encodings/base64.so", O_RDONLY) = -1 ENOENT (No such file or directory) open("/usr/lib/python2.7/encodings/base64module.so", O_RDONLY) = -1 ENOENT (No such file or directory) open("/usr/lib/python2.7/encodings/base64.py", O_RDONLY) = -1 ENOENT (No such file or directory) open("/usr/lib/python2.7/encodings/base64.pyc", O_RDONLY) = -1 ENOENT (No such file or directory) write(2, "Traceback (most recent call last):\n", 35) = 35 write(2, " File \"\", line 1, in \n", 39) = 39 --- tests/ansible/integration/stub_connections/kubectl.yml | 1 + tests/ansible/integration/stub_connections/lxc.yml | 1 + tests/ansible/integration/stub_connections/lxd.yml | 1 + tests/ansible/integration/stub_connections/mitogen_doas.yml | 1 + tests/ansible/integration/stub_connections/mitogen_sudo.yml | 1 + tests/ansible/integration/stub_connections/setns_lxc.yml | 1 + tests/ansible/integration/stub_connections/setns_lxd.yml | 1 + 7 files changed, 7 insertions(+) diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index ba53d1e0..867a8c17 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -13,6 +13,7 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index 7a2cd81c..1dbe2a48 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxc + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index 86f4b185..7839a35f 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxd + ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 3c1459e9..5387744e 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_doas + ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_doas_exe: stub-doas.py ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index b7ca3d26..e78afebc 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -10,6 +10,7 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo + ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index c57a8c5c..efef3761 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -18,6 +18,7 @@ -i localhost, -c setns -e mitogen_kind=lxc + -e ansible_python_interpreter=python -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py -m shell -a "echo hi" diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index 7db47661..adee0b14 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -18,6 +18,7 @@ -i localhost, -c setns -e mitogen_kind=lxd + -e ansible_python_interpreter=python -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py -m shell -a "echo hi" From d981a382c97ee86f276d9f2944da82347135e5b6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 4 Jun 2019 01:34:42 +0100 Subject: [PATCH 16/17] ci: work around various broken aspects of Travis VM image - Symlink broken Ubuntu Python package pieces back together. See many Google hits about this issue. - Remove apt sources that can no longer be updated. --- .ci/ansible_tests.py | 2 ++ .travis.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 8d2d8bba..51eab874 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -63,6 +63,8 @@ with ci_lib.Fold('job_setup'): run("sudo apt-get update") run("sudo apt-get install -y sshpass") + run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'") + run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'") with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') diff --git a/.travis.yml b/.travis.yml index 206499c1..eae04cb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ cache: - /home/travis/virtualenv install: +- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v - .ci/${MODE}_install.py script: From ee7dae75146be984abf68279151311b1b8e3dc73 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 4 Jun 2019 14:19:08 +0100 Subject: [PATCH 17/17] ci: Another round of fixes for random Ansible UI breakage in 2.7/2.8 --- .../runner/custom_binary_single_null.yml | 4 ++-- ...custom_python_new_style_missing_interpreter.yml | 14 +++++++------- .../runner/custom_python_new_style_module.yml | 14 +++++++------- .../custom_python_new_style_missing_interpreter.py | 4 ++++ .../lib/modules/custom_python_new_style_module.py | 4 ++++ 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml index d8a1af0c..8e215bf3 100644 --- a/tests/ansible/integration/runner/custom_binary_single_null.yml +++ b/tests/ansible/integration/runner/custom_binary_single_null.yml @@ -17,8 +17,8 @@ - "out.results[0].msg.startswith('MODULE FAILURE')" - "out.results[0].module_stdout.startswith('/bin/sh: ')" - | - out.results[0].module_stdout.endswith('/custom_binary_single_null: cannot execute binary file\r\n') or - out.results[0].module_stdout.endswith('/custom_binary_single_null: Exec format error\r\n') + out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or + out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n') # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 9f7d08ba..77f2cb5c 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -5,13 +5,13 @@ tasks: - custom_python_new_style_missing_interpreter: foo: true - with_sequence: start=1 end={{end|default(1)}} + with_sequence: start=0 end={{end|default(1)}} register: out - assert: - that: | - (not out.changed) and - (not out.results[0].changed) and - out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and - out.results[0].msg == 'Here is my input' - + that: + - "not out.changed" + - "not out.results[0].changed" + # Random breaking interface change since 2.7.x + #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" + - "out.results[0].msg == 'Here is my input'" diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index d86bff4a..0d29d0ac 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -4,16 +4,16 @@ tasks: - custom_python_new_style_module: foo: true - with_sequence: start=1 end={{end|default(1)}} + with_sequence: start=0 end={{end|default(1)}} register: out - assert: - that: | - (not out.changed) and - (not out.results[0].changed) and - out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and - out.results[0].msg == 'Here is my input' - + that: + - "not out.changed" + - "not out.results[0].changed" + # Random breaking interface change since 2.7.x + #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" + - "out.results[0].msg == 'Here is my input'" # Verify sys.argv is not Unicode. - custom_python_detect_environment: diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 66264010..eea4baa4 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -17,3 +17,7 @@ print(" \"changed\": false,") print(" \"msg\": \"Here is my input\",") print(" \"input\": [%s]" % (input_json,)) print("}") + +# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module +# process to exit itself. So needless. +sys.exit(0) diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 70ee062d..f9c176c1 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -23,3 +23,7 @@ print(" \"__package__\": \"%s\"," % (__package__,)) print(" \"msg\": \"Here is my input\",") print(" \"input\": [%s]" % (input_json,)) print("}") + +# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module +# process to exit itself. So needless. +sys.exit(0)