Merge remote-tracking branch 'origin/ci280'

* origin/ci280:
  ci: Another round of fixes for random Ansible UI breakage in 2.7/2.8
  ci: work around various broken aspects of Travis VM image
  Use virtualenv Python for stub connections to workaround problem
  ci: Ansible 2.8 requires Python 2.7.
  tests: add 2.8 format async error timeout message
  ansible: prevent tempfile.mkstemp() leaks.
  update gitignore again
  ci: try bumping more Travis jobs to Ansible 2.8.
  add .*.pid to gitignore
  tests: allow running without hdrhistograms library.
  issue #578: update Changelog.
  travis: exclude docs-master from CI
  issue #589: ensure real FileService/PushFileService are in the docs
  issue #589: ensure real FileService/PushFileService are in the docs
  docs: add new contributor entry
  issue #589: remove outdated/incomplete examples
  issue #589: split services example out and make it run.

.ci/ansible_tests.py

@@ -63,6 +63,8 @@ with ci_lib.Fold('job_setup'):
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')

.gitignore

@@ -14,3 +14,4 @@ htmlcov/
 *.egg-info
 __pycache__/
 extra
+**/.*.pid

.travis.yml

@@ -6,12 +6,17 @@ notifications:
 
 language: python
 
+branches:
+  except:
+  - docs-master
+
 cache:
 - pip
 - directories:
   - /home/travis/virtualenv
 
 install:
+- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
 - .ci/${MODE}_install.py
 
 script:
@@ -43,9 +48,9 @@ matrix:
     # 2.4.6.0; 2.7 -> 2.7
     - python: "2.7"
       env: MODE=debops_common VER=2.4.6.0
-    # 2.5.7; 3.6 -> 2.7
+    # 2.8.0; 3.6 -> 2.7
     - python: "3.6"
-      env: MODE=debops_common VER=2.6.2
+      env: MODE=debops_common VER=2.8.0
 
     # ansible_mitogen tests.
 
@@ -56,15 +61,16 @@ matrix:
     # 2.6 -> {debian, centos6, centos7}
     - python: "2.6"
       env: MODE=ansible VER=2.4.6.0
-    - python: "2.6"
+    # 2.7 -> {debian, centos6, centos7}
-      env: MODE=ansible VER=2.6.2
+    - python: "2.7"
+      env: MODE=ansible VER=2.8.0
 
     # 3.6 -> {debian, centos6, centos7}
     - python: "3.6"
       env: MODE=ansible VER=2.4.6.0
     - python: "3.6"
-      env: MODE=ansible VER=2.6.2
+      env: MODE=ansible VER=2.8.0
 
     # Sanity check against vanilla Ansible. One job suffices.
     - python: "2.7"
-      env: MODE=ansible VER=2.6.2 DISTROS=debian STRATEGY=linear
+      env: MODE=ansible VER=2.8.0 DISTROS=debian STRATEGY=linear

ansible_mitogen/runner.py

@@ -112,6 +112,45 @@ else:
                 for token in shlex.split(str(s), comments=comments)]
 
 
+class TempFileWatcher(object):
+    """
+    Since Ansible 2.7.0, lineinfile leaks file descriptors returned by
+    :func:`tempfile.mkstemp` (ansible/ansible#57327). Handle this and all
+    similar cases by recording descriptors produced by mkstemp during module
+    execution, and cleaning up any leaked descriptors on completion.
+    """
+    def __init__(self):
+        self._real_mkstemp = tempfile.mkstemp
+        # (fd, st.st_dev, st.st_ino)
+        self._fd_dev_inode = []
+        tempfile.mkstemp = self._wrap_mkstemp
+
+    def _wrap_mkstemp(self, *args, **kwargs):
+        fd, path = self._real_mkstemp(*args, **kwargs)
+        st = os.fstat(fd)
+        self._fd_dev_inode.append((fd, st.st_dev, st.st_ino))
+        return fd, path
+
+    def revert(self):
+        tempfile.mkstemp = self._real_mkstemp
+        for tup in self._fd_dev_inode:
+            self._revert_one(*tup)
+
+    def _revert_one(self, fd, st_dev, st_ino):
+        try:
+            st = os.fstat(fd)
+        except OSError:
+            # FD no longer exists.
+            return
+
+        if not (st.st_dev == st_dev and st.st_ino == st_ino):
+            # FD reused.
+            return
+
+        LOG.info("a tempfile.mkstemp() FD was leaked during the last task")
+        os.close(fd)
+
+
 class EnvironmentFileWatcher(object):
     """
     Usually Ansible edits to /etc/environment and ~/.pam_environment are
@@ -803,6 +842,7 @@ class NewStyleRunner(ScriptRunner):
         # module, but this has never been a bug report. Instead act like an
         # interpreter that had its script piped on stdin.
         self._argv = TemporaryArgv([''])
+        self._temp_watcher = TempFileWatcher()
         self._importer = ModuleUtilsImporter(
             context=self.service_context,
             module_utils=self.module_map['custom'],
@@ -818,6 +858,7 @@ class NewStyleRunner(ScriptRunner):
 
     def revert(self):
         self.atexit_wrapper.revert()
+        self._temp_watcher.revert()
         self._argv.revert()
         self._stdio.revert()
         self._revert_excepthook()

docs/changelog.rst

@@ -34,6 +34,9 @@ Enhancements
 Fixes
 ^^^^^
 
+* `#578 <https://github.com/dw/mitogen/issues/578>`_: the extension could crash
+  while rendering an error message, due to an incorrect format string.
+
 * `#590 <https://github.com/dw/mitogen/issues/590>`_: the importer can handle
   modules that replace themselves in :mod:`sys.modules` during import.
 
@@ -59,8 +62,10 @@ bug reports, testing, features and fixes in this release contributed by
 `Anton Markelov <https://github.com/strangeman>`_,
 `Nigel Metheringham <https://github.com/nigelm>`_,
 `Orion Poplawski <https://github.com/opoplawski>`_,
-`Ulrich Schreiner <https://github.com/ulrichSchreiner>`_, and
+`Szabó Dániel Ernő <https://github.com/r3ap3rpy>`_,
-`Yuki Nishida <https://github.com/yuki-nishida-exa>`_.
+`Ulrich Schreiner <https://github.com/ulrichSchreiner>`_,
+`Yuki Nishida <https://github.com/yuki-nishida-exa>`_, and
+`@ghp-rr <https://github.com/ghp-rr>`_.
 
 
 v0.2.7 (2019-05-19)

docs/contributors.rst

@@ -88,6 +88,9 @@ sponsorship and outstanding future-thinking of its early adopters.
     <h3>Private Sponsors</h3>
 
     <ul style="line-height: 120% !important;">
+    <li><a href="https://skunkwerks.at/">SkunkWerks</a> &mdash;
+        <em>Mitogen on FreeBSD runs like a kid in a candy store: fast &amp;
+        sweet.</em></li>
     <li>Donald Clark Jackson &mdash;
         <em>Mitogen is an exciting project, and I am happy to support its
         development.</em></li>

docs/services.rst

@@ -61,55 +61,7 @@ Pool
 Example
 -------
 
-.. code-block:: python
+.. literalinclude:: ../examples/service/self_contained.py
-
-    import mitogen
-    import mitogen.service
-
-
-    class FileService(mitogen.service.Service):
-        """
-        Simple file server, for demonstration purposes only! Use of this in
-        real code would be a security vulnerability as it would permit children
-        to read arbitrary files from the master's disk.
-        """
-        handle = 500
-        required_args = {
-            'path': str
-        }
-
-        def dispatch(self, args, msg):
-            with open(args['path'], 'r') as fp:
-                return fp.read()
-
-
-    def download_file(context, path):
-        s = mitogen.service.call(context, FileService.handle, {
-            'path': path
-        })
-
-        with open(path, 'w') as fp:
-            fp.write(s)
-
-
-    @mitogen.core.takes_econtext
-    def download_some_files(paths, econtext):
-        for path in paths:
-            download_file(econtext.master, path)
-
-
-    @mitogen.main()
-    def main(router):
-        pool = mitogen.service.Pool(router, size=1, services=[
-            FileService(router),
-        ])
-
-        remote = router.ssh(hostname='k3')
-        remote.call(download_some_files, [
-            '/etc/passwd',
-            '/etc/hosts',
-        ])
-        pool.stop()
 
 
 Reference
@@ -134,3 +86,12 @@ Reference
 .. autoclass:: mitogen.service.Pool
     :members:
 
+
+Built-in Services
+-----------------
+
+.. autoclass:: mitogen.service.FileService
+    :members:
+
+.. autoclass:: mitogen.service.PushFileService
+    :members:

examples/service/client.py

@@ -1,15 +0,0 @@
-
-import mitogen.master
-import mitogen.unix
-import mitogen.service
-import mitogen.utils
-
-
-PING = 500
-
-
-mitogen.utils.log_to_file()
-
-router, parent = mitogen.unix.connect('/tmp/mitosock')
-with router:
-    print(mitogen.service.call(parent, CONNECT_BY_ID, {}))

examples/service/self_contained.py

@@ -0,0 +1,52 @@
+import mitogen
+import mitogen.service
+
+
+class FileService(mitogen.service.Service):
+    """
+    Simple file server, for demonstration purposes only! Use of this in
+    real code would be a security vulnerability as it would permit children
+    to read any file from the master's disk.
+    """
+
+    @mitogen.service.expose(policy=mitogen.service.AllowAny())
+    @mitogen.service.arg_spec(spec={
+        'path': str
+    })
+    def read_file(self, path):
+        with open(path, 'rb') as fp:
+            return fp.read()
+
+
+def download_file(source_context, path):
+    s = source_context.call_service(
+        service_name=FileService,  # may also be string 'pkg.mod.FileService'
+        method_name='read_file',
+        path=path,
+    )
+
+    with open(path, 'w') as fp:
+        fp.write(s)
+
+
+def download_some_files(source_context, paths):
+    for path in paths:
+        download_file(source_context, path)
+
+
+@mitogen.main()
+def main(router):
+    pool = mitogen.service.Pool(router, services=[
+        FileService(router),
+    ])
+
+    remote = router.ssh(hostname='k3')
+    remote.call(download_some_files,
+        source_context=router.myself(),
+        paths=[
+            '/etc/passwd',
+            '/etc/hosts',
+        ]
+    )
+    pool.stop()
+

examples/service/server.py

@@ -1,20 +0,0 @@
-
-# The service framework will fundamentally change (i.e. become much nicer, and
-# hopefully lose those hard-coded magic numbers somehow), but meanwhile this is
-# a taster of how it looks today.
-
-import mitogen
-import mitogen.service
-import mitogen.unix
-
-
-class PingService(mitogen.service.Service):
-    def dispatch(self, dct, msg):
-        return 'Hello, world'
-
-
-@mitogen.main()
-def main(router):
-    listener = mitogen.unix.Listener(router, path='/tmp/mitosock')
-    service = PingService(router)
-    service.run()

mitogen/service.py

@@ -625,7 +625,7 @@ class PushFileService(Service):
     """
     Push-based file service. Files are delivered and cached in RAM, sent
     recursively from parent to child. A child that requests a file via
-    :meth:`get` will block until it has ben delivered by a parent.
+    :meth:`get` will block until it has been delivered by a parent.
 
     This service will eventually be merged into FileService.
     """

tests/ansible/integration/async/runner_with_polling_and_timeout.yml

@@ -20,5 +20,6 @@
         - job1.failed == True
         - |
           job1.msg == "async task did not complete within the requested time" or
+          job1.msg == "async task did not complete within the requested time - 1s" or
           job1.msg == "Job reached maximum time limit of 1 seconds."
 

tests/ansible/integration/runner/custom_binary_single_null.yml

@@ -17,8 +17,8 @@
       - "out.results[0].msg.startswith('MODULE FAILURE')"
       - "out.results[0].module_stdout.startswith('/bin/sh: ')"
       - |
-        out.results[0].module_stdout.endswith('/custom_binary_single_null: cannot execute binary file\r\n') or
+        out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or
-        out.results[0].module_stdout.endswith('/custom_binary_single_null: Exec format error\r\n')
+        out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n')
 
 
 # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the

tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml

@@ -5,13 +5,13 @@
   tasks:
     - custom_python_new_style_missing_interpreter:
         foo: true
-      with_sequence: start=1 end={{end|default(1)}}
+      with_sequence: start=0 end={{end|default(1)}}
       register: out
 
     - assert:
-        that: |
+        that:
-          (not out.changed) and
+        - "not out.changed"
-          (not out.results[0].changed) and
+        - "not out.results[0].changed"
-          out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and
+        # Random breaking interface change since 2.7.x
-          out.results[0].msg == 'Here is my input'
+        #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo"
-
+        - "out.results[0].msg == 'Here is my input'"

tests/ansible/integration/runner/custom_python_new_style_module.yml

@@ -4,16 +4,16 @@
   tasks:
     - custom_python_new_style_module:
         foo: true
-      with_sequence: start=1 end={{end|default(1)}}
+      with_sequence: start=0 end={{end|default(1)}}
       register: out
 
     - assert:
-        that: |
+        that:
-          (not out.changed) and
+        - "not out.changed"
-          (not out.results[0].changed) and
+        - "not out.results[0].changed"
-          out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and
+        # Random breaking interface change since 2.7.x
-          out.results[0].msg == 'Here is my input'
+        #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo"
-
+        - "out.results[0].msg == 'Here is my input'"
 
     # Verify sys.argv is not Unicode.
     - custom_python_detect_environment:

tests/ansible/integration/stub_connections/kubectl.yml

@@ -13,6 +13,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: kubectl
+      ansible_python_interpreter: python  # avoid Travis virtualenv breakage
       mitogen_kubectl_path: stub-kubectl.py
     register: out
 

tests/ansible/integration/stub_connections/lxc.yml

@@ -10,6 +10,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: lxc
+      ansible_python_interpreter: python  # avoid Travis virtualenv breakage
       mitogen_lxc_attach_path: stub-lxc-attach.py
     register: out
 

tests/ansible/integration/stub_connections/lxd.yml

@@ -10,6 +10,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: lxd
+      ansible_python_interpreter: python  # avoid Travis virtualenv breakage
       mitogen_lxc_path: stub-lxc.py
     register: out
 

tests/ansible/integration/stub_connections/mitogen_doas.yml

@@ -10,6 +10,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: mitogen_doas
+      ansible_python_interpreter: python  # avoid Travis virtualenv breakage
       ansible_doas_exe: stub-doas.py
       ansible_user: someuser
     register: out

tests/ansible/integration/stub_connections/mitogen_sudo.yml

@@ -10,6 +10,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: mitogen_sudo
+      ansible_python_interpreter: python  # avoid Travis virtualenv breakage
       ansible_user: root
       ansible_become_exe: stub-sudo.py
       ansible_become_flags: -H --type=sometype --role=somerole

tests/ansible/integration/stub_connections/setns_lxc.yml

@@ -18,6 +18,7 @@
         -i localhost,
         -c setns
         -e mitogen_kind=lxc
+        -e ansible_python_interpreter=python
         -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py
         -m shell
         -a "echo hi"

tests/ansible/integration/stub_connections/setns_lxd.yml

@@ -18,6 +18,7 @@
         -i localhost,
         -c setns
         -e mitogen_kind=lxd
+        -e ansible_python_interpreter=python
         -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py
         -m shell
         -a "echo hi"

tests/ansible/lib/callback/fork_histogram.py

@@ -10,7 +10,11 @@ import sys
 import time
 
 import ansible.plugins.callback
-import hdrh.histogram
+
+try:
+    import hdrh.histogram
+except ImportError:
+    hdrh = None
 
 
 def get_fault_count(who=resource.RUSAGE_CHILDREN):
@@ -25,9 +29,9 @@ class CallbackModule(ansible.plugins.callback.CallbackBase):
         if self.hist is not None:
             return
 
+        if hdrh and 'FORK_HISTOGRAM' in os.environ:
             self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3)
             self.fork_latency_sum_usec = 0.0
-        if 'FORK_HISTOGRAM' in os.environ:
             self.install()
 
     def install(self):
@@ -54,7 +58,7 @@ class CallbackModule(ansible.plugins.callback.CallbackBase):
             self.hist.record_value(latency_usec)
 
     def playbook_on_stats(self, stats):
-        if 'FORK_HISTOGRAM' not in os.environ:
+        if hdrh is None or 'FORK_HISTOGRAM' not in os.environ:
             return
 
         self_faults = get_fault_count(resource.RUSAGE_SELF) - self.faults_at_start

tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py

@@ -17,3 +17,7 @@ print("  \"changed\": false,")
 print("  \"msg\": \"Here is my input\",")
 print("  \"input\": [%s]" % (input_json,))
 print("}")
+
+# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module
+# process to exit itself. So needless.
+sys.exit(0)

tests/ansible/lib/modules/custom_python_new_style_module.py

@@ -23,3 +23,7 @@ print("  \"__package__\": \"%s\"," % (__package__,))
 print("  \"msg\": \"Here is my input\",")
 print("  \"input\": [%s]" % (input_json,))
 print("}")
+
+# Ansible since 2.7.0/52449cc01a7 broke __file__ and *requires* the module
+# process to exit itself. So needless.
+sys.exit(0)