Merge remote-tracking branch 'origin/dmw'

* origin/dmw:
  Motivational shame badges back in README
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused variable (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused variable (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: unused variable (reported by LGTM)
  issue #61: unused import (reported by LGTM)
  issue #61: fix bare except (reported by LGTM)
  issue #61: unused variable (reported by LGTM)
  issue #61: remove duplicate method (reported by LGTM)
  issue #61: add missing close() implementation (reported by LGTM)
  issue #61: add inverse comparison (reported by LGTM)
  issue #61: remove duplicated method (reported by LGTM)
  issue #424: ansible: make put_file() raise AnsibleFileNotFound
  issue #407: update Changelog.

README.md

@@ -5,3 +5,9 @@
 <a href="https://mitogen.readthedocs.io/">Please see the documentation</a>.
 
 ![](https://i.imgur.com/eBM6LhJ.gif)
+
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/dw/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/dw/mitogen/alerts/)
+
+[![Build Status](https://travis-ci.org/dw/mitogen.svg?branch=master)](https://travis-ci.org/dw/mitogen)
+
+[![Pipelines Status](https://dev.azure.com/dw-mitogen/Mitogen/_apis/build/status/dw.mitogen?branchName=master)](https://dev.azure.com/dw-mitogen/Mitogen/_build/latest?definitionId=1?branchName=master)

ansible_mitogen/connection.py

@@ -29,6 +29,7 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import errno
 import logging
 import os
 import pprint
@@ -1007,6 +1008,11 @@ class Connection(ansible.plugins.connection.ConnectionBase):
     #: slightly more overhead, so just randomly subtract 4KiB.
     SMALL_FILE_LIMIT = mitogen.core.CHUNK_SIZE - 4096
 
+    def _throw_io_error(self, e, path):
+        if e.args[0] == errno.ENOENT:
+            s = 'file or module does not exist: ' + path
+            raise ansible.errors.AnsibleFileNotFound(s)
+
     def put_file(self, in_path, out_path):
         """
         Implement put_file() by streamily transferring the file via
@@ -1017,7 +1023,12 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         :param str out_path:
             Remote filesystem path to write.
         """
-        st = os.stat(in_path)
+        try:
+            st = os.stat(in_path)
+        except OSError as e:
+            self._throw_io_error(e, in_path)
+            raise
+
         if not stat.S_ISREG(st.st_mode):
             raise IOError('%r is not a regular file.' % (in_path,))
 
@@ -1025,17 +1036,22 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         # rather than introducing an extra RTT for the child to request it from
         # FileService.
         if st.st_size <= self.SMALL_FILE_LIMIT:
-            fp = open(in_path, 'rb')
             try:
-                s = fp.read(self.SMALL_FILE_LIMIT + 1)
+                fp = open(in_path, 'rb')
-            finally:
+                try:
-                fp.close()
+                    s = fp.read(self.SMALL_FILE_LIMIT + 1)
+                finally:
+                    fp.close()
+            except OSError:
+                self._throw_io_error(e, in_path)
+                raise
 
             # Ensure did not grow during read.
             if len(s) == st.st_size:
                 return self.put_data(out_path, s, mode=st.st_mode,
                                      utimes=(st.st_atime, st.st_mtime))
 
+        self._connect()
         self.parent.call_service(
             service_name='mitogen.service.FileService',
             method_name='register',

ansible_mitogen/logging.py

@@ -29,7 +29,6 @@
 from __future__ import absolute_import
 import logging
 import os
-import sys
 
 import mitogen.core
 import mitogen.utils

ansible_mitogen/mixins.py

@@ -30,7 +30,6 @@ from __future__ import absolute_import
 import logging
 import os
 import pwd
-import shutil
 import traceback
 
 try:

ansible_mitogen/planner.py

@@ -205,15 +205,13 @@ class ScriptPlanner(BinaryPlanner):
             involved here, the vanilla implementation uses it and that use is
             exploited in common playbooks.
         """
+        key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
         try:
-            key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
             template = self._inv.task_vars[key]
         except KeyError:
             return path
 
-        return mitogen.utils.cast(
+        return mitogen.utils.cast(self._inv.templar.template(template))
-            self._inv.templar.template(self._inv.task_vars[key])
-        )
 
     def _get_interpreter(self):
         path, arg = ansible_mitogen.parsing.parse_hashbang(

ansible_mitogen/process.py

@@ -154,13 +154,16 @@ class MuxProcess(object):
     _instance = None
 
     @classmethod
-    def start(cls):
+    def start(cls, _init_logging=True):
         """
         Arrange for the subprocess to be started, if it is not already running.
 
         The parent process picks a UNIX socket path the child will use prior to
         fork, creates a socketpair used essentially as a semaphore, then blocks
         waiting for the child to indicate the UNIX socket is ready for use.
+
+        :param bool _init_logging:
+            For testing, if :data:`False`, don't initialize logging.
         """
         if cls.worker_sock is not None:
             return
@@ -180,7 +183,8 @@ class MuxProcess(object):
 
         cls.original_env = dict(os.environ)
         cls.child_pid = os.fork()
-        ansible_mitogen.logging.setup()
+        if _init_logging:
+            ansible_mitogen.logging.setup()
         if cls.child_pid:
             cls.child_sock.close()
             cls.child_sock = None

ansible_mitogen/runner.py

@@ -40,7 +40,6 @@ from __future__ import unicode_literals
 
 import atexit
 import ctypes
-import errno
 import imp
 import json
 import logging

ansible_mitogen/strategy.py

@@ -75,7 +75,6 @@ def wrap_action_loader__get(name, *args, **kwargs):
     """
     klass = action_loader__get(name, class_only=True)
     if klass:
-        wrapped_name = 'MitogenActionModule_' + name
         bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
         adorned_klass = type(str(name), bases, {})
         if kwargs.get('class_only'):

docs/changelog.rst

@@ -141,7 +141,8 @@ Enhancements
   disconnected targets. This ensures a task will gracefully fail rather than
   hang, for example on network failure or EC2 instance maintenance.
 
-* `#369 <https://github.com/dw/mitogen/issues/369>`_: :meth:`Connection.reset`
+* `#369 <https://github.com/dw/mitogen/issues/369>`_,
+  `#407 <https://github.com/dw/mitogen/issues/407>`_: :meth:`Connection.reset`
   is implemented, allowing `meta: reset_connection
   <https://docs.ansible.com/ansible/latest/modules/meta_module.html>`_ to shut
   down the remote interpreter as documented, and improving support for the
@@ -391,6 +392,7 @@ bug reports, testing, features and fixes in this release contributed by
 `Brian Candler <https://github.com/candlerb>`_,
 `Duane Zamrok <https://github.com/dewthefifth>`_,
 `Eric Chang <https://github.com/changchichung>`_,
+`Gerben Meijer <https://github.com/infernix>`_,
 `Guy Knights <https://github.com/knightsg>`_,
 `Jesse London <https://github.com/jesteria>`_,
 `Jiří Vávra <https://github.com/Houbovo>`_,

examples/mitogen-fuse.py

@@ -20,7 +20,6 @@ import mitogen.master
 import mitogen.utils
 
 import __main__
-import posix
 import os
 
 

examples/service/client.py

@@ -1,6 +1,4 @@
 
-import socket
-
 import mitogen.master
 import mitogen.unix
 import mitogen.service

examples/service/server.py

@@ -3,8 +3,6 @@
 # hopefully lose those hard-coded magic numbers somehow), but meanwhile this is
 # a taster of how it looks today.
 
-import time
-
 import mitogen
 import mitogen.service
 import mitogen.unix

mitogen/core.py

@@ -261,7 +261,6 @@ class CallError(Error):
         else:
             e = fmt
             fmt = '%s.%s: %s' % (type(e).__module__, type(e).__name__, e)
-            args = ()
             tb = sys.exc_info()[2]
             if tb:
                 fmt += '\n'
@@ -920,6 +919,10 @@ class Channel(Sender, Receiver):
         Sender.__init__(self, context, dst_handle)
         Receiver.__init__(self, router, handle)
 
+    def close(self):
+        Receiver.close(self)
+        Sender.close(self)
+
     def __repr__(self):
         return 'Channel(%s, %s)' % (
             Sender.__repr__(self),

mitogen/debug.py

@@ -99,10 +99,6 @@ def get_router_info():
     }
 
 
-def get_router_info(router):
-    pass
-
-
 def get_stream_info(router_id):
     router = get_routers().get(router_id)
     return {

mitogen/doas.py

@@ -27,7 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 import logging
-import os
 
 import mitogen.core
 import mitogen.parent

mitogen/master.py

@@ -1025,8 +1025,6 @@ class IdAllocator(object):
 
         id_, last_id = self.allocate_block()
         requestee = self.router.context_by_id(msg.src_id)
-        allocated = self.router.context_by_id(id_, msg.src_id)
-
         LOG.debug('%r: allocating [%r..%r) to %r',
                   self, id_, last_id, requestee)
         msg.reply((id_, last_id))

mitogen/parent.py

@@ -70,7 +70,7 @@ else:
 
 try:
     SC_OPEN_MAX = os.sysconf('SC_OPEN_MAX')
-except:
+except ValueError:
     SC_OPEN_MAX = 1024
 
 OPENPTY_MSG = (
@@ -1554,6 +1554,9 @@ class Context(mitogen.core.Context):
         super(Context, self).__init__(*args, **kwargs)
         self.default_call_chain = self.call_chain_class(self)
 
+    def __ne__(self, other):
+        return not (self == other)
+
     def __eq__(self, other):
         return (isinstance(other, mitogen.core.Context) and
                 (other.context_id == self.context_id) and
@@ -2079,15 +2082,6 @@ class ModuleForwarder(object):
         callback = lambda: self._on_cache_callback(msg, fullname)
         self.importer._request_module(fullname, callback)
 
-    def _send_one_module(self, msg, tup):
-        self.router._async_route(
-            mitogen.core.Message.pickled(
-                tup,
-                dst_id=msg.src_id,
-                handle=mitogen.core.LOAD_MODULE,
-            )
-        )
-
     def _on_cache_callback(self, msg, fullname):
         LOG.debug('%r._on_get_module(): sending %r', self, fullname)
         stream = self.router.stream_by_id(msg.src_id)

mitogen/ssh.py

@@ -32,7 +32,6 @@ Functionality to allow establishing new slave contexts over an SSH connection.
 
 import logging
 import re
-import time
 
 try:
     from shlex import quote as shlex_quote

mitogen/su.py

@@ -27,7 +27,6 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 import logging
-import os
 
 import mitogen.core
 import mitogen.parent

mitogen/sudo.py

@@ -28,8 +28,6 @@
 
 import logging
 import optparse
-import os
-import time
 
 import mitogen.core
 import mitogen.parent

tests/ansible/tests/connection_test.py

@@ -1,19 +1,33 @@
 
 from __future__ import absolute_import
+import os
 import os.path
 import subprocess
 import tempfile
+import time
+
 import unittest2
 
 import mock
+import ansible.errors
+import ansible.playbook.play_context
 
+import mitogen.core
 import ansible_mitogen.connection
+import ansible_mitogen.plugins.connection.mitogen_local
+import ansible_mitogen.process
 import testlib
 
 
 LOGGER_NAME = ansible_mitogen.target.LOG.name
 
 
+# TODO: fixtureize
+import mitogen.utils
+mitogen.utils.log_to_file()
+ansible_mitogen.process.MuxProcess.start(_init_logging=False)
+
+
 class OptionalIntTest(unittest2.TestCase):
     func = staticmethod(ansible_mitogen.connection.optional_int)
 
@@ -34,5 +48,84 @@ class OptionalIntTest(unittest2.TestCase):
         self.assertEquals(None, self.func({1:2}))
 
 
+class ConnectionMixin(object):
+    klass = ansible_mitogen.plugins.connection.mitogen_local.Connection
+
+    def make_connection(self):
+        play_context = ansible.playbook.play_context.PlayContext()
+        return self.klass(play_context, new_stdin=False)
+
+    def wait_for_completion(self):
+        # put_data() is asynchronous, must wait for operation to happen. Do
+        # that by making RPC for some junk that must run on the thread once op
+        # completes.
+        self.conn.get_chain().call(os.getpid)
+
+    def setUp(self):
+        super(ConnectionMixin, self).setUp()
+        self.conn = self.make_connection()
+
+    def tearDown(self):
+        self.conn.close()
+        super(ConnectionMixin, self).tearDown()
+
+
+class PutDataTest(ConnectionMixin, unittest2.TestCase):
+    def test_out_path(self):
+        path = tempfile.mktemp(prefix='mitotest')
+        contents = mitogen.core.b('contents')
+
+        self.conn.put_data(path, contents)
+        self.wait_for_completion()
+        self.assertEquals(contents, open(path, 'rb').read())
+        os.unlink(path)
+
+    def test_mode(self):
+        path = tempfile.mktemp(prefix='mitotest')
+        contents = mitogen.core.b('contents')
+
+        self.conn.put_data(path, contents, mode=int('0123', 8))
+        self.wait_for_completion()
+        st = os.stat(path)
+        self.assertEquals(int('0123', 8), st.st_mode & int('0777', 8))
+        os.unlink(path)
+
+
+class PutFileTest(ConnectionMixin, unittest2.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(PutFileTest, cls).setUpClass()
+        cls.big_path = tempfile.mktemp(prefix='mitotestbig')
+        open(cls.big_path, 'w').write('x'*1048576)
+
+    @classmethod
+    def tearDownClass(cls):
+        os.unlink(cls.big_path)
+        super(PutFileTest, cls).tearDownClass()
+
+    def test_out_path_tiny(self):
+        path = tempfile.mktemp(prefix='mitotest')
+        self.conn.put_file(in_path=__file__, out_path=path)
+        self.wait_for_completion()
+        self.assertEquals(open(path, 'rb').read(),
+                          open(__file__, 'rb').read())
+
+        os.unlink(path)
+
+    def test_out_path_big(self):
+        path = tempfile.mktemp(prefix='mitotest')
+        self.conn.put_file(in_path=self.big_path, out_path=path)
+        self.wait_for_completion()
+        self.assertEquals(open(path, 'rb').read(),
+                          open(self.big_path, 'rb').read())
+        #self._compare_times_modes(path, __file__)
+        os.unlink(path)
+
+    def test_big_in_path_not_found(self):
+        path = tempfile.mktemp(prefix='mitotest')
+        self.assertRaises(ansible.errors.AnsibleFileNotFound,
+            lambda: self.conn.put_file(in_path='/nonexistent', out_path=path))
+
+
 if __name__ == '__main__':
     unittest2.main()