Merge remote-tracking branch 'origin/azure3'

* origin/azure3:
  docs: update Changelog; closes #542.
  issue #542: return of select poller, new selection logic
  issue #542: .ci: move some tests to Azure and enable Mac job.
  ansible: create stub __init__.py for sdist.

.ci/azure-pipelines-steps.yml

@@ -0,0 +1,20 @@
+
+parameters:
+  name: ''
+  pool: ''
+  sign: false
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+    architecture: 'x64'
+
+- script: .ci/prep_azure.py
+  displayName: "Install requirements."
+
+- script: .ci/$(MODE)_install.py
+  displayName: "Install requirements."
+
+- script: .ci/$(MODE)_tests.py
+  displayName: Run tests.

.ci/azure-pipelines.yml

@@ -5,79 +5,85 @@
 
 jobs:
 
-- job: 'MitogenTests'
+- job: Mac
+  steps:
+  - template: azure-pipelines-steps.yml
   pool:
-    vmImage: 'Ubuntu 16.04'
+    vmImage: macOS-10.13
   strategy:
     matrix:
-      Mitogen27Debian_27:
+      Mito27_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: debian
 
-      MitogenPy27CentOS6_26:
+
+- job: Linux
+  pool:
+    vmImage: "Ubuntu 16.04"
+  steps:
+  - template: azure-pipelines-steps.yml
+  strategy:
+    matrix:
+      #
+      # Confirmed working
+      #
+      Mito27Debian_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: centos6
+        DISTRO: debian
 
-      #Py26CentOS7:
+      #MitoPy27CentOS6_26:
         #python.version: '2.7'
         #MODE: mitogen
         #DISTRO: centos6
 
-      Mitogen36CentOS6_26:
+      Mito36CentOS6_26:
         python.version: '3.6'
         MODE: mitogen
         DISTRO: centos6
 
-      DebOps_2460_27_27:
-        python.version: '2.7'
-        MODE: debops_common
-        VER: 2.4.6.0
-
-      DebOps_262_36_27:
-        python.version: '3.6'
-        MODE: debops_common
-        VER: 2.6.2
-
-      Ansible_2460_26:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.4.6.0
+      #
+      #
+      #
 
-      Ansible_262_26:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.6.2
+      #Py26CentOS7:
+        #python.version: '2.7'
+        #MODE: mitogen
+        #DISTRO: centos6
 
-      Ansible_2460_36:
-        python.version: '3.6'
-        MODE: ansible
-        VER: 2.4.6.0
+      #DebOps_2460_27_27:
+        #python.version: '2.7'
+        #MODE: debops_common
+        #VER: 2.4.6.0
 
-      Ansible_262_36:
-        python.version: '3.6'
-        MODE: ansible
-        VER: 2.6.2
+      #DebOps_262_36_27:
+        #python.version: '3.6'
+        #MODE: debops_common
+        #VER: 2.6.2
 
-      Vanilla_262_27:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.6.2
-        DISTROS: debian
-        STRATEGY: linear
+      #Ansible_2460_26:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.4.6.0
 
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
-      architecture: 'x64'
+      #Ansible_262_26:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.6.2
 
-  - script: .ci/prep_azure.py
-    displayName: "Install requirements."
+      #Ansible_2460_36:
+        #python.version: '3.6'
+        #MODE: ansible
+        #VER: 2.4.6.0
 
-  - script: .ci/$(MODE)_install.py
-    displayName: "Install requirements."
+      #Ansible_262_36:
+        #python.version: '3.6'
+        #MODE: ansible
+        #VER: 2.6.2
 
-  - script: .ci/$(MODE)_tests.py
-    displayName: Run tests.
+      #Vanilla_262_27:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.6.2
+        #DISTROS: debian
+        #STRATEGY: linear

.ci/ci_lib.py

@@ -43,6 +43,18 @@ if not hasattr(subprocess, 'check_output'):
     subprocess.check_output = subprocess__check_output
 
 
+# ------------------
+
+def have_apt():
+    proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
+
+def have_docker():
+    proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
+
 # -----------------
 
 # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars.

.ci/mitogen_install.py

@@ -6,10 +6,12 @@ batches = [
     [
         'pip install "pycparser<2.19" "idna<2.7"',
         'pip install -r tests/requirements.txt',
-    ],
-    [
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ]
 ]
 
+if ci_lib.have_docker():
+    batches.append([
+        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
+    ])
+
 ci_lib.run_batches(batches)

.ci/mitogen_tests.py

@@ -11,4 +11,7 @@ os.environ.update({
     'SKIP_ANSIBLE': '1',
 })
 
+if not ci_lib.have_docker():
+    os.environ['SKIP_DOCKER_TESTS'] = '1'
+
 ci_lib.run('./run_tests -v')

.ci/prep_azure.py

@@ -1,22 +1,30 @@
 #!/usr/bin/env python
 
+import os
+import sys
+
 import ci_lib
 
 batches = []
-batches.append([
+
+if ci_lib.have_apt():
+    batches.append([
         'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
         'sudo add-apt-repository ppa:deadsnakes/ppa',
         'sudo apt-get update',
         'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
-])
+    ])
 
-batches.append([
-    'pip install -r dev_requirements.txt',
-])
 
-batches.extend(
+#batches.append([
+    #'pip install -r dev_requirements.txt',
+#])
+
+if ci_lib.have_docker():
+    batches.extend(
         ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
         for distro in ci_lib.DISTROS
-)
+    )
+
 
 ci_lib.run_batches(batches)

.travis.yml

@@ -27,9 +27,7 @@ matrix:
     # 2.4 -> 2.4
     - language: c
       env: MODE=mitogen_py24 DISTRO=centos5
-    # 2.7 -> 2.7
-    - python: "2.7"
-      env: MODE=mitogen DISTRO=debian
+    # 2.7 -> 2.7 -- moved to Azure
     # 2.7 -> 2.6
     #- python: "2.7"
       #env: MODE=mitogen DISTRO=centos6
@@ -39,9 +37,7 @@ matrix:
     # 2.6 -> 3.5
     - python: "2.6"
       env: MODE=mitogen DISTRO=debian-py3
-    # 3.6 -> 2.6
-    - python: "3.6"
-      env: MODE=mitogen DISTRO=centos6
+    # 3.6 -> 2.6 -- moved to Azure
 
     # Debops tests.
     # 2.4.6.0; 2.7 -> 2.7

docs/changelog.rst

@@ -125,6 +125,26 @@ Core Library
   series.
 
 
+v0.2.6 (2019-02-??)
+-------------------
+
+Fixes
+~~~~~
+
+* `#542 <https://github.com/dw/mitogen/issues/542>`_: some versions of OS X
+  ship a default Python that does not support :func:`select.poll`. Restore the
+  0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer
+  :func:`select.poll` if it is available.
+
+
+Thanks!
+~~~~~~~
+
+Mitogen would not be possible without the support of users. A huge thanks for
+bug reports, testing, features and fixes in this release contributed by
+`Petr Enkov <https://github.com/enkov>`_.
+
+
 v0.2.5 (2019-02-14)
 -------------------
 

mitogen/core.py

@@ -1912,6 +1912,8 @@ class Poller(object):
 
     Pollers may only be used by one thread at a time.
     """
+    SUPPORTED = True
+
     # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
     # no upper FD limit, it is suitable for use with Latch, which must handle
     # FDs larger than select's limit during many-host runs. We want this
@@ -1928,11 +1930,16 @@ class Poller(object):
     def __init__(self):
         self._rfds = {}
         self._wfds = {}
-        self._pollobj = select.poll()
 
     def __repr__(self):
         return '%s(%#x)' % (type(self).__name__, id(self))
 
+    def _update(self, fd):
+        """
+        Required by PollPoller subclass.
+        """
+        pass
+
     @property
     def readers(self):
         """
@@ -1955,20 +1962,6 @@ class Poller(object):
         """
         pass
 
-    _readmask = select.POLLIN | select.POLLHUP
-    # TODO: no proof we dont need writemask too
-
-    def _update(self, fd):
-        mask = (((fd in self._rfds) and self._readmask) |
-                ((fd in self._wfds) and select.POLLOUT))
-        if mask:
-            self._pollobj.register(fd, mask)
-        else:
-            try:
-                self._pollobj.unregister(fd)
-            except KeyError:
-                pass
-
     def start_receive(self, fd, data=None):
         """
         Cause :meth:`poll` to yield `data` when `fd` is readable.
@@ -2004,22 +1997,27 @@ class Poller(object):
         self._update(fd)
 
     def _poll(self, timeout):
-        if timeout:
-            timeout *= 1000
+        (rfds, wfds, _), _ = io_op(select.select,
+            self._rfds,
+            self._wfds,
+            (), timeout
+        )
 
-        events, _ = io_op(self._pollobj.poll, timeout)
-        for fd, event in events:
-            if event & self._readmask:
-                _vv and IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd)
+        for fd in rfds:
+            _vv and IOLOG.debug('%r: POLLIN for %r', self, fd)
             data, gen = self._rfds.get(fd, (None, None))
             if gen and gen < self._generation:
                 yield data
-            if event & select.POLLOUT:
+
+        for fd in wfds:
             _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd)
             data, gen = self._wfds.get(fd, (None, None))
             if gen and gen < self._generation:
                 yield data
 
+        if timeout:
+            timeout *= 1000
+
     def poll(self, timeout=None):
         """
         Block the calling thread until one or more FDs are ready for IO.

mitogen/parent.py

@@ -890,10 +890,58 @@ class CallSpec(object):
         )
 
 
+class PollPoller(mitogen.core.Poller):
+    """
+    Poller based on the POSIX poll(2) interface. Not available on some versions
+    of OS X, otherwise it is the preferred poller for small FD counts.
+    """
+    SUPPORTED = hasattr(select, 'poll')
+    _repr = 'PollPoller()'
+
+    def __init__(self):
+        super(PollPoller, self).__init__()
+        self._pollobj = select.poll()
+
+    # TODO: no proof we dont need writemask too
+    _readmask = (
+        getattr(select, 'POLLIN', 0) |
+        getattr(select, 'POLLHUP', 0)
+    )
+
+    def _update(self, fd):
+        mask = (((fd in self._rfds) and self._readmask) |
+                ((fd in self._wfds) and select.POLLOUT))
+        if mask:
+            self._pollobj.register(fd, mask)
+        else:
+            try:
+                self._pollobj.unregister(fd)
+            except KeyError:
+                pass
+
+    def _poll(self, timeout):
+        if timeout:
+            timeout *= 1000
+
+        events, _ = mitogen.core.io_op(self._pollobj.poll, timeout)
+        for fd, event in events:
+            if event & self._readmask:
+                IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd)
+                data, gen = self._rfds.get(fd, (None, None))
+                if gen and gen < self._generation:
+                    yield data
+            if event & select.POLLOUT:
+                IOLOG.debug('%r: POLLOUT for %r', self, fd)
+                data, gen = self._wfds.get(fd, (None, None))
+                if gen and gen < self._generation:
+                    yield data
+
+
 class KqueuePoller(mitogen.core.Poller):
     """
     Poller based on the FreeBSD/Darwin kqueue(2) interface.
     """
+    SUPPORTED = hasattr(select, 'kqueue')
     _repr = 'KqueuePoller()'
 
     def __init__(self):
@@ -971,6 +1019,7 @@ class EpollPoller(mitogen.core.Poller):
     """
     Poller based on the Linux epoll(2) interface.
     """
+    SUPPORTED = hasattr(select, 'epoll')
     _repr = 'EpollPoller()'
 
     def __init__(self):
@@ -1041,20 +1090,18 @@ class EpollPoller(mitogen.core.Poller):
                     yield data
 
 
-if sys.version_info < (2, 6):
-    # 2.4 and 2.5 only had select.select() and select.poll().
-    POLLER_BY_SYSNAME = {}
-else:
-    POLLER_BY_SYSNAME = {
-        'Darwin': KqueuePoller,
-        'FreeBSD': KqueuePoller,
-        'Linux': EpollPoller,
-    }
+# 2.4 and 2.5 only had select.select() and select.poll().
+for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
+    if _klass.SUPPORTED:
+        PREFERRED_POLLER = _klass
 
-PREFERRED_POLLER = POLLER_BY_SYSNAME.get(
-    os.uname()[0],
-    mitogen.core.Poller,
-)
+# For apps that start threads dynamically, it's possible Latch will also get
+# very high-numbered wait fds when there are many connections, and so select()
+# becomes useless there too. So swap in our favourite poller.
+if PollPoller.SUPPORTED:
+    mitogen.core.Latch.poller_class = PollPoller
+else:
+    mitogen.core.Latch.poller_class = PREFERRED_POLLER
 
 
 class DiagLogStream(mitogen.core.BasicStream):

tests/ansible/tests/affinity_test.py

@@ -17,6 +17,10 @@ class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy):
         self.mask = mask
 
 
+@unittest2.skipIf(
+    reason='Linux only',
+    condition=(not os.uname()[0] == 'Linux')
+)
 class FixedPolicyTest(testlib.TestCase):
     klass = NullFixedPolicy
 

tests/file_service_test.py

@@ -1,4 +1,6 @@
 
+import sys
+
 import unittest2
 
 import mitogen.service
@@ -32,10 +34,15 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase):
         expect = service.unregistered_msg % ('/etc/shadow',)
         self.assertTrue(expect in e.args[0])
 
+    if sys.platform == 'darwin':
+        ROOT_GROUP = 'wheel'
+    else:
+        ROOT_GROUP = 'root'
+
     def _validate_response(self, resp):
         self.assertTrue(isinstance(resp, dict))
         self.assertEquals('root', resp['owner'])
-        self.assertEquals('root', resp['group'])
+        self.assertEquals(self.ROOT_GROUP, resp['group'])
         self.assertTrue(isinstance(resp['mode'], int))
         self.assertTrue(isinstance(resp['mtime'], float))
         self.assertTrue(isinstance(resp['atime'], float))

tests/poller_test.py

@@ -401,16 +401,25 @@ class SelectTest(AllMixin, testlib.TestCase):
     klass = mitogen.core.Poller
 
 SelectTest = unittest2.skipIf(
-    condition=not hasattr(select, 'select'),
+    condition=(not SelectTest.klass.SUPPORTED),
     reason='select.select() not supported'
 )(SelectTest)
 
 
+class PollTest(AllMixin, testlib.TestCase):
+    klass = mitogen.parent.PollPoller
+
+PollTest = unittest2.skipIf(
+    condition=(not PollTest.klass.SUPPORTED),
+    reason='select.poll() not supported'
+)(PollTest)
+
+
 class KqueueTest(AllMixin, testlib.TestCase):
     klass = mitogen.parent.KqueuePoller
 
 KqueueTest = unittest2.skipIf(
-    condition=not hasattr(select, 'kqueue'),
+    condition=(not KqueueTest.klass.SUPPORTED),
     reason='select.kqueue() not supported'
 )(KqueueTest)
 
@@ -419,7 +428,7 @@ class EpollTest(AllMixin, testlib.TestCase):
     klass = mitogen.parent.EpollPoller
 
 EpollTest = unittest2.skipIf(
-    condition=not hasattr(select, 'epoll'),
+    condition=(not EpollTest.klass.SUPPORTED),
     reason='select.epoll() not supported'
 )(EpollTest)
 

tests/testlib.py

@@ -450,6 +450,8 @@ class DockerMixin(RouterMixin):
     @classmethod
     def setUpClass(cls):
         super(DockerMixin, cls).setUpClass()
+        if os.environ.get('SKIP_DOCKER_TESTS'):
+            raise unittest2.SkipTest('SKIP_DOCKER_TESTS is set')
         cls.dockerized_ssh = DockerizedSshDaemon()
         cls.dockerized_ssh.wait_for_sshd()