diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml new file mode 100644 index 00000000..a377d795 --- /dev/null +++ b/.ci/azure-pipelines-steps.yml @@ -0,0 +1,20 @@ + +parameters: + name: '' + pool: '' + sign: false + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + +- script: .ci/prep_azure.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_install.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_tests.py + displayName: Run tests. diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index fbbb9640..dc5f7162 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -5,79 +5,85 @@ jobs: -- job: 'MitogenTests' +- job: Mac + steps: + - template: azure-pipelines-steps.yml pool: - vmImage: 'Ubuntu 16.04' + vmImage: macOS-10.13 strategy: matrix: - Mitogen27Debian_27: + Mito27_27: python.version: '2.7' MODE: mitogen - DISTRO: debian - MitogenPy27CentOS6_26: + +- job: Linux + pool: + vmImage: "Ubuntu 16.04" + steps: + - template: azure-pipelines-steps.yml + strategy: + matrix: + # + # Confirmed working + # + Mito27Debian_27: python.version: '2.7' MODE: mitogen - DISTRO: centos6 + DISTRO: debian - #Py26CentOS7: + #MitoPy27CentOS6_26: #python.version: '2.7' #MODE: mitogen #DISTRO: centos6 - Mitogen36CentOS6_26: + Mito36CentOS6_26: python.version: '3.6' MODE: mitogen DISTRO: centos6 - DebOps_2460_27_27: - python.version: '2.7' - MODE: debops_common - VER: 2.4.6.0 - - DebOps_262_36_27: - python.version: '3.6' - MODE: debops_common - VER: 2.6.2 - - Ansible_2460_26: - python.version: '2.7' - MODE: ansible - VER: 2.4.6.0 + # + # + # - Ansible_262_26: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 + #Py26CentOS7: + #python.version: '2.7' + #MODE: mitogen + #DISTRO: centos6 - Ansible_2460_36: - python.version: '3.6' - MODE: ansible - VER: 2.4.6.0 + #DebOps_2460_27_27: + #python.version: '2.7' + #MODE: debops_common + #VER: 2.4.6.0 - Ansible_262_36: - python.version: '3.6' - MODE: ansible - VER: 2.6.2 + #DebOps_262_36_27: + #python.version: '3.6' + #MODE: debops_common + #VER: 2.6.2 - Vanilla_262_27: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 - DISTROS: debian - STRATEGY: linear + #Ansible_2460_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.4.6.0 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + #Ansible_262_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/prep_azure.py - displayName: "Install requirements." + #Ansible_2460_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.4.6.0 - - script: .ci/$(MODE)_install.py - displayName: "Install requirements." + #Ansible_262_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/$(MODE)_tests.py - displayName: Run tests. + #Vanilla_262_27: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 + #DISTROS: debian + #STRATEGY: linear diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 10e9d11e..d4f32f55 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -43,6 +43,18 @@ if not hasattr(subprocess, 'check_output'): subprocess.check_output = subprocess__check_output +# ------------------ + +def have_apt(): + proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + +def have_docker(): + proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + # ----------------- # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 10813b55..72bc75e3 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -6,10 +6,12 @@ batches = [ [ 'pip install "pycparser<2.19" "idna<2.7"', 'pip install -r tests/requirements.txt', - ], - [ - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ] ] +if ci_lib.have_docker(): + batches.append([ + 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), + ]) + ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4ba796c2..36928ac9 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -11,4 +11,7 @@ os.environ.update({ 'SKIP_ANSIBLE': '1', }) +if not ci_lib.have_docker(): + os.environ['SKIP_DOCKER_TESTS'] = '1' + ci_lib.run('./run_tests -v') diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 10126df2..5199a87e 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -1,22 +1,30 @@ #!/usr/bin/env python +import os +import sys + import ci_lib batches = [] -batches.append([ - 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - 'sudo add-apt-repository ppa:deadsnakes/ppa', - 'sudo apt-get update', - 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', -]) - -batches.append([ - 'pip install -r dev_requirements.txt', -]) - -batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] - for distro in ci_lib.DISTROS -) + +if ci_lib.have_apt(): + batches.append([ + 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', + 'sudo add-apt-repository ppa:deadsnakes/ppa', + 'sudo apt-get update', + 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', + ]) + + +#batches.append([ + #'pip install -r dev_requirements.txt', +#]) + +if ci_lib.have_docker(): + batches.extend( + ['docker pull %s' % (ci_lib.image_for_distro(distro),)] + for distro in ci_lib.DISTROS + ) + ci_lib.run_batches(batches) diff --git a/.travis.yml b/.travis.yml index aee14c00..921ad12b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,7 @@ matrix: # 2.4 -> 2.4 - language: c env: MODE=mitogen_py24 DISTRO=centos5 - # 2.7 -> 2.7 - - python: "2.7" - env: MODE=mitogen DISTRO=debian + # 2.7 -> 2.7 -- moved to Azure # 2.7 -> 2.6 #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 @@ -39,9 +37,7 @@ matrix: # 2.6 -> 3.5 - python: "2.6" env: MODE=mitogen DISTRO=debian-py3 - # 3.6 -> 2.6 - - python: "3.6" - env: MODE=mitogen DISTRO=centos6 + # 3.6 -> 2.6 -- moved to Azure # Debops tests. # 2.4.6.0; 2.7 -> 2.7 diff --git a/ansible_mitogen/compat/__init__.py b/ansible_mitogen/compat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/changelog.rst b/docs/changelog.rst index f7ecadbd..887a9e85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,6 +125,26 @@ Core Library series. +v0.2.6 (2019-02-??) +------------------- + +Fixes +~~~~~ + +* `#542 `_: some versions of OS X + ship a default Python that does not support :func:`select.poll`. Restore the + 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer + :func:`select.poll` if it is available. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +bug reports, testing, features and fixes in this release contributed by +`Petr Enkov `_. + + v0.2.5 (2019-02-14) ------------------- diff --git a/mitogen/core.py b/mitogen/core.py index 470b00ca..cfdf996b 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1912,6 +1912,8 @@ class Poller(object): Pollers may only be used by one thread at a time. """ + SUPPORTED = True + # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has # no upper FD limit, it is suitable for use with Latch, which must handle # FDs larger than select's limit during many-host runs. We want this @@ -1928,11 +1930,16 @@ class Poller(object): def __init__(self): self._rfds = {} self._wfds = {} - self._pollobj = select.poll() def __repr__(self): return '%s(%#x)' % (type(self).__name__, id(self)) + def _update(self, fd): + """ + Required by PollPoller subclass. + """ + pass + @property def readers(self): """ @@ -1955,20 +1962,6 @@ class Poller(object): """ pass - _readmask = select.POLLIN | select.POLLHUP - # TODO: no proof we dont need writemask too - - def _update(self, fd): - mask = (((fd in self._rfds) and self._readmask) | - ((fd in self._wfds) and select.POLLOUT)) - if mask: - self._pollobj.register(fd, mask) - else: - try: - self._pollobj.unregister(fd) - except KeyError: - pass - def start_receive(self, fd, data=None): """ Cause :meth:`poll` to yield `data` when `fd` is readable. @@ -2004,22 +1997,27 @@ class Poller(object): self._update(fd) def _poll(self, timeout): + (rfds, wfds, _), _ = io_op(select.select, + self._rfds, + self._wfds, + (), timeout + ) + + for fd in rfds: + _vv and IOLOG.debug('%r: POLLIN for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + for fd in wfds: + _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + if timeout: timeout *= 1000 - events, _ = io_op(self._pollobj.poll, timeout) - for fd, event in events: - if event & self._readmask: - _vv and IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd) - data, gen = self._rfds.get(fd, (None, None)) - if gen and gen < self._generation: - yield data - if event & select.POLLOUT: - _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd) - data, gen = self._wfds.get(fd, (None, None)) - if gen and gen < self._generation: - yield data - def poll(self, timeout=None): """ Block the calling thread until one or more FDs are ready for IO. diff --git a/mitogen/parent.py b/mitogen/parent.py index 7e567aaa..f793f234 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -890,10 +890,58 @@ class CallSpec(object): ) +class PollPoller(mitogen.core.Poller): + """ + Poller based on the POSIX poll(2) interface. Not available on some versions + of OS X, otherwise it is the preferred poller for small FD counts. + """ + SUPPORTED = hasattr(select, 'poll') + _repr = 'PollPoller()' + + def __init__(self): + super(PollPoller, self).__init__() + self._pollobj = select.poll() + + # TODO: no proof we dont need writemask too + _readmask = ( + getattr(select, 'POLLIN', 0) | + getattr(select, 'POLLHUP', 0) + ) + + def _update(self, fd): + mask = (((fd in self._rfds) and self._readmask) | + ((fd in self._wfds) and select.POLLOUT)) + if mask: + self._pollobj.register(fd, mask) + else: + try: + self._pollobj.unregister(fd) + except KeyError: + pass + + def _poll(self, timeout): + if timeout: + timeout *= 1000 + + events, _ = mitogen.core.io_op(self._pollobj.poll, timeout) + for fd, event in events: + if event & self._readmask: + IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + if event & select.POLLOUT: + IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + class KqueuePoller(mitogen.core.Poller): """ Poller based on the FreeBSD/Darwin kqueue(2) interface. """ + SUPPORTED = hasattr(select, 'kqueue') _repr = 'KqueuePoller()' def __init__(self): @@ -971,6 +1019,7 @@ class EpollPoller(mitogen.core.Poller): """ Poller based on the Linux epoll(2) interface. """ + SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' def __init__(self): @@ -1041,20 +1090,18 @@ class EpollPoller(mitogen.core.Poller): yield data -if sys.version_info < (2, 6): - # 2.4 and 2.5 only had select.select() and select.poll(). - POLLER_BY_SYSNAME = {} -else: - POLLER_BY_SYSNAME = { - 'Darwin': KqueuePoller, - 'FreeBSD': KqueuePoller, - 'Linux': EpollPoller, - } +# 2.4 and 2.5 only had select.select() and select.poll(). +for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: + if _klass.SUPPORTED: + PREFERRED_POLLER = _klass -PREFERRED_POLLER = POLLER_BY_SYSNAME.get( - os.uname()[0], - mitogen.core.Poller, -) +# For apps that start threads dynamically, it's possible Latch will also get +# very high-numbered wait fds when there are many connections, and so select() +# becomes useless there too. So swap in our favourite poller. +if PollPoller.SUPPORTED: + mitogen.core.Latch.poller_class = PollPoller +else: + mitogen.core.Latch.poller_class = PREFERRED_POLLER class DiagLogStream(mitogen.core.BasicStream): diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 8fa8cdb6..102608d4 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -17,6 +17,10 @@ class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy): self.mask = mask +@unittest2.skipIf( + reason='Linux only', + condition=(not os.uname()[0] == 'Linux') +) class FixedPolicyTest(testlib.TestCase): klass = NullFixedPolicy diff --git a/tests/file_service_test.py b/tests/file_service_test.py index 135d8e14..b9034bb1 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -1,4 +1,6 @@ +import sys + import unittest2 import mitogen.service @@ -32,10 +34,15 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): expect = service.unregistered_msg % ('/etc/shadow',) self.assertTrue(expect in e.args[0]) + if sys.platform == 'darwin': + ROOT_GROUP = 'wheel' + else: + ROOT_GROUP = 'root' + def _validate_response(self, resp): self.assertTrue(isinstance(resp, dict)) self.assertEquals('root', resp['owner']) - self.assertEquals('root', resp['group']) + self.assertEquals(self.ROOT_GROUP, resp['group']) self.assertTrue(isinstance(resp['mode'], int)) self.assertTrue(isinstance(resp['mtime'], float)) self.assertTrue(isinstance(resp['atime'], float)) diff --git a/tests/poller_test.py b/tests/poller_test.py index 1d1e0cd0..e2e3cdd7 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -401,16 +401,25 @@ class SelectTest(AllMixin, testlib.TestCase): klass = mitogen.core.Poller SelectTest = unittest2.skipIf( - condition=not hasattr(select, 'select'), + condition=(not SelectTest.klass.SUPPORTED), reason='select.select() not supported' )(SelectTest) +class PollTest(AllMixin, testlib.TestCase): + klass = mitogen.parent.PollPoller + +PollTest = unittest2.skipIf( + condition=(not PollTest.klass.SUPPORTED), + reason='select.poll() not supported' +)(PollTest) + + class KqueueTest(AllMixin, testlib.TestCase): klass = mitogen.parent.KqueuePoller KqueueTest = unittest2.skipIf( - condition=not hasattr(select, 'kqueue'), + condition=(not KqueueTest.klass.SUPPORTED), reason='select.kqueue() not supported' )(KqueueTest) @@ -419,7 +428,7 @@ class EpollTest(AllMixin, testlib.TestCase): klass = mitogen.parent.EpollPoller EpollTest = unittest2.skipIf( - condition=not hasattr(select, 'epoll'), + condition=(not EpollTest.klass.SUPPORTED), reason='select.epoll() not supported' )(EpollTest) diff --git a/tests/testlib.py b/tests/testlib.py index ef401a78..75061b26 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -450,6 +450,8 @@ class DockerMixin(RouterMixin): @classmethod def setUpClass(cls): super(DockerMixin, cls).setUpClass() + if os.environ.get('SKIP_DOCKER_TESTS'): + raise unittest2.SkipTest('SKIP_DOCKER_TESTS is set') cls.dockerized_ssh = DockerizedSshDaemon() cls.dockerized_ssh.wait_for_sshd()