From 458a4faa979cac0e7d4a5431ec1f84da64a5787f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 01:27:49 +0000 Subject: [PATCH 01/49] ansible: create stub __init__.py for sdist. This went into 0.2.5 sdist tarball but it's not checked in. --- ansible_mitogen/compat/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ansible_mitogen/compat/__init__.py diff --git a/ansible_mitogen/compat/__init__.py b/ansible_mitogen/compat/__init__.py new file mode 100644 index 00000000..e69de29b From 0aa4c9d8fc9d2ffe972b65afa79a979b4e6c2b28 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 11:17:52 +0000 Subject: [PATCH 02/49] issue #542: .ci: move some tests to Azure and enable Mac job. --- .ci/azure-pipelines-steps.yml | 20 +++++ .ci/azure-pipelines.yml | 108 ++++++++++++++------------- .ci/ci_lib.py | 12 +++ .ci/mitogen_install.py | 8 +- .ci/mitogen_tests.py | 3 + .ci/prep_azure.py | 38 ++++++---- .travis.yml | 8 +- tests/ansible/tests/affinity_test.py | 4 + tests/file_service_test.py | 9 ++- tests/testlib.py | 2 + 10 files changed, 136 insertions(+), 76 deletions(-) create mode 100644 .ci/azure-pipelines-steps.yml diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml new file mode 100644 index 00000000..a377d795 --- /dev/null +++ b/.ci/azure-pipelines-steps.yml @@ -0,0 +1,20 @@ + +parameters: + name: '' + pool: '' + sign: false + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + +- script: .ci/prep_azure.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_install.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_tests.py + displayName: Run tests. diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index fbbb9640..dc5f7162 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -5,79 +5,85 @@ jobs: -- job: 'MitogenTests' +- job: Mac + steps: + - template: azure-pipelines-steps.yml pool: - vmImage: 'Ubuntu 16.04' + vmImage: macOS-10.13 strategy: matrix: - Mitogen27Debian_27: + Mito27_27: python.version: '2.7' MODE: mitogen - DISTRO: debian - MitogenPy27CentOS6_26: + +- job: Linux + pool: + vmImage: "Ubuntu 16.04" + steps: + - template: azure-pipelines-steps.yml + strategy: + matrix: + # + # Confirmed working + # + Mito27Debian_27: python.version: '2.7' MODE: mitogen - DISTRO: centos6 + DISTRO: debian - #Py26CentOS7: + #MitoPy27CentOS6_26: #python.version: '2.7' #MODE: mitogen #DISTRO: centos6 - Mitogen36CentOS6_26: + Mito36CentOS6_26: python.version: '3.6' MODE: mitogen DISTRO: centos6 - DebOps_2460_27_27: - python.version: '2.7' - MODE: debops_common - VER: 2.4.6.0 - - DebOps_262_36_27: - python.version: '3.6' - MODE: debops_common - VER: 2.6.2 - - Ansible_2460_26: - python.version: '2.7' - MODE: ansible - VER: 2.4.6.0 + # + # + # - Ansible_262_26: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 + #Py26CentOS7: + #python.version: '2.7' + #MODE: mitogen + #DISTRO: centos6 - Ansible_2460_36: - python.version: '3.6' - MODE: ansible - VER: 2.4.6.0 + #DebOps_2460_27_27: + #python.version: '2.7' + #MODE: debops_common + #VER: 2.4.6.0 - Ansible_262_36: - python.version: '3.6' - MODE: ansible - VER: 2.6.2 + #DebOps_262_36_27: + #python.version: '3.6' + #MODE: debops_common + #VER: 2.6.2 - Vanilla_262_27: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 - DISTROS: debian - STRATEGY: linear + #Ansible_2460_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.4.6.0 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + #Ansible_262_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/prep_azure.py - displayName: "Install requirements." + #Ansible_2460_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.4.6.0 - - script: .ci/$(MODE)_install.py - displayName: "Install requirements." + #Ansible_262_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/$(MODE)_tests.py - displayName: Run tests. + #Vanilla_262_27: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 + #DISTROS: debian + #STRATEGY: linear diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 10e9d11e..d4f32f55 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -43,6 +43,18 @@ if not hasattr(subprocess, 'check_output'): subprocess.check_output = subprocess__check_output +# ------------------ + +def have_apt(): + proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + +def have_docker(): + proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + # ----------------- # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 10813b55..72bc75e3 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -6,10 +6,12 @@ batches = [ [ 'pip install "pycparser<2.19" "idna<2.7"', 'pip install -r tests/requirements.txt', - ], - [ - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ] ] +if ci_lib.have_docker(): + batches.append([ + 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), + ]) + ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4ba796c2..36928ac9 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -11,4 +11,7 @@ os.environ.update({ 'SKIP_ANSIBLE': '1', }) +if not ci_lib.have_docker(): + os.environ['SKIP_DOCKER_TESTS'] = '1' + ci_lib.run('./run_tests -v') diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 10126df2..5199a87e 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -1,22 +1,30 @@ #!/usr/bin/env python +import os +import sys + import ci_lib batches = [] -batches.append([ - 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - 'sudo add-apt-repository ppa:deadsnakes/ppa', - 'sudo apt-get update', - 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', -]) - -batches.append([ - 'pip install -r dev_requirements.txt', -]) - -batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] - for distro in ci_lib.DISTROS -) + +if ci_lib.have_apt(): + batches.append([ + 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', + 'sudo add-apt-repository ppa:deadsnakes/ppa', + 'sudo apt-get update', + 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', + ]) + + +#batches.append([ + #'pip install -r dev_requirements.txt', +#]) + +if ci_lib.have_docker(): + batches.extend( + ['docker pull %s' % (ci_lib.image_for_distro(distro),)] + for distro in ci_lib.DISTROS + ) + ci_lib.run_batches(batches) diff --git a/.travis.yml b/.travis.yml index aee14c00..921ad12b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,7 @@ matrix: # 2.4 -> 2.4 - language: c env: MODE=mitogen_py24 DISTRO=centos5 - # 2.7 -> 2.7 - - python: "2.7" - env: MODE=mitogen DISTRO=debian + # 2.7 -> 2.7 -- moved to Azure # 2.7 -> 2.6 #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 @@ -39,9 +37,7 @@ matrix: # 2.6 -> 3.5 - python: "2.6" env: MODE=mitogen DISTRO=debian-py3 - # 3.6 -> 2.6 - - python: "3.6" - env: MODE=mitogen DISTRO=centos6 + # 3.6 -> 2.6 -- moved to Azure # Debops tests. # 2.4.6.0; 2.7 -> 2.7 diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 8fa8cdb6..102608d4 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -17,6 +17,10 @@ class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy): self.mask = mask +@unittest2.skipIf( + reason='Linux only', + condition=(not os.uname()[0] == 'Linux') +) class FixedPolicyTest(testlib.TestCase): klass = NullFixedPolicy diff --git a/tests/file_service_test.py b/tests/file_service_test.py index 135d8e14..b9034bb1 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -1,4 +1,6 @@ +import sys + import unittest2 import mitogen.service @@ -32,10 +34,15 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): expect = service.unregistered_msg % ('/etc/shadow',) self.assertTrue(expect in e.args[0]) + if sys.platform == 'darwin': + ROOT_GROUP = 'wheel' + else: + ROOT_GROUP = 'root' + def _validate_response(self, resp): self.assertTrue(isinstance(resp, dict)) self.assertEquals('root', resp['owner']) - self.assertEquals('root', resp['group']) + self.assertEquals(self.ROOT_GROUP, resp['group']) self.assertTrue(isinstance(resp['mode'], int)) self.assertTrue(isinstance(resp['mtime'], float)) self.assertTrue(isinstance(resp['atime'], float)) diff --git a/tests/testlib.py b/tests/testlib.py index ef401a78..75061b26 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -450,6 +450,8 @@ class DockerMixin(RouterMixin): @classmethod def setUpClass(cls): super(DockerMixin, cls).setUpClass() + if os.environ.get('SKIP_DOCKER_TESTS'): + raise unittest2.SkipTest('SKIP_DOCKER_TESTS is set') cls.dockerized_ssh = DockerizedSshDaemon() cls.dockerized_ssh.wait_for_sshd() From 9bcd2ec56c3162fc5455effa6ffc178c690ca2f1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 12:36:51 +0000 Subject: [PATCH 03/49] issue #542: return of select poller, new selection logic --- mitogen/core.py | 54 ++++++++++++++++---------------- mitogen/parent.py | 73 ++++++++++++++++++++++++++++++++++++-------- tests/poller_test.py | 15 +++++++-- 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 470b00ca..cfdf996b 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1912,6 +1912,8 @@ class Poller(object): Pollers may only be used by one thread at a time. """ + SUPPORTED = True + # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has # no upper FD limit, it is suitable for use with Latch, which must handle # FDs larger than select's limit during many-host runs. We want this @@ -1928,11 +1930,16 @@ class Poller(object): def __init__(self): self._rfds = {} self._wfds = {} - self._pollobj = select.poll() def __repr__(self): return '%s(%#x)' % (type(self).__name__, id(self)) + def _update(self, fd): + """ + Required by PollPoller subclass. + """ + pass + @property def readers(self): """ @@ -1955,20 +1962,6 @@ class Poller(object): """ pass - _readmask = select.POLLIN | select.POLLHUP - # TODO: no proof we dont need writemask too - - def _update(self, fd): - mask = (((fd in self._rfds) and self._readmask) | - ((fd in self._wfds) and select.POLLOUT)) - if mask: - self._pollobj.register(fd, mask) - else: - try: - self._pollobj.unregister(fd) - except KeyError: - pass - def start_receive(self, fd, data=None): """ Cause :meth:`poll` to yield `data` when `fd` is readable. @@ -2004,22 +1997,27 @@ class Poller(object): self._update(fd) def _poll(self, timeout): + (rfds, wfds, _), _ = io_op(select.select, + self._rfds, + self._wfds, + (), timeout + ) + + for fd in rfds: + _vv and IOLOG.debug('%r: POLLIN for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + for fd in wfds: + _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + if timeout: timeout *= 1000 - events, _ = io_op(self._pollobj.poll, timeout) - for fd, event in events: - if event & self._readmask: - _vv and IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd) - data, gen = self._rfds.get(fd, (None, None)) - if gen and gen < self._generation: - yield data - if event & select.POLLOUT: - _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd) - data, gen = self._wfds.get(fd, (None, None)) - if gen and gen < self._generation: - yield data - def poll(self, timeout=None): """ Block the calling thread until one or more FDs are ready for IO. diff --git a/mitogen/parent.py b/mitogen/parent.py index 7e567aaa..f793f234 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -890,10 +890,58 @@ class CallSpec(object): ) +class PollPoller(mitogen.core.Poller): + """ + Poller based on the POSIX poll(2) interface. Not available on some versions + of OS X, otherwise it is the preferred poller for small FD counts. + """ + SUPPORTED = hasattr(select, 'poll') + _repr = 'PollPoller()' + + def __init__(self): + super(PollPoller, self).__init__() + self._pollobj = select.poll() + + # TODO: no proof we dont need writemask too + _readmask = ( + getattr(select, 'POLLIN', 0) | + getattr(select, 'POLLHUP', 0) + ) + + def _update(self, fd): + mask = (((fd in self._rfds) and self._readmask) | + ((fd in self._wfds) and select.POLLOUT)) + if mask: + self._pollobj.register(fd, mask) + else: + try: + self._pollobj.unregister(fd) + except KeyError: + pass + + def _poll(self, timeout): + if timeout: + timeout *= 1000 + + events, _ = mitogen.core.io_op(self._pollobj.poll, timeout) + for fd, event in events: + if event & self._readmask: + IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + if event & select.POLLOUT: + IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + class KqueuePoller(mitogen.core.Poller): """ Poller based on the FreeBSD/Darwin kqueue(2) interface. """ + SUPPORTED = hasattr(select, 'kqueue') _repr = 'KqueuePoller()' def __init__(self): @@ -971,6 +1019,7 @@ class EpollPoller(mitogen.core.Poller): """ Poller based on the Linux epoll(2) interface. """ + SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' def __init__(self): @@ -1041,20 +1090,18 @@ class EpollPoller(mitogen.core.Poller): yield data -if sys.version_info < (2, 6): - # 2.4 and 2.5 only had select.select() and select.poll(). - POLLER_BY_SYSNAME = {} -else: - POLLER_BY_SYSNAME = { - 'Darwin': KqueuePoller, - 'FreeBSD': KqueuePoller, - 'Linux': EpollPoller, - } +# 2.4 and 2.5 only had select.select() and select.poll(). +for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: + if _klass.SUPPORTED: + PREFERRED_POLLER = _klass -PREFERRED_POLLER = POLLER_BY_SYSNAME.get( - os.uname()[0], - mitogen.core.Poller, -) +# For apps that start threads dynamically, it's possible Latch will also get +# very high-numbered wait fds when there are many connections, and so select() +# becomes useless there too. So swap in our favourite poller. +if PollPoller.SUPPORTED: + mitogen.core.Latch.poller_class = PollPoller +else: + mitogen.core.Latch.poller_class = PREFERRED_POLLER class DiagLogStream(mitogen.core.BasicStream): diff --git a/tests/poller_test.py b/tests/poller_test.py index 1d1e0cd0..e2e3cdd7 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -401,16 +401,25 @@ class SelectTest(AllMixin, testlib.TestCase): klass = mitogen.core.Poller SelectTest = unittest2.skipIf( - condition=not hasattr(select, 'select'), + condition=(not SelectTest.klass.SUPPORTED), reason='select.select() not supported' )(SelectTest) +class PollTest(AllMixin, testlib.TestCase): + klass = mitogen.parent.PollPoller + +PollTest = unittest2.skipIf( + condition=(not PollTest.klass.SUPPORTED), + reason='select.poll() not supported' +)(PollTest) + + class KqueueTest(AllMixin, testlib.TestCase): klass = mitogen.parent.KqueuePoller KqueueTest = unittest2.skipIf( - condition=not hasattr(select, 'kqueue'), + condition=(not KqueueTest.klass.SUPPORTED), reason='select.kqueue() not supported' )(KqueueTest) @@ -419,7 +428,7 @@ class EpollTest(AllMixin, testlib.TestCase): klass = mitogen.parent.EpollPoller EpollTest = unittest2.skipIf( - condition=not hasattr(select, 'epoll'), + condition=(not EpollTest.klass.SUPPORTED), reason='select.epoll() not supported' )(EpollTest) From 2cde51ea631cd0a67bcdadaa01065cdbfeb6a571 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 12:54:09 +0000 Subject: [PATCH 04/49] docs: update Changelog; closes #542. --- docs/changelog.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f7ecadbd..887a9e85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,6 +125,26 @@ Core Library series. +v0.2.6 (2019-02-??) +------------------- + +Fixes +~~~~~ + +* `#542 `_: some versions of OS X + ship a default Python that does not support :func:`select.poll`. Restore the + 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer + :func:`select.poll` if it is available. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +bug reports, testing, features and fixes in this release contributed by +`Petr Enkov `_. + + v0.2.5 (2019-02-14) ------------------- From d865fb797d3a86d49475caf9f5ec87bd04a3b5be Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 13:35:40 +0000 Subject: [PATCH 05/49] docs: change 'unreleased' Changelog format and add a hint. --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 887a9e85..222fee63 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,9 +125,12 @@ Core Library series. -v0.2.6 (2019-02-??) +v0.2.6 (unreleased) ------------------- +To avail of fixes in an unreleased version, please download a ZIP file +`directly from GitHub `_. + Fixes ~~~~~ From ca63c26e0141fcf9a2d4018525a7182d7b7a3653 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 14:58:55 +0000 Subject: [PATCH 06/49] core: Make Latch.put(obj=) optional. --- mitogen/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index cfdf996b..dcca2e91 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2240,11 +2240,14 @@ class Latch(object): finally: self._lock.release() - def put(self, obj): + def put(self, obj=None): """ Enqueue an object, waking the first thread waiting for a result, if one exists. + :param obj: + Object to enqueue. Defaults to :data:`None` as a convenience when + using :class:`Latch` only for synchronization. :raises mitogen.core.LatchError: :meth:`close` has been called, and the object is no longer valid. """ From 14e6c6e49ee5321023605cbe0f2176bebe829f88 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 15:00:17 +0000 Subject: [PATCH 07/49] docs: update Changelog. --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 222fee63..169e4095 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -139,6 +139,12 @@ Fixes 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer :func:`select.poll` if it is available. +Core Library +~~~~~~~~~~~~ + +* `ca63c26e `_: + :meth:`mitogen.core.Latch.put`'s `obj` argument was made optional. + Thanks! ~~~~~~~ From c0db283ac7e02723499b55ed7675252958e7b431 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 21:21:45 +0000 Subject: [PATCH 08/49] docs: update copyright year. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 3708a943..a6bc2cbc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,7 +6,7 @@ import mitogen VERSION = '%s.%s.%s' % mitogen.__version__ author = u'David Wilson' -copyright = u'2018, David Wilson' +copyright = u'2019, David Wilson' exclude_patterns = ['_build'] extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput'] html_show_sourcelink = False From ffdf31edd74babcc4dbb183b90554c04f187fbbe Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 21:21:56 +0000 Subject: [PATCH 09/49] tests/bench: set process affinity in throughput.py. --- tests/bench/throughput.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/bench/throughput.py b/tests/bench/throughput.py index 896ee9ac..42604826 100644 --- a/tests/bench/throughput.py +++ b/tests/bench/throughput.py @@ -9,6 +9,7 @@ import time import mitogen import mitogen.service +import ansible_mitogen.affinity def prepare(): @@ -45,6 +46,8 @@ def run_test(router, fp, s, context): @mitogen.main() def main(router): + ansible_mitogen.affinity.policy.assign_muxprocess() + bigfile = tempfile.NamedTemporaryFile() fill_with_random(bigfile, 1048576*512) From e517810e5a3b1826e8ec1f53160345b8847c0a18 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 15 Feb 2019 12:18:28 +0000 Subject: [PATCH 10/49] tests: ensure serialization restrictions are in effect --- tests/serialization_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/serialization_test.py b/tests/serialization_test.py index 23c4a2d9..6cf5f8b7 100644 --- a/tests/serialization_test.py +++ b/tests/serialization_test.py @@ -8,11 +8,23 @@ from mitogen.core import b import testlib +class EvilObject(object): + pass + + def roundtrip(v): msg = mitogen.core.Message.pickled(v) return mitogen.core.Message(data=msg.data).unpickle() +class EvilObjectTest(testlib.TestCase): + def test_deserialization_fails(self): + msg = mitogen.core.Message.pickled(EvilObject()) + e = self.assertRaises(mitogen.core.StreamError, + lambda: msg.unpickle() + ) + + class BlobTest(testlib.TestCase): klass = mitogen.core.Blob From 7d0480e8bd410bd589328785f98457cf3f86bb55 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 16 Feb 2019 16:53:49 +0000 Subject: [PATCH 11/49] core: increase cookie field lengths to 64-bit; closes #545. --- docs/changelog.rst | 7 +++++++ mitogen/core.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 169e4095..157ed902 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -139,6 +139,12 @@ Fixes 0.2.3 behaviour of defaulting to Kqueue in this case, but still prefer :func:`select.poll` if it is available. +* `#545 `_: an optimization + introduced in `#493 `_ caused a + 64-bit integer to be assigned to a 32-bit field on ARM 32-bit targets, + causing runs to fail. + + Core Library ~~~~~~~~~~~~ @@ -151,6 +157,7 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for bug reports, testing, features and fixes in this release contributed by +`Fabian Arrotin `_, and `Petr Enkov `_. diff --git a/mitogen/core.py b/mitogen/core.py index dcca2e91..25732896 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2140,7 +2140,7 @@ class Latch(object): return rsock, wsock COOKIE_MAGIC, = struct.unpack('L', b('LTCH') * (struct.calcsize('L')//4)) - COOKIE_FMT = 'Llll' + COOKIE_FMT = '>Qqqq' # #545: id() and get_ident() may exceed long on armhfp. COOKIE_SIZE = struct.calcsize(COOKIE_FMT) def _make_cookie(self): From b3f592acee44ffc469d2773a69bd8ddf0f2e2902 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 17 Feb 2019 01:41:40 +0000 Subject: [PATCH 12/49] issue #535: core/select: support selecting from Latches. --- mitogen/core.py | 4 + mitogen/select.py | 86 ++++++++++++++++---- tests/select_test.py | 189 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 254 insertions(+), 25 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 25732896..6bca1b1a 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2051,6 +2051,8 @@ class Latch(object): """ poller_class = Poller + notify = None + # The _cls_ prefixes here are to make it crystal clear in the code which # state mutation isn't covered by :attr:`_lock`. @@ -2264,6 +2266,8 @@ class Latch(object): _vv and IOLOG.debug('%r.put() -> waking wfd=%r', self, wsock.fileno()) self._wake(wsock, cookie) + elif self.notify: + self.notify(self) finally: self._lock.release() diff --git a/mitogen/select.py b/mitogen/select.py index fd2cbe9a..651dcb5f 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -35,12 +35,25 @@ class Error(mitogen.core.Error): pass +class Event(object): + """ + Represents one selected event. + """ + #: The first Receiver or Latch the event traversed. + source = None + + #: The :class:`mitogen.core.Message` delivered to a receiver, or the object + #: posted to a latch. + data = None + + class Select(object): """ Support scatter/gather asynchronous calls and waiting on multiple - receivers, channels, and sub-Selects. Accepts a sequence of - :class:`mitogen.core.Receiver` or :class:`mitogen.select.Select` instances - and returns the first value posted to any receiver or select. + receivers, channels, latches, and sub-Selects. Accepts a sequence of + :class:`mitogen.core.Receiver`, :class:`mitogen.select.Select` or + :class:`mitogen.core.Latch` instances and returns the first value posted to + any receiver or select. If `oneshot` is :data:`True`, then remove each receiver as it yields a result; since :meth:`__iter__` terminates once the final receiver is @@ -84,6 +97,19 @@ class Select(object): for msg in mitogen.select.Select(selects): print(msg.unpickle()) + + :class:`Select` may be used to mix inter-thread and inter-process IO: + + latch = mitogen.core.Latch() + start_thread(latch) + recv = remote_host.call_async(os.getuid) + + sel = Select([latch, recv]) + event = sel.get_event() + if event.source is latch: + # woken by a local thread + else: + # woken by function call result """ notify = None @@ -145,14 +171,29 @@ class Select(object): def __exit__(self, e_type, e_val, e_tb): self.close() - def __iter__(self): + def iter_data(self): """ - Yield the result of :meth:`get` until no receivers remain in the - select, either because `oneshot` is :data:`True`, or each receiver was + Yield :attr:`Event.data` until no receivers remain in the select, + either because `oneshot` is :data:`True`, or each receiver was explicitly removed via :meth:`remove`. + + :meth:`__iter__` is an alias for :meth:`iter_data`, allowing loops + like:: + + for msg in Select([recv1, recv2]): + print msg.unpickle() """ while self._receivers: - yield self.get() + yield self.get_event().data + + __iter__ = iter_data + + def iter_events(self): + """ + Yield :class:`Event` instances until no receivers remain in the select. + """ + while self._receivers: + yield self.get_event() loop_msg = 'Adding this Select instance would create a Select cycle' @@ -170,8 +211,8 @@ class Select(object): def add(self, recv): """ - Add the :class:`mitogen.core.Receiver` or :class:`Select` `recv` to the - select. + Add a :class:`mitogen.core.Receiver`, :class:`Select` or + :class:`mitogen.core.Latch` to the select. :raises mitogen.select.Error: An attempt was made to add a :class:`Select` to which this select @@ -193,10 +234,9 @@ class Select(object): def remove(self, recv): """ - Remove the :class:`mitogen.core.Receiver` or :class:`Select` `recv` - from the select. Note that if the receiver has notified prior to - :meth:`remove`, it will still be returned by a subsequent :meth:`get`. - This may change in a future version. + Remove an object from from the select. Note that if the receiver has + notified prior to :meth:`remove`, it will still be returned by a + subsequent :meth:`get`. This may change in a future version. """ try: if recv.notify != self._put: @@ -239,6 +279,13 @@ class Select(object): empty_msg = 'Cannot get(), Select instance is empty' def get(self, timeout=None, block=True): + """ + Call `get_event(timeout, block)` returning :attr:`Event.data` of the + first available event. + """ + return self.get_event(timeout, block).data + + def get_event(self, timeout=None, block=True): """ Fetch the next available value from any receiver, or raise :class:`mitogen.core.TimeoutError` if no value is available within @@ -263,14 +310,21 @@ class Select(object): if not self._receivers: raise Error(self.empty_msg) + event = Event() while True: recv = self._latch.get(timeout=timeout, block=block) try: - msg = recv.get(block=False) + if isinstance(recv, Select): + event = recv.get_event(block=False) + else: + event.source = recv + event.data = recv.get(block=False) if self._oneshot: self.remove(recv) - msg.receiver = recv - return msg + if isinstance(recv, mitogen.core.Receiver): + # Remove in 0.3.x. + event.data.receiver = recv + return event except mitogen.core.TimeoutError: # A receiver may have been queued with no result if another # thread drained it before we woke up, or because another diff --git a/tests/select_test.py b/tests/select_test.py index 7e6256c8..f08c9f3a 100644 --- a/tests/select_test.py +++ b/tests/select_test.py @@ -9,6 +9,18 @@ import testlib class BoolTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select + def test_latch(self): + latch = mitogen.core.Latch() # oneshot + select = self.klass() + self.assertFalse(select) + select.add(latch) + self.assertTrue(select) + + latch.put(123) + self.assertTrue(select) + self.assertEquals(123, select.get()) + self.assertFalse(select) + def test_receiver(self): recv = mitogen.core.Receiver(self.router) # oneshot select = self.klass() @@ -25,6 +37,14 @@ class BoolTest(testlib.RouterMixin, testlib.TestCase): class AddTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select + def test_latch(self): + latch = mitogen.core.Latch() + select = self.klass() + select.add(latch) + self.assertEquals(1, len(select._receivers)) + self.assertEquals(latch, select._receivers[0]) + self.assertEquals(select._put, latch.notify) + def test_receiver(self): recv = mitogen.core.Receiver(self.router) select = self.klass() @@ -98,14 +118,14 @@ class AddTest(testlib.RouterMixin, testlib.TestCase): class RemoveTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select - def test_empty(self): + def test_receiver_empty(self): select = self.klass() recv = mitogen.core.Receiver(self.router) exc = self.assertRaises(mitogen.select.Error, lambda: select.remove(recv)) self.assertEquals(str(exc), self.klass.not_present_msg) - def test_absent(self): + def test_receiver_absent(self): select = self.klass() recv = mitogen.core.Receiver(self.router) recv2 = mitogen.core.Receiver(self.router) @@ -114,7 +134,7 @@ class RemoveTest(testlib.RouterMixin, testlib.TestCase): lambda: select.remove(recv)) self.assertEquals(str(exc), self.klass.not_present_msg) - def test_present(self): + def test_receiver_present(self): select = self.klass() recv = mitogen.core.Receiver(self.router) select.add(recv) @@ -122,6 +142,30 @@ class RemoveTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(0, len(select._receivers)) self.assertEquals(None, recv.notify) + def test_latch_empty(self): + select = self.klass() + latch = mitogen.core.Latch() + exc = self.assertRaises(mitogen.select.Error, + lambda: select.remove(latch)) + self.assertEquals(str(exc), self.klass.not_present_msg) + + def test_latch_absent(self): + select = self.klass() + latch = mitogen.core.Latch() + latch2 = mitogen.core.Latch() + select.add(latch2) + exc = self.assertRaises(mitogen.select.Error, + lambda: select.remove(latch)) + self.assertEquals(str(exc), self.klass.not_present_msg) + + def test_latch_present(self): + select = self.klass() + latch = mitogen.core.Latch() + select.add(latch) + select.remove(latch) + self.assertEquals(0, len(select._receivers)) + self.assertEquals(None, latch.notify) + class CloseTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select @@ -130,6 +174,18 @@ class CloseTest(testlib.RouterMixin, testlib.TestCase): select = self.klass() select.close() # No effect. + def test_one_latch(self): + select = self.klass() + latch = mitogen.core.Latch() + select.add(latch) + + self.assertEquals(1, len(select._receivers)) + self.assertEquals(select._put, latch.notify) + + select.close() + self.assertEquals(0, len(select._receivers)) + self.assertEquals(None, latch.notify) + def test_one_receiver(self): select = self.klass() recv = mitogen.core.Receiver(self.router) @@ -174,18 +230,35 @@ class EmptyTest(testlib.RouterMixin, testlib.TestCase): select = self.klass([recv]) self.assertTrue(select.empty()) - def test_nonempty_before_add(self): + def test_nonempty_receiver_before_add(self): recv = mitogen.core.Receiver(self.router) recv._on_receive(mitogen.core.Message.pickled('123')) select = self.klass([recv]) self.assertFalse(select.empty()) - def test_nonempty_after_add(self): + def test_nonempty__receiver_after_add(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) recv._on_receive(mitogen.core.Message.pickled('123')) self.assertFalse(select.empty()) + def test_empty_latch(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + self.assertTrue(select.empty()) + + def test_nonempty_latch_before_add(self): + latch = mitogen.core.Latch() + latch.put(123) + select = self.klass([latch]) + self.assertFalse(select.empty()) + + def test_nonempty__latch_after_add(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + latch.put(123) + self.assertFalse(select.empty()) + class IterTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select @@ -194,18 +267,24 @@ class IterTest(testlib.RouterMixin, testlib.TestCase): select = self.klass() self.assertEquals([], list(select)) - def test_nonempty(self): + def test_nonempty_receiver(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) msg = mitogen.core.Message.pickled('123') recv._on_receive(msg) self.assertEquals([msg], list(select)) + def test_nonempty_latch(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + latch.put(123) + self.assertEquals([123], list(select)) + class OneShotTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select - def test_true_removed_after_get(self): + def test_true_receiver_removed_after_get(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) msg = mitogen.core.Message.pickled('123') @@ -215,7 +294,7 @@ class OneShotTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(0, len(select._receivers)) self.assertEquals(None, recv.notify) - def test_false_persists_after_get(self): + def test_false_receiver_persists_after_get(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv], oneshot=False) msg = mitogen.core.Message.pickled('123') @@ -226,8 +305,26 @@ class OneShotTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(recv, select._receivers[0]) self.assertEquals(select._put, recv.notify) + def test_true_latch_removed_after_get(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + latch.put(123) + self.assertEquals(123, select.get()) + self.assertEquals(0, len(select._receivers)) + self.assertEquals(None, latch.notify) -class GetTest(testlib.RouterMixin, testlib.TestCase): + def test_false_latch_persists_after_get(self): + latch = mitogen.core.Latch() + select = self.klass([latch], oneshot=False) + latch.put(123) + + self.assertEquals(123, select.get()) + self.assertEquals(1, len(select._receivers)) + self.assertEquals(latch, select._receivers[0]) + self.assertEquals(select._put, latch.notify) + + +class GetReceiverTest(testlib.RouterMixin, testlib.TestCase): klass = mitogen.select.Select def test_no_receivers(self): @@ -285,5 +382,79 @@ class GetTest(testlib.RouterMixin, testlib.TestCase): lambda: select.get(timeout=0.0)) +class GetLatchTest(testlib.RouterMixin, testlib.TestCase): + klass = mitogen.select.Select + + def test_no_latches(self): + select = self.klass() + exc = self.assertRaises(mitogen.select.Error, + lambda: select.get()) + self.assertEquals(str(exc), self.klass.empty_msg) + + def test_timeout_no_receivers(self): + select = self.klass() + exc = self.assertRaises(mitogen.select.Error, + lambda: select.get(timeout=1.0)) + self.assertEquals(str(exc), self.klass.empty_msg) + + def test_zero_timeout(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + self.assertRaises(mitogen.core.TimeoutError, + lambda: select.get(timeout=0.0)) + + def test_timeout(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + self.assertRaises(mitogen.core.TimeoutError, + lambda: select.get(timeout=0.1)) + + def test_nonempty_before_add(self): + latch = mitogen.core.Latch() + latch.put(123) + select = self.klass([latch]) + self.assertEquals(123, select.get()) + + def test_nonempty_after_add(self): + latch = mitogen.core.Latch() + select = self.klass([latch]) + latch.put(123) + self.assertEquals(123, latch.get()) + + def test_drained_by_other_thread(self): + latch = mitogen.core.Latch() + latch.put(123) + select = self.klass([latch]) + self.assertEquals(123, latch.get()) + self.assertRaises(mitogen.core.TimeoutError, + lambda: select.get(timeout=0.0)) + + +class GetEventTest(testlib.RouterMixin, testlib.TestCase): + klass = mitogen.select.Select + + def test_empty(self): + select = self.klass() + exc = self.assertRaises(mitogen.select.Error, + lambda: select.get()) + self.assertEquals(str(exc), self.klass.empty_msg) + + def test_latch(self): + latch = mitogen.core.Latch() + latch.put(123) + select = self.klass([latch]) + event = select.get_event() + self.assertEquals(latch, event.source) + self.assertEquals(123, event.data) + + def test_receiver(self): + recv = mitogen.core.Receiver(self.router) + recv._on_receive(mitogen.core.Message.pickled('123')) + select = self.klass([recv]) + event = select.get_event() + self.assertEquals(recv, event.source) + self.assertEquals('123', event.data.unpickle()) + + if __name__ == '__main__': unittest2.main() From 28aa8b3b278c173163cf68b6781f5059deedc182 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 17 Feb 2019 01:44:46 +0000 Subject: [PATCH 13/49] issue #535: docs: update Changelog. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 157ed902..8a01c033 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -151,6 +151,11 @@ Core Library * `ca63c26e `_: :meth:`mitogen.core.Latch.put`'s `obj` argument was made optional. +* `2c921fea `_: to support + function calls on a service pool from another thread, + :class:`mitogen.select.Select` additionally permits waiting on + :class:`mitogen.core.Latch`. + Thanks! ~~~~~~~ From 72862f0bb90dbc45988b1c135f181d07a37d415d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 17 Feb 2019 04:49:51 +0000 Subject: [PATCH 14/49] issue #535: docs: fix up Select doc --- docs/api.rst | 4 ++++ mitogen/select.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 3fd70bea..2502766f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -579,6 +579,10 @@ Select Class .. module:: mitogen.select .. currentmodule:: mitogen.select + +.. autoclass:: Event + :members: + .. autoclass:: Select :members: diff --git a/mitogen/select.py b/mitogen/select.py index 651dcb5f..51aebc22 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -50,10 +50,10 @@ class Event(object): class Select(object): """ Support scatter/gather asynchronous calls and waiting on multiple - receivers, channels, latches, and sub-Selects. Accepts a sequence of - :class:`mitogen.core.Receiver`, :class:`mitogen.select.Select` or - :class:`mitogen.core.Latch` instances and returns the first value posted to - any receiver or select. + :class:`receivers `, + :class:`channels `, + :class:`latches `, and + :class:`sub-selects