Merge remote-tracking branch 'origin/026' into stable
* origin/026: docs: update Changelog for release. Bump version for release. issue #555: ansible: workaround ancient reload(sys) hack. issue #554: mitogen_action_script fix issue #554: fix Ansible 2.4 compatibility issue #554: don't rely on tmp_path autoremoval in test. issue #554: track and remove multiple make_tmp_path() calls. docs: update Changelog. docs: drastically simplify install/changelog. issue #552: include process identity in log messages. issue #550: update Changelog. issue #550: parent: add explanatory comment. issue #550: fix up TTY ioctls on WSL 2016 Anniversary Update docs: update Changelog. service: make service list optional. docs: update Changelog; closes #548. issue #548: always treat transport=smart as 'ssh' for mitogen_via=. docs: better intro paragraph. .ci: copy private key file to tempdir. os_fork: more doc tweaks os_fork: more doc tweaks os_fork: yet more doc tidyup os_fork: more doc tweaks os_fork: clean up docs .ci: import soak scripts. .ci: allow containers for different jobs to run simultaneously os_fork: python 3 fixes and tests. issue #535: activate Corker on 2.4 in master too. issue #535: update Changelog. issue #535: wire mitogen.os_fork into Broker and Pool. issue #535: parent: add create_socketpair(size=..) parameter. issue #535: introduce mitogen.os_fork module and Corker class. issue #535: docs: update Changelog issue #535: service: support Pool.defer() like Broker.defer() issue #535: core: unicode.encode() may take importer lock on 2.x issue #535: docs: fix up Select doc issue #535: docs: update Changelog. issue #535: core/select: support selecting from Latches. core: increase cookie field lengths to 64-bit; closes #545. tests: ensure serialization restrictions are in effect tests/bench: set process affinity in throughput.py. docs: update copyright year. docs: update Changelog. core: Make Latch.put(obj=) optional. docs: change 'unreleased' Changelog format and add a hint. docs: update Changelog; closes #542. issue #542: return of select poller, new selection logic issue #542: .ci: move some tests to Azure and enable Mac job. ansible: create stub __init__.py for sdist.pull/862/head v0.2.6
David Wilson
7 years ago
commit
407307adf6
@ -0,0 +1,20 @@
|
||||
|
||||
parameters:
|
||||
name: ''
|
||||
pool: ''
|
||||
sign: false
|
||||
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
architecture: 'x64'
|
||||
|
||||
- script: .ci/prep_azure.py
|
||||
displayName: "Install requirements."
|
||||
|
||||
- script: .ci/$(MODE)_install.py
|
||||
displayName: "Install requirements."
|
||||
|
||||
- script: .ci/$(MODE)_tests.py
|
||||
displayName: Run tests.
|
||||
@ -1,22 +1,30 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import ci_lib
|
||||
|
||||
batches = []
|
||||
batches.append([
|
||||
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
|
||||
'sudo add-apt-repository ppa:deadsnakes/ppa',
|
||||
'sudo apt-get update',
|
||||
'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
|
||||
])
|
||||
|
||||
batches.append([
|
||||
'pip install -r dev_requirements.txt',
|
||||
])
|
||||
|
||||
batches.extend(
|
||||
['docker pull %s' % (ci_lib.image_for_distro(distro),)]
|
||||
for distro in ci_lib.DISTROS
|
||||
)
|
||||
|
||||
if ci_lib.have_apt():
|
||||
batches.append([
|
||||
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
|
||||
'sudo add-apt-repository ppa:deadsnakes/ppa',
|
||||
'sudo apt-get update',
|
||||
'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
|
||||
])
|
||||
|
||||
|
||||
#batches.append([
|
||||
#'pip install -r dev_requirements.txt',
|
||||
#])
|
||||
|
||||
if ci_lib.have_docker():
|
||||
batches.extend(
|
||||
['docker pull %s' % (ci_lib.image_for_distro(distro),)]
|
||||
for distro in ci_lib.DISTROS
|
||||
)
|
||||
|
||||
|
||||
ci_lib.run_batches(batches)
|
||||
|
||||
@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
|
||||
export NOCOVERAGE=1
|
||||
|
||||
# Make Docker containers once.
|
||||
/usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
|
||||
export KEEP=1
|
||||
|
||||
i=0
|
||||
while :
|
||||
do
|
||||
i=$((i + 1))
|
||||
/usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
|
||||
done
|
||||
|
||||
echo $i
|
||||
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
export NOCOVERAGE=1
|
||||
export DISTROS="debian*4"
|
||||
|
||||
# Make Docker containers once.
|
||||
/usr/bin/time -v ./.ci/ansible_tests.py "$@"
|
||||
export KEEP=1
|
||||
|
||||
i=0
|
||||
while :
|
||||
do
|
||||
i=$((i + 1))
|
||||
/usr/bin/time -v ./.ci/ansible_tests.py "$@" || break
|
||||
done
|
||||
|
||||
echo $i
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
export NOCOVERAGE=1
|
||||
|
||||
i=0
|
||||
while :
|
||||
do
|
||||
i=$((i + 1))
|
||||
/usr/bin/time -v ./.ci/mitogen_py24_tests.py "$@" || break
|
||||
done
|
||||
|
||||
echo $i
|
||||
@ -0,0 +1,183 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Support for operating in a mixed threading/forking environment.
|
||||
"""
|
||||
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import weakref
|
||||
|
||||
import mitogen.core
|
||||
|
||||
|
||||
# List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this
|
||||
# list and mitogen.service registers its Pool too.
|
||||
_brokers = weakref.WeakKeyDictionary()
|
||||
_pools = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def _notice_broker_or_pool(obj):
|
||||
"""
|
||||
Used by :mod:`mitogen.core` and :mod:`mitogen.service` to automatically
|
||||
register every broker and pool on Python 2.4/2.5.
|
||||
"""
|
||||
if isinstance(obj, mitogen.core.Broker):
|
||||
_brokers[obj] = True
|
||||
else:
|
||||
_pools[obj] = True
|
||||
|
||||
|
||||
def wrap_os__fork():
|
||||
corker = Corker(
|
||||
brokers=list(_brokers),
|
||||
pools=list(_pools),
|
||||
)
|
||||
try:
|
||||
corker.cork()
|
||||
return os__fork()
|
||||
finally:
|
||||
corker.uncork()
|
||||
|
||||
|
||||
# If Python 2.4/2.5 where threading state is not fixed up, subprocess.Popen()
|
||||
# may still deadlock due to the broker thread. In this case, pause os.fork() so
|
||||
# that all active threads are paused during fork.
|
||||
if sys.version_info < (2, 6):
|
||||
os__fork = os.fork
|
||||
os.fork = wrap_os__fork
|
||||
|
||||
|
||||
class Corker(object):
|
||||
"""
|
||||
Arrange for :class:`mitogen.core.Broker` and optionally
|
||||
:class:`mitogen.service.Pool` to be temporarily "corked" while fork
|
||||
operations may occur.
|
||||
|
||||
In a mixed threading/forking environment, it is critical no threads are
|
||||
active at the moment of fork, as they could hold mutexes whose state is
|
||||
unrecoverably snapshotted in the locked state in the fork child, causing
|
||||
deadlocks at random future moments.
|
||||
|
||||
To ensure a target thread has all locks dropped, it is made to write a
|
||||
large string to a socket with a small buffer that has :data:`os.O_NONBLOCK`
|
||||
disabled. CPython will drop the GIL and enter the ``write()`` system call,
|
||||
where it will block until the socket buffer is drained, or the write side
|
||||
is closed.
|
||||
|
||||
:class:`mitogen.core.Poller` is used to ensure the thread really has
|
||||
blocked outside any Python locks, by checking if the socket buffer has
|
||||
started to fill.
|
||||
|
||||
Since this necessarily involves posting a message to every existent thread
|
||||
and verifying acknowledgement, it will never be a fast operation.
|
||||
|
||||
This does not yet handle the case of corking being initiated from within a
|
||||
thread that is also a cork target.
|
||||
|
||||
:param brokers:
|
||||
Sequence of :class:`mitogen.core.Broker` instances to cork.
|
||||
:param pools:
|
||||
Sequence of :class:`mitogen.core.Pool` instances to cork.
|
||||
"""
|
||||
def __init__(self, brokers=(), pools=()):
|
||||
self.brokers = brokers
|
||||
self.pools = pools
|
||||
|
||||
def _do_cork(self, s, wsock):
|
||||
try:
|
||||
try:
|
||||
while True:
|
||||
# at least EINTR is possible. Do our best to keep handling
|
||||
# outside the GIL in this case using sendall().
|
||||
wsock.sendall(s)
|
||||
except socket.error:
|
||||
pass
|
||||
finally:
|
||||
wsock.close()
|
||||
|
||||
def _cork_one(self, s, obj):
|
||||
"""
|
||||
Construct a socketpair, saving one side of it, and passing the other to
|
||||
`obj` to be written to by one of its threads.
|
||||
"""
|
||||
rsock, wsock = mitogen.parent.create_socketpair(size=4096)
|
||||
mitogen.core.set_cloexec(rsock.fileno())
|
||||
mitogen.core.set_cloexec(wsock.fileno())
|
||||
mitogen.core.set_block(wsock) # gevent
|
||||
self._rsocks.append(rsock)
|
||||
obj.defer(self._do_cork, s, wsock)
|
||||
|
||||
def _verify_one(self, rsock):
|
||||
"""
|
||||
Pause until the socket `rsock` indicates readability, due to
|
||||
:meth:`_do_cork` triggering a blocking write on another thread.
|
||||
"""
|
||||
poller = mitogen.core.Poller()
|
||||
poller.start_receive(rsock.fileno())
|
||||
try:
|
||||
while True:
|
||||
for fd in poller.poll():
|
||||
return
|
||||
finally:
|
||||
poller.close()
|
||||
|
||||
def cork(self):
|
||||
"""
|
||||
Arrange for any associated brokers and pools to be paused with no locks
|
||||
held. This will not return until each thread acknowledges it has ceased
|
||||
execution.
|
||||
"""
|
||||
s = mitogen.core.b('CORK') * ((128 // 4) * 1024)
|
||||
self._rsocks = []
|
||||
|
||||
# Pools must be paused first, as existing work may require the
|
||||
# participation of a broker in order to complete.
|
||||
for pool in self.pools:
|
||||
if not pool.closed:
|
||||
for x in range(pool.size):
|
||||
self._cork_one(s, pool)
|
||||
|
||||
for broker in self.brokers:
|
||||
if broker._alive:
|
||||
self._cork_one(s, broker)
|
||||
|
||||
# Pause until we can detect every thread has entered write().
|
||||
for rsock in self._rsocks:
|
||||
self._verify_one(rsock)
|
||||
|
||||
def uncork(self):
|
||||
"""
|
||||
Arrange for paused threads to resume operation.
|
||||
"""
|
||||
for rsock in self._rsocks:
|
||||
rsock.close()
|
||||
@ -0,0 +1,20 @@
|
||||
# issue #554: double calls to make_tmp_path() fail with assertion error. Ensure
|
||||
# they succeed and are cleaned up correctly.
|
||||
|
||||
- hosts: target
|
||||
tasks:
|
||||
- mitogen_action_script:
|
||||
script: |
|
||||
result['t1'] = self._make_tmp_path()
|
||||
result['t2'] = self._make_tmp_path()
|
||||
assert result['t1'] != result['t2']
|
||||
assert self._remote_file_exists(result['t1'])
|
||||
assert self._remote_file_exists(result['t2'])
|
||||
self._remove_tmp_path(result['t1'])
|
||||
self._remove_tmp_path(result['t2'])
|
||||
register: out
|
||||
|
||||
- mitogen_action_script:
|
||||
script: |
|
||||
assert not self._remote_file_exists("{{ out.t1 }}")
|
||||
assert not self._remote_file_exists("{{ out.t2 }}")
|
||||
@ -0,0 +1,10 @@
|
||||
# issue #555
|
||||
|
||||
- name: integration/runner/custom_python_prehistoric_module.yml
|
||||
hosts: test-targets
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- custom_python_prehistoric_module:
|
||||
register: out
|
||||
|
||||
- assert: that=out.ok
|
||||
@ -0,0 +1,49 @@
|
||||
# issue #548
|
||||
# Each case is followed by mitogen_via= case to test hostvars method.
|
||||
|
||||
|
||||
# When no ansible_connection= is set, transport comes via ansible.cfg ("smart"
|
||||
# is parsed away to either paramiko or ssh).
|
||||
- name: integration/transport_config/transport.yml
|
||||
hosts: tc-transport-smart
|
||||
tasks:
|
||||
- include: ../_mitogen_only.yml
|
||||
- {mitogen_get_stack: {}, register: out}
|
||||
- assert_equal:
|
||||
left: out.result[0].method
|
||||
right: "ssh"
|
||||
|
||||
- hosts: tc-transport-local
|
||||
vars: {mitogen_via: tc-transport-smart}
|
||||
tasks:
|
||||
- include: ../_mitogen_only.yml
|
||||
- {mitogen_get_stack: {}, register: out}
|
||||
- assert_equal:
|
||||
left: out.result[0].method
|
||||
right: "ssh"
|
||||
- assert_equal:
|
||||
left: out.result[1].method
|
||||
right: "local"
|
||||
|
||||
|
||||
# ansible_connection=local
|
||||
|
||||
- hosts: tc-transport-local
|
||||
tasks:
|
||||
- include: ../_mitogen_only.yml
|
||||
- {mitogen_get_stack: {}, register: out}
|
||||
- assert_equal:
|
||||
left: out.result[0].method
|
||||
right: "local"
|
||||
|
||||
- hosts: tc-transport-smart
|
||||
vars: {mitogen_via: tc-transport-local}
|
||||
tasks:
|
||||
- include: ../_mitogen_only.yml
|
||||
- {mitogen_get_stack: {}, register: out}
|
||||
- assert_equal:
|
||||
left: out.result[0].method
|
||||
right: "local"
|
||||
- assert_equal:
|
||||
left: out.result[1].method
|
||||
right: "ssh"
|
||||
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/python
|
||||
# issue #555: I'm a module that cutpastes an old hack.
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.basic import get_module_path
|
||||
from ansible.module_utils import six
|
||||
|
||||
import os
|
||||
import pwd
|
||||
import socket
|
||||
import sys
|
||||
|
||||
import sys
|
||||
reload(sys)
|
||||
sys.setdefaultencoding('utf8')
|
||||
|
||||
|
||||
def main():
|
||||
module = AnsibleModule(argument_spec={})
|
||||
module.exit_json(ok=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,52 @@
|
||||
|
||||
import testlib
|
||||
import unittest2
|
||||
|
||||
import mitogen.os_fork
|
||||
import mitogen.service
|
||||
|
||||
|
||||
class CorkTest(testlib.RouterMixin, testlib.TestCase):
|
||||
klass = mitogen.os_fork.Corker
|
||||
|
||||
def ping(self, latch):
|
||||
latch.put('pong')
|
||||
|
||||
def test_cork_broker(self):
|
||||
latch = mitogen.core.Latch()
|
||||
self.broker.defer(self.ping, latch)
|
||||
self.assertEquals('pong', latch.get())
|
||||
|
||||
corker = self.klass(brokers=(self.broker,))
|
||||
corker.cork()
|
||||
|
||||
latch = mitogen.core.Latch()
|
||||
self.broker.defer(self.ping, latch)
|
||||
self.assertRaises(mitogen.core.TimeoutError,
|
||||
lambda: latch.get(timeout=0.5))
|
||||
corker.uncork()
|
||||
self.assertEquals('pong', latch.get())
|
||||
|
||||
def test_cork_pool(self):
|
||||
pool = mitogen.service.Pool(self.router, services=(), size=4)
|
||||
try:
|
||||
latch = mitogen.core.Latch()
|
||||
pool.defer(self.ping, latch)
|
||||
self.assertEquals('pong', latch.get())
|
||||
|
||||
corker = self.klass(pools=(pool,))
|
||||
corker.cork()
|
||||
|
||||
latch = mitogen.core.Latch()
|
||||
pool.defer(self.ping, latch)
|
||||
self.assertRaises(mitogen.core.TimeoutError,
|
||||
lambda: latch.get(timeout=0.5))
|
||||
corker.uncork()
|
||||
self.assertEquals('pong', latch.get())
|
||||
finally:
|
||||
pool.stop(join=True)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest2.main()
|
||||
Loading…
Reference in New Issue