Merge remote-tracking branch 'origin/dmw'
- issue #411: fix check_host_keys="accept" - issue #305: dead message if max message size exceeded - issue #369: implement Connection.reset() - issue #76: disconnect propagation - log format string fixes - various 2/3 test fixes - large message benchmark - centralize stub client utils in data/stubs/ - activate faulthandler in tests - better OpenSSH 7.5+ permission denied handlingissue260
David Wilson
6 years ago
commit
9ba0561dd2
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Print shell environment exports adding ARA plugins to the list of plugins
|
||||
from ansible.cfg in the CWD.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import ara.setup
|
||||
import ansible.constants as C
|
||||
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
print('export ANSIBLE_ACTION_PLUGINS=%s:%s' % (
|
||||
':'.join(C.DEFAULT_ACTION_PLUGIN_PATH),
|
||||
ara.setup.action_plugins,
|
||||
))
|
||||
|
||||
print('export ANSIBLE_CALLBACK_PLUGINS=%s:%s' % (
|
||||
':'.join(C.DEFAULT_CALLBACK_PLUGIN_PATH),
|
||||
ara.setup.callback_plugins,
|
||||
))
|
||||
|
||||
print('export ANSIBLE_LIBRARY=%s:%s' % (
|
||||
':'.join(C.DEFAULT_MODULE_PATH),
|
||||
ara.setup.library,
|
||||
))
|
||||
@ -0,0 +1,25 @@
|
||||
|
||||
- hosts: all
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
|
||||
- name: Create file tree
|
||||
connection: local
|
||||
shell: >
|
||||
mkdir -p /tmp/filetree.in;
|
||||
for i in `seq -f /tmp/filetree.in/%g 1 100`; do echo $RANDOM > $i; done;
|
||||
|
||||
- name: Delete remote file tree
|
||||
file: path=/tmp/filetree.out state=absent
|
||||
when: 0
|
||||
|
||||
- file:
|
||||
state: directory
|
||||
path: /tmp/filetree.out
|
||||
|
||||
- name: Trigger nasty process pileup
|
||||
copy:
|
||||
src: "{{item.src}}"
|
||||
dest: "/tmp/filetree.out/{{item.path}}"
|
||||
with_filetree: /tmp/filetree.in
|
||||
when: item.state == 'file'
|
||||
@ -0,0 +1,25 @@
|
||||
z
|
||||
|
||||
[z-x10]
|
||||
z-[01:10]
|
||||
|
||||
[z-x20]
|
||||
z-[01:20]
|
||||
|
||||
[z-x50]
|
||||
z-[01:50]
|
||||
|
||||
[z-x100]
|
||||
z-[001:100]
|
||||
|
||||
[z-x200]
|
||||
z-[001:200]
|
||||
|
||||
[z-x300]
|
||||
z-[001:300]
|
||||
|
||||
[z-x400]
|
||||
z-[001:400]
|
||||
|
||||
[z-x500]
|
||||
z-[001:500]
|
||||
@ -1,5 +1,8 @@
|
||||
---
|
||||
|
||||
- import_playbook: disconnect_during_module.yml
|
||||
- import_playbook: disconnect_resets_connection.yml
|
||||
- import_playbook: exec_command.yml
|
||||
- import_playbook: put_small_file.yml
|
||||
- import_playbook: put_large_file.yml
|
||||
- import_playbook: put_small_file.yml
|
||||
- import_playbook: reset.yml
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
# issue 352: test ability to notice disconnection during a module invocation.
|
||||
---
|
||||
|
||||
- name: integration/connection/disconnect_during_module.yml
|
||||
hosts: test-targets localhost
|
||||
gather_facts: no
|
||||
any_errors_fatal: false
|
||||
tasks:
|
||||
- run_once: true # don't run against localhost
|
||||
shell: |
|
||||
kill -9 $PPID
|
||||
register: out
|
||||
ignore_errors: true
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- out.msg.startswith('Mitogen was disconnected from the remote environment while a call was in-progress.')
|
||||
|
||||
- meta: clear_host_errors
|
||||
@ -0,0 +1,44 @@
|
||||
# issue 370: Connection should reset to 'disconnected' state when disconnect
|
||||
# detected
|
||||
#
|
||||
# Previously the 'Mitogen was disconnected' error would fail the first task,
|
||||
# but the Connection instance would still think it still had a valid
|
||||
# connection.
|
||||
#
|
||||
# See also disconnect_during_module.yml
|
||||
|
||||
---
|
||||
|
||||
- name: integration/connection/disconnect_resets_connection.yml
|
||||
hosts: test-targets
|
||||
gather_facts: no
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- mitogen_action_script:
|
||||
script: |
|
||||
import sys
|
||||
from ansible.errors import AnsibleConnectionFailure
|
||||
|
||||
assert not self._connection.connected, \
|
||||
"Connection was not initially disconnected."
|
||||
|
||||
self._low_level_execute_command('echo')
|
||||
assert self._connection.connected, \
|
||||
"Connection was not connected after good command."
|
||||
|
||||
try:
|
||||
self._low_level_execute_command('kill -9 $PPID')
|
||||
assert 0, 'AnsibleConnectionFailure was not raised'
|
||||
except AnsibleConnectionFailure:
|
||||
e = sys.exc_info()[1]
|
||||
assert str(e).startswith('Mitogen was disconnected')
|
||||
|
||||
assert not self._connection.connected, \
|
||||
"Connection did not reset."
|
||||
|
||||
try:
|
||||
self._low_level_execute_command('kill -9 $PPID')
|
||||
assert 0, 'AnsibleConnectionFailure was not raised'
|
||||
except AnsibleConnectionFailure:
|
||||
e = sys.exc_info()[1]
|
||||
assert str(e).startswith('Mitogen was disconnected')
|
||||
@ -0,0 +1,38 @@
|
||||
# issue #369: Connection.reset() should cause destruction of the remote
|
||||
# interpreter and any children.
|
||||
|
||||
---
|
||||
|
||||
- name: integration/connection/reset.yml
|
||||
hosts: test-targets
|
||||
tasks:
|
||||
- when: is_mitogen
|
||||
block:
|
||||
- custom_python_detect_environment:
|
||||
register: out
|
||||
|
||||
- custom_python_detect_environment:
|
||||
become: true
|
||||
register: out_become
|
||||
|
||||
- meta: reset_connection
|
||||
|
||||
- custom_python_detect_environment:
|
||||
register: out2
|
||||
|
||||
- custom_python_detect_environment:
|
||||
register: out_become2
|
||||
|
||||
- assert:
|
||||
that:
|
||||
# Interpreter PID has changed.
|
||||
- out.pid != out2.pid
|
||||
|
||||
# SSH PID has changed.
|
||||
- out.ppid != out2.ppid
|
||||
|
||||
# Interpreter PID has changed.
|
||||
- out_become.pid != out_become2.pid
|
||||
|
||||
# sudo PID has changed.
|
||||
- out_become.ppid != out_become2.ppid
|
||||
@ -0,0 +1,28 @@
|
||||
# I am an Ansible action plug-in. I run the script provided in the parameter in
|
||||
# the context of the action.
|
||||
|
||||
import sys
|
||||
|
||||
from ansible.plugins.action import ActionBase
|
||||
|
||||
|
||||
def execute(s, gbls, lcls):
|
||||
if sys.version_info > (3,):
|
||||
exec(s, gbls, lcls)
|
||||
else:
|
||||
exec('exec s in gbls, lcls')
|
||||
|
||||
|
||||
class ActionModule(ActionBase):
|
||||
def run(self, tmp=None, task_vars=None):
|
||||
super(ActionModule, self).run(tmp=tmp, task_vars=task_vars)
|
||||
lcls = {
|
||||
'self': self,
|
||||
'result': {}
|
||||
}
|
||||
execute(self._task.args['script'], globals(), lcls)
|
||||
return lcls['result']
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,2 @@
|
||||
paramiko==2.3.2 # Last 2.6-compat version.
|
||||
google-api-python-client==1.6.5
|
||||
@ -0,0 +1,5 @@
|
||||
|
||||
# tests/bench/
|
||||
|
||||
Various manually executed scripts to aid benchmarking, or trigger old
|
||||
performance problems.
|
||||
@ -0,0 +1,28 @@
|
||||
|
||||
# Verify _receive_one() quadratic behaviour fixed.
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
import socket
|
||||
import mitogen
|
||||
|
||||
|
||||
@mitogen.main()
|
||||
def main(router):
|
||||
c = router.fork()
|
||||
|
||||
n = 1048576 * 127
|
||||
s = ' ' * n
|
||||
print('bytes in %.2fMiB string...' % (n/1048576.0),)
|
||||
|
||||
t0 = time.time()
|
||||
for x in range(10):
|
||||
tt0 = time.time()
|
||||
assert n == c.call(len, s)
|
||||
print('took %dms' % (1000 * (time.time() - tt0),))
|
||||
t1 = time.time()
|
||||
print('total %dms / %dms avg / %.2fMiB/sec' % (
|
||||
1000 * (t1 - t0),
|
||||
(1000 * (t1 - t0)) / (x + 1),
|
||||
((n * (x + 1)) / (t1 - t0)) / 1048576.0,
|
||||
))
|
||||
@ -0,0 +1,23 @@
|
||||
"""
|
||||
Measure latency of local service RPC.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import mitogen.service
|
||||
import mitogen
|
||||
|
||||
|
||||
class MyService(mitogen.service.Service):
|
||||
@mitogen.service.expose(policy=mitogen.service.AllowParents())
|
||||
def ping(self):
|
||||
return 'pong'
|
||||
|
||||
|
||||
@mitogen.main()
|
||||
def main(router):
|
||||
f = router.fork()
|
||||
t0 = time.time()
|
||||
for x in range(1000):
|
||||
f.call_service(service_name=MyService, method_name='ping')
|
||||
print('++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec')
|
||||
@ -0,0 +1,32 @@
|
||||
|
||||
import threading
|
||||
|
||||
import unittest2
|
||||
|
||||
import testlib
|
||||
|
||||
import mitogen.core
|
||||
|
||||
|
||||
class DeferSyncTest(testlib.TestCase):
|
||||
klass = mitogen.core.Broker
|
||||
|
||||
def test_okay(self):
|
||||
broker = self.klass()
|
||||
try:
|
||||
th = broker.defer_sync(lambda: threading.currentThread())
|
||||
self.assertEquals(th, broker._thread)
|
||||
finally:
|
||||
broker.shutdown()
|
||||
|
||||
def test_exception(self):
|
||||
broker = self.klass()
|
||||
try:
|
||||
self.assertRaises(ValueError,
|
||||
broker.defer_sync, lambda: int('dave'))
|
||||
finally:
|
||||
broker.shutdown()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest2.main()
|
||||
@ -0,0 +1,12 @@
|
||||
|
||||
import logging
|
||||
import mitogen.master
|
||||
|
||||
def foo():
|
||||
pass
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
router = mitogen.master.Router()
|
||||
|
||||
l = router.local()
|
||||
l.call(foo)
|
||||
@ -0,0 +1,5 @@
|
||||
|
||||
# tests/data/stubs/
|
||||
|
||||
Dummy implementations of various third party tools that just spawn local Python
|
||||
interpreters. Used to roughly test the tools' associated Mitogen classes.
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
os.environ['ORIGINAL_ARGV'] = repr(sys.argv)
|
||||
os.execv(sys.executable, sys.argv[sys.argv.index('-c') - 1:])
|
||||
@ -0,0 +1,28 @@
|
||||
import os
|
||||
|
||||
import mitogen
|
||||
|
||||
import unittest2
|
||||
|
||||
import testlib
|
||||
|
||||
|
||||
class ConstructorTest(testlib.RouterMixin, unittest2.TestCase):
|
||||
def test_okay(self):
|
||||
docker_path = testlib.data_path('stubs/docker.py')
|
||||
context = self.router.docker(
|
||||
container='container_name',
|
||||
docker_path=docker_path,
|
||||
)
|
||||
stream = self.router.stream_by_id(context.context_id)
|
||||
|
||||
argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV'))
|
||||
self.assertEquals(argv[0], docker_path)
|
||||
self.assertEquals(argv[1], 'exec')
|
||||
self.assertEquals(argv[2], '--interactive')
|
||||
self.assertEquals(argv[3], 'container_name')
|
||||
self.assertEquals(argv[4], stream.python_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest2.main()
|
||||
@ -0,0 +1,119 @@
|
||||
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
import unittest2
|
||||
|
||||
import mitogen
|
||||
import mitogen.fork
|
||||
import mitogen.master
|
||||
import mitogen.service
|
||||
import mitogen.unix
|
||||
|
||||
import testlib
|
||||
|
||||
|
||||
class MyService(mitogen.service.Service):
|
||||
def __init__(self, latch, **kwargs):
|
||||
super(MyService, self).__init__(**kwargs)
|
||||
# used to wake up main thread once client has made its request
|
||||
self.latch = latch
|
||||
|
||||
@mitogen.service.expose(policy=mitogen.service.AllowParents())
|
||||
def ping(self, msg):
|
||||
self.latch.put(None)
|
||||
return {
|
||||
'src_id': msg.src_id,
|
||||
'auth_id': msg.auth_id,
|
||||
}
|
||||
|
||||
|
||||
class IsPathDeadTest(unittest2.TestCase):
|
||||
func = staticmethod(mitogen.unix.is_path_dead)
|
||||
path = '/tmp/stale-socket'
|
||||
|
||||
def test_does_not_exist(self):
|
||||
self.assertTrue(self.func('/tmp/does-not-exist'))
|
||||
|
||||
def make_socket(self):
|
||||
if os.path.exists(self.path):
|
||||
os.unlink(self.path)
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.bind(self.path)
|
||||
return s
|
||||
|
||||
def test_conn_refused(self):
|
||||
s = self.make_socket()
|
||||
s.close()
|
||||
self.assertTrue(self.func(self.path))
|
||||
|
||||
def test_is_alive(self):
|
||||
s = self.make_socket()
|
||||
s.listen(5)
|
||||
self.assertFalse(self.func(self.path))
|
||||
s.close()
|
||||
os.unlink(self.path)
|
||||
|
||||
|
||||
class ListenerTest(testlib.RouterMixin, unittest2.TestCase):
|
||||
klass = mitogen.unix.Listener
|
||||
|
||||
def test_constructor_basic(self):
|
||||
listener = self.klass(router=self.router)
|
||||
self.assertFalse(mitogen.unix.is_path_dead(listener.path))
|
||||
os.unlink(listener.path)
|
||||
|
||||
|
||||
class ClientTest(unittest2.TestCase):
|
||||
klass = mitogen.unix.Listener
|
||||
|
||||
def _try_connect(self, path):
|
||||
# give server a chance to setup listener
|
||||
for x in range(10):
|
||||
try:
|
||||
return mitogen.unix.connect(path)
|
||||
except socket.error:
|
||||
if x == 9:
|
||||
raise
|
||||
time.sleep(0.1)
|
||||
|
||||
def _test_simple_client(self, path):
|
||||
router, context = self._try_connect(path)
|
||||
self.assertEquals(0, context.context_id)
|
||||
self.assertEquals(1, mitogen.context_id)
|
||||
self.assertEquals(0, mitogen.parent_id)
|
||||
resp = context.call_service(service_name=MyService, method_name='ping')
|
||||
self.assertEquals(mitogen.context_id, resp['src_id'])
|
||||
self.assertEquals(0, resp['auth_id'])
|
||||
|
||||
def _test_simple_server(self, path):
|
||||
router = mitogen.master.Router()
|
||||
latch = mitogen.core.Latch()
|
||||
try:
|
||||
try:
|
||||
listener = self.klass(path=path, router=router)
|
||||
pool = mitogen.service.Pool(router=router, services=[
|
||||
MyService(latch=latch, router=router),
|
||||
])
|
||||
latch.get()
|
||||
# give broker a chance to deliver service resopnse
|
||||
time.sleep(0.1)
|
||||
finally:
|
||||
pool.shutdown()
|
||||
router.broker.shutdown()
|
||||
finally:
|
||||
os._exit(0)
|
||||
|
||||
def test_simple(self):
|
||||
path = mitogen.unix.make_socket_path()
|
||||
if os.fork():
|
||||
self._test_simple_client(path)
|
||||
else:
|
||||
mitogen.fork.on_fork()
|
||||
self._test_simple_server(path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest2.main()
|
||||
Loading…
Reference in New Issue