You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
mitogen/tests/router_test.py

583 lines
19 KiB
Python

import errno
import os
import sys
import zlib
import testlib
import mitogen.core
import mitogen.master
import mitogen.parent
import mitogen.utils
from mitogen.core import b
try:
import Queue
except ImportError:
import queue as Queue
def ping():
return True
@mitogen.core.takes_router
def ping_context(other, router):
other = mitogen.parent.Context(router, other.context_id)
other.call(ping)
@mitogen.core.takes_router
def return_router_max_message_size(router):
return router.max_message_size
def send_n_sized_reply(sender, n):
sender.send(' ' * n)
return 123
class SourceVerifyTest(testlib.RouterMixin, testlib.TestCase):
def setUp(self):
super(SourceVerifyTest, self).setUp()
# Create some children, ping them, and store what their messages look
# like so we can mess with them later.
self.child1 = self.router.local()
self.child1_msg = self.child1.call_async(ping).get()
self.child1_stream = self.router._stream_by_id[self.child1.context_id]
self.child2 = self.router.local()
self.child2_msg = self.child2.call_async(ping).get()
self.child2_stream = self.router._stream_by_id[self.child2.context_id]
def test_bad_auth_id(self):
# Deliver a message locally from child2, but using child1's stream.
log = testlib.LogCapturer()
log.start()
# Used to ensure the message was dropped rather than routed after the
# error is logged.
recv = mitogen.core.Receiver(self.router)
self.child2_msg.handle = recv.handle
self.broker.defer_sync(
lambda: self.router._async_route(
self.child2_msg,
in_stream=self.child1_stream
)
)
# Ensure message wasn't forwarded.
self.assertTrue(recv.empty())
# Ensure error was logged.
expect = 'bad auth_id: got %r via' % (self.child2_msg.auth_id,)
self.assertTrue(expect in log.stop())
def test_parent_unaware_of_disconnect(self):
# Parent -> Child A -> Child B. B disconnects concurrent to Parent
# sending message. Parent does not yet know B has disconnected, A
# receives message from Parent with Parent's auth_id, for a stream that
# no longer exists.
c1 = self.router.local()
strm = self.router.stream_by_id(c1.context_id)
recv = mitogen.core.Receiver(self.router)
self.broker.defer(lambda:
strm.protocol._send(
mitogen.core.Message(
dst_id=1234, # nonexistent child
handle=1234,
src_id=mitogen.context_id,
reply_to=recv.handle,
)
)
)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: recv.get().unpickle()
)
self.assertEqual(e.args[0], self.router.no_route_msg % (
1234,
c1.context_id,
))
def test_bad_src_id(self):
# Deliver a message locally from child2 with the correct auth_id, but
# the wrong src_id.
log = testlib.LogCapturer()
log.start()
# Used to ensure the message was dropped rather than routed after the
# error is logged.
recv = mitogen.core.Receiver(self.router)
self.child2_msg.handle = recv.handle
self.child2_msg.src_id = self.child1.context_id
self.broker.defer(self.router._async_route,
self.child2_msg,
self.child2_stream)
# Wait for IO loop to finish everything above.
self.sync_with_broker()
# Ensure message wasn't forwarded.
self.assertTrue(recv.empty())
# Ensure error was lgoged.
expect = 'bad src_id: got %d via' % (self.child1_msg.src_id,)
self.assertTrue(expect in log.stop())
class PolicyTest(testlib.RouterMixin, testlib.TestCase):
def test_allow_any(self):
# This guy gets everything.
recv = mitogen.core.Receiver(self.router)
recv.to_sender().send(123)
self.sync_with_broker()
self.assertFalse(recv.empty())
self.assertEqual(123, recv.get().unpickle())
def test_refuse_all(self):
# Deliver a message locally from child2 with the correct auth_id, but
# the wrong src_id.
log = testlib.LogCapturer()
log.start()
# This guy never gets anything.
recv = mitogen.core.Receiver(
router=self.router,
policy=(lambda msg, stream: False),
)
# This guy becomes the reply_to of our refused message.
reply_target = mitogen.core.Receiver(self.router)
# Send the message.
self.router.route(
mitogen.core.Message(
dst_id=mitogen.context_id,
handle=recv.handle,
reply_to=reply_target.handle,
)
)
# Wait for IO loop.
self.sync_with_broker()
# Verify log.
self.assertTrue(self.router.refused_msg in log.stop())
# Verify message was not delivered.
self.assertTrue(recv.empty())
# Verify CallError received by reply_to target.
e = self.assertRaises(mitogen.core.ChannelError,
lambda: reply_target.get().unpickle())
self.assertEqual(e.args[0], self.router.refused_msg)
class CrashTest(testlib.BrokerMixin, testlib.TestCase):
# This is testing both Broker's ability to crash nicely, and Router's
# ability to respond to the crash event.
klass = mitogen.master.Router
def _naughty(self):
raise ValueError('eek')
def test_shutdown(self):
router = self.klass(self.broker)
sem = mitogen.core.Latch()
router.add_handler(sem.put)
log = testlib.LogCapturer()
log.start()
# Force a crash and ensure it wakes up.
self.broker._loop_once = self._naughty
self.broker.defer(lambda: None)
# sem should have received dead message.
self.assertTrue(sem.get().is_dead)
# Ensure it was logged.
expect = 'broker crashed'
self.assertTrue(expect in log.stop())
self.broker.join()
class AddHandlerTest(testlib.TestCase):
klass = mitogen.master.Router
def test_dead_message_sent_at_shutdown(self):
router = self.klass()
queue = Queue.Queue()
handle = router.add_handler(queue.put)
router.broker.shutdown()
self.assertTrue(queue.get(timeout=5).is_dead)
router.broker.join()
def test_cannot_double_register(self):
router = self.klass()
try:
router.add_handler((lambda: None), handle=1234)
e = self.assertRaises(mitogen.core.Error,
lambda: router.add_handler((lambda: None), handle=1234))
self.assertEqual(router.duplicate_handle_msg, e.args[0])
router.del_handler(1234)
finally:
router.broker.shutdown()
router.broker.join()
def test_can_reregister(self):
router = self.klass()
try:
router.add_handler((lambda: None), handle=1234)
router.del_handler(1234)
router.add_handler((lambda: None), handle=1234)
router.del_handler(1234)
finally:
router.broker.shutdown()
router.broker.join()
class MyselfTest(testlib.RouterMixin, testlib.TestCase):
def test_myself(self):
myself = self.router.myself()
self.assertEqual(myself.context_id, mitogen.context_id)
# TODO: context should know its own name too.
self.assertEqual(myself.name, 'self')
class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase):
klass = mitogen.master.Router
def test_local_exceeded(self):
router = self.klass(broker=self.broker, max_message_size=4096)
logs = testlib.LogCapturer()
logs.start()
# Send message and block for one IO loop, so _async_route can run.
router.route(mitogen.core.Message.pickled(' '*8192))
router.broker.defer_sync(lambda: None)
expect = 'message too large (max 4096 bytes)'
self.assertTrue(expect in logs.stop())
def test_local_dead_message(self):
# Local router should generate dead message when reply_to is set.
router = self.klass(broker=self.broker, max_message_size=4096)
logs = testlib.LogCapturer()
logs.start()
expect = router.too_large_msg % (4096,)
# Try function call. Receiver should be woken by a dead message sent by
# router due to message size exceeded.
child = router.local()
e = self.assertRaises(mitogen.core.ChannelError,
lambda: child.call(zlib.crc32, ' '*8192))
self.assertEqual(e.args[0], expect)
self.assertTrue(expect in logs.stop())
def test_remote_dead_message(self):
# Router should send dead message to original recipient when reply_to
# is unset.
router = self.klass(broker=self.broker, max_message_size=4096)
# Try function call. Receiver should be woken by a dead message sent by
# router due to message size exceeded.
child = router.local()
recv = mitogen.core.Receiver(router)
recv.to_sender().send(b('x') * 4097)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: recv.get().unpickle()
)
expect = router.too_large_msg % (4096,)
self.assertEqual(e.args[0], expect)
def test_remote_configured(self):
router = self.klass(broker=self.broker, max_message_size=64*1024)
remote = router.local()
size = remote.call(return_router_max_message_size)
self.assertEqual(size, 64*1024)
def test_remote_of_remote_configured(self):
router = self.klass(broker=self.broker, max_message_size=64*1024)
remote = router.local()
remote2 = router.local(via=remote)
size = remote2.call(return_router_max_message_size)
self.assertEqual(size, 64*1024)
def test_remote_exceeded(self):
# Ensure new contexts receive a router with the same value.
router = self.klass(broker=self.broker, max_message_size=64*1024)
recv = mitogen.core.Receiver(router)
logs = testlib.LogCapturer()
logs.start()
remote = router.local()
remote.call(send_n_sized_reply, recv.to_sender(), 128*1024)
expect = 'message too large (max %d bytes)' % (64*1024,)
self.assertTrue(expect in logs.stop())
class NoRouteTest(testlib.RouterMixin, testlib.TestCase):
def test_invalid_handle_returns_dead(self):
# Verify sending a message to an invalid handle yields a dead message
# from the target context.
l1 = self.router.local()
recv = l1.send_async(mitogen.core.Message(handle=999))
msg = recv.get(throw_dead=False)
self.assertEqual(msg.is_dead, True)
self.assertEqual(msg.src_id, l1.context_id)
self.assertEqual(msg.data, self.router.invalid_handle_msg.encode())
recv = l1.send_async(mitogen.core.Message(handle=999))
e = self.assertRaises(mitogen.core.ChannelError,
lambda: recv.get())
self.assertEqual(e.args[0], self.router.invalid_handle_msg)
def test_totally_invalid_context_returns_dead(self):
recv = mitogen.core.Receiver(self.router)
msg = mitogen.core.Message(
dst_id=1234,
handle=1234,
reply_to=recv.handle,
)
self.router.route(msg)
rmsg = recv.get(throw_dead=False)
self.assertEqual(rmsg.is_dead, True)
self.assertEqual(rmsg.src_id, mitogen.context_id)
self.assertEqual(rmsg.data, (self.router.no_route_msg % (
1234,
mitogen.context_id,
)).encode())
self.router.route(msg)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: recv.get())
self.assertEqual(e.args[0], (self.router.no_route_msg % (
1234,
mitogen.context_id,
)))
def test_previously_alive_context_returns_dead(self):
l1 = self.router.local()
l1.shutdown(wait=True)
recv = mitogen.core.Receiver(self.router)
msg = mitogen.core.Message(
dst_id=l1.context_id,
handle=mitogen.core.CALL_FUNCTION,
reply_to=recv.handle,
)
self.router.route(msg)
rmsg = recv.get(throw_dead=False)
self.assertEqual(rmsg.is_dead, True)
self.assertEqual(rmsg.src_id, mitogen.context_id)
self.assertEqual(rmsg.data, (self.router.no_route_msg % (
l1.context_id,
mitogen.context_id,
)).encode())
self.router.route(msg)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: recv.get())
self.assertEqual(e.args[0], self.router.no_route_msg % (
l1.context_id,
mitogen.context_id,
))
def test_siblings_cant_talk(router):
l1 = router.local()
l2 = router.local()
logs = testlib.LogCapturer()
logs.start()
try:
l2.call(ping_context, l1)
except mitogen.core.CallError:
e = sys.exc_info()[1]
msg = mitogen.core.Router.unidirectional_msg % (
l2.context_id,
l1.context_id,
mitogen.context_id,
)
assert msg in str(e)
assert 'routing mode prevents forward of ' in logs.stop()
@mitogen.core.takes_econtext
def test_siblings_cant_talk_remote(econtext):
mitogen.parent.upgrade_router(econtext)
test_siblings_cant_talk(econtext.router)
class UnidirectionalTest(testlib.RouterMixin, testlib.TestCase):
def test_siblings_cant_talk_master(self):
self.router.unidirectional = True
test_siblings_cant_talk(self.router)
def test_siblings_cant_talk_parent(self):
# ensure 'unidirectional' attribute is respected for contexts started
# by children.
self.router.unidirectional = True
parent = self.router.local()
parent.call(test_siblings_cant_talk_remote)
def test_auth_id_can_talk(self):
self.router.unidirectional = True
# One stream has auth_id stamped to that of the master, so it should be
# treated like a parent.
l1 = self.router.local()
l1s = self.router.stream_by_id(l1.context_id)
l1s.protocol.auth_id = mitogen.context_id
l1s.protocol.is_privileged = True
l2 = self.router.local()
e = self.assertRaises(mitogen.core.CallError,
lambda: l2.call(ping_context, l1))
msg = 'mitogen.core.ChannelError: %s' % (self.router.refused_msg,)
self.assertTrue(str(e).startswith(msg))
class EgressIdsTest(testlib.RouterMixin, testlib.TestCase):
def test_egress_ids_populated(self):
# Ensure Stream.egress_ids is populated on message reception.
c1 = self.router.local(name='c1')
c2 = self.router.local(name='c2')
c1s = self.router.stream_by_id(c1.context_id)
try:
c1.call(ping_context, c2)
except mitogen.core.CallError:
# Fails because siblings cant call funcs in each other, but this
# causes messages to be sent.
pass
self.assertEqual(c1s.protocol.egress_ids, set([
mitogen.context_id,
c2.context_id,
]))
class ShutdownTest(testlib.RouterMixin, testlib.TestCase):
# 613: tests for all the weird shutdown() variants we ended up with.
def test_shutdown_wait_false(self):
l1 = self.router.local()
pid = l1.call(os.getpid)
strm = self.router.stream_by_id(l1.context_id)
exitted = mitogen.core.Latch()
# It is possible for Process 'exit' signal to fire immediately during
# processing of Stream 'disconnect' signal, so we must wait for both,
# otherwise ChannelError below will return 'respondent context has
# disconnected' rather than 'no route', because RouteMonitor hasn't run
# yet and the Receiver caught Context 'disconnect' signal instead of a
# dead message.
mitogen.core.listen(strm.conn.proc, 'exit', exitted.put)
mitogen.core.listen(strm, 'disconnect', exitted.put)
l1.shutdown(wait=False)
exitted.get()
exitted.get()
e = self.assertRaises(OSError,
lambda: os.waitpid(pid, 0))
self.assertEqual(e.args[0], errno.ECHILD)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: l1.call(os.getpid))
self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % (
l1.context_id,
mitogen.context_id,
))
def test_shutdown_wait_true(self):
l1 = self.router.local()
pid = l1.call(os.getpid)
conn = self.router.stream_by_id(l1.context_id).conn
exitted = mitogen.core.Latch()
mitogen.core.listen(conn.proc, 'exit', exitted.put)
l1.shutdown(wait=True)
exitted.get()
e = self.assertRaises(OSError,
lambda: os.waitpid(pid, 0))
self.assertEqual(e.args[0], errno.ECHILD)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: l1.call(os.getpid))
self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % (
l1.context_id,
mitogen.context_id,
))
def test_disconnect_invalid_context(self):
self.router.disconnect(
mitogen.core.Context(self.router, 1234)
)
def test_disconnect_valid_context(self):
l1 = self.router.local()
pid = l1.call(os.getpid)
strm = self.router.stream_by_id(l1.context_id)
exitted = mitogen.core.Latch()
mitogen.core.listen(strm.conn.proc, 'exit', exitted.put)
self.router.disconnect_stream(strm)
exitted.get()
e = self.assertRaises(OSError,
lambda: os.waitpid(pid, 0))
self.assertEqual(e.args[0], errno.ECHILD)
e = self.assertRaises(mitogen.core.ChannelError,
lambda: l1.call(os.getpid))
self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % (
l1.context_id,
mitogen.context_id,
))
def test_disconnect_all(self):
l1 = self.router.local()
l2 = self.router.local()
pids = [l1.call(os.getpid), l2.call(os.getpid)]
exitted = mitogen.core.Latch()
for ctx in l1, l2:
strm = self.router.stream_by_id(ctx.context_id)
mitogen.core.listen(strm.conn.proc, 'exit', exitted.put)
self.router.disconnect_all()
exitted.get()
exitted.get()
for pid in pids:
e = self.assertRaises(OSError,
lambda: os.waitpid(pid, 0))
self.assertEqual(e.args[0], errno.ECHILD)
for ctx in l1, l2:
e = self.assertRaises(mitogen.core.ChannelError,
lambda: ctx.call(os.getpid))
self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % (
ctx.context_id,
mitogen.context_id,
))