issue #532: PushFileService race.
There has always been a race in PushFileService since given a parent
asked to forward modules to two children via some intermediary:
interm = router.local()
c1 = router.local(via=interm)
c2 = router.local(via=interm)
service.propagate_to(c1, 'foo/bar.py')
service.propagate_to(c2, 'foo/bar.py')
Two calls will be emitted to 'interm':
PushFileService.store_and_forward(c1, 'foo/bar.py', [blob])
PushFileService.store(c2, 'foo/bar.py')
Which will be processed in-order up to the point where service pool
threads in 'interm' are woken to process the message.
While it is guaranteed store_and_forward() will be processed first, no
guarantee existed that its assigned pool thread would wake and take
_lock first, thus it was possible for forward() to win the race, and for
a request to arrive to forward a file that had not been placed in local
cache yet.
Here we get rid of SerializedInvoker entirely, as it is partially to
blame for hiding the race: SerializedInvoker can only ensure no two
messages are processed simultaneously, it cannot ensure the messages are
processed in their intended order.
Instead, teach forward() that it may be called before
store_and_forward(), and if that is the case, to place the forward
request on to _waiters alongside any local threads blocked in get().
6 years ago
|
|
|
import tempfile
|
|
|
|
|
|
|
|
import mitogen.core
|
|
|
|
import mitogen.service
|
|
|
|
import testlib
|
|
|
|
from mitogen.core import b
|
|
|
|
|
|
|
|
|
|
|
|
def prepare():
|
|
|
|
# ensure module loading delay is complete before loading PushFileService.
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
@mitogen.core.takes_router
|
|
|
|
def wait_for_file(path, router):
|
|
|
|
pool = mitogen.service.get_or_create_pool(router=router)
|
|
|
|
service = pool.get_service(u'mitogen.service.PushFileService')
|
|
|
|
return service.get(path)
|
|
|
|
|
|
|
|
|
|
|
|
class PropagateToTest(testlib.RouterMixin, testlib.TestCase):
|
|
|
|
klass = mitogen.service.PushFileService
|
|
|
|
|
|
|
|
def test_two_grandchild_one_intermediary(self):
|
|
|
|
tf = tempfile.NamedTemporaryFile()
|
|
|
|
path = mitogen.core.to_text(tf.name)
|
|
|
|
|
|
|
|
try:
|
|
|
|
tf.write(b('test'))
|
|
|
|
tf.flush()
|
|
|
|
|
|
|
|
interm = self.router.local(name='interm')
|
|
|
|
c1 = self.router.local(via=interm, name='c1')
|
|
|
|
c2 = self.router.local(via=interm)
|
|
|
|
|
|
|
|
c1.call(prepare)
|
|
|
|
c2.call(prepare)
|
|
|
|
|
|
|
|
service = self.klass(router=self.router)
|
|
|
|
service.propagate_to(context=c1, path=path)
|
|
|
|
service.propagate_to(context=c2, path=path)
|
|
|
|
|
|
|
|
s = c1.call(wait_for_file, path=path)
|
|
|
|
self.assertEqual(b('test'), s)
|
issue #532: PushFileService race.
There has always been a race in PushFileService since given a parent
asked to forward modules to two children via some intermediary:
interm = router.local()
c1 = router.local(via=interm)
c2 = router.local(via=interm)
service.propagate_to(c1, 'foo/bar.py')
service.propagate_to(c2, 'foo/bar.py')
Two calls will be emitted to 'interm':
PushFileService.store_and_forward(c1, 'foo/bar.py', [blob])
PushFileService.store(c2, 'foo/bar.py')
Which will be processed in-order up to the point where service pool
threads in 'interm' are woken to process the message.
While it is guaranteed store_and_forward() will be processed first, no
guarantee existed that its assigned pool thread would wake and take
_lock first, thus it was possible for forward() to win the race, and for
a request to arrive to forward a file that had not been placed in local
cache yet.
Here we get rid of SerializedInvoker entirely, as it is partially to
blame for hiding the race: SerializedInvoker can only ensure no two
messages are processed simultaneously, it cannot ensure the messages are
processed in their intended order.
Instead, teach forward() that it may be called before
store_and_forward(), and if that is the case, to place the forward
request on to _waiters alongside any local threads blocked in get().
6 years ago
|
|
|
|
|
|
|
s = c2.call(wait_for_file, path=path)
|
|
|
|
self.assertEqual(b('test'), s)
|
issue #532: PushFileService race.
There has always been a race in PushFileService since given a parent
asked to forward modules to two children via some intermediary:
interm = router.local()
c1 = router.local(via=interm)
c2 = router.local(via=interm)
service.propagate_to(c1, 'foo/bar.py')
service.propagate_to(c2, 'foo/bar.py')
Two calls will be emitted to 'interm':
PushFileService.store_and_forward(c1, 'foo/bar.py', [blob])
PushFileService.store(c2, 'foo/bar.py')
Which will be processed in-order up to the point where service pool
threads in 'interm' are woken to process the message.
While it is guaranteed store_and_forward() will be processed first, no
guarantee existed that its assigned pool thread would wake and take
_lock first, thus it was possible for forward() to win the race, and for
a request to arrive to forward a file that had not been placed in local
cache yet.
Here we get rid of SerializedInvoker entirely, as it is partially to
blame for hiding the race: SerializedInvoker can only ensure no two
messages are processed simultaneously, it cannot ensure the messages are
processed in their intended order.
Instead, teach forward() that it may be called before
store_and_forward(), and if that is the case, to place the forward
request on to _waiters alongside any local threads blocked in get().
6 years ago
|
|
|
finally:
|
|
|
|
tf.close()
|