From 6a2f88d6a350e5d1556fee6facf9ab8fe79acd33 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 23 Jan 2019 12:44:08 +0000 Subject: [PATCH] issue #477: make mitogen.fork unsupported on Py<2.6. --- docs/api.rst | 7 ++++++- mitogen/fork.py | 22 ++++++++++++++++++++++ tests/fork_test.py | 46 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 78bb57f4..b1fa215c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -115,6 +115,9 @@ Connection Methods and router, and responds to function calls identically to children created using other methods. + The use of this method is strongly discouraged. It requires Python 2.6 or + newer, as older Pythons made no effort to reset threading state upon fork. + For long-lived processes, :meth:`local` is always better as it guarantees a pristine interpreter state that inherited little from the parent. Forking should only be used in performance-sensitive scenarios @@ -158,7 +161,9 @@ Connection Methods * Locks held in the parent causing random deadlocks in the child, such as when another thread emits a log entry via the :mod:`logging` - package concurrent to another thread calling :meth:`fork`. + package concurrent to another thread calling :meth:`fork`, or when a C + extension module calls the C library allocator, or when a thread is using + the C library DNS resolver, for example via :func:`socket.gethostbyname`. * Objects existing in Thread-Local Storage of every non-:meth:`fork` thread becoming permanently inaccessible, and never having their diff --git a/mitogen/fork.py b/mitogen/fork.py index 1343f7a9..6859a140 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -39,6 +39,18 @@ import mitogen.parent LOG = logging.getLogger('mitogen') +# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up +# interpreter state. So 2.4/2.5 interpreters start .local() contexts for +# isolation instead. Since we don't have any crazy memory sharing problems to +# avoid, there is no virginal fork parent either. The child is started directly +# from the login/become process. In future this will be default everywhere, +# fork is brainwrong from the stone age. +FORK_SUPPORTED = sys.version_info >= (2, 6) + + +class Error(mitogen.core.StreamError): + pass + def fixup_prngs(): """ @@ -113,9 +125,19 @@ class Stream(mitogen.parent.Stream): #: User-supplied function for cleaning up child process state. on_fork = None + python_version_msg = ( + "The mitogen.fork method is not supported on Python versions " + "prior to 2.6, since those versions made no attempt to repair " + "critical interpreter state following a fork. Please use the " + "local() method instead." + ) + def construct(self, old_router, max_message_size, on_fork=None, debug=False, profiling=False, unidirectional=False, on_start=None): + if not FORK_SUPPORTED: + raise Error(self.python_version_msg) + # fork method only supports a tiny subset of options. super(Stream, self).construct(max_message_size=max_message_size, debug=debug, profiling=profiling, diff --git a/tests/fork_test.py b/tests/fork_test.py index 39f5352e..7ca41194 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -1,12 +1,25 @@ -import _ssl -import ctypes import os import random -import ssl import struct import sys +try: + import _ssl +except ImportError: + _ssl = None + +try: + import ssl +except ImportError: + ssl = None + +try: + import ctypes +except ImportError: + # Python 2.4 + ctypes = None + import mitogen import unittest2 @@ -29,16 +42,17 @@ def _find_ssl_darwin(): return bits[1] -if sys.platform.startswith('linux'): +if ctypes and sys.platform.startswith('linux'): LIBSSL_PATH = _find_ssl_linux() -elif sys.platform == 'darwin': +elif ctypes and sys.platform == 'darwin': LIBSSL_PATH = _find_ssl_darwin() else: - assert 0, "Don't know how to find libssl on this platform" + LIBSSL_PATH = None -c_ssl = ctypes.CDLL(LIBSSL_PATH) -c_ssl.RAND_pseudo_bytes.argtypes = [ctypes.c_char_p, ctypes.c_int] -c_ssl.RAND_pseudo_bytes.restype = ctypes.c_int +if ctypes and LIBSSL_PATH: + c_ssl = ctypes.CDLL(LIBSSL_PATH) + c_ssl.RAND_pseudo_bytes.argtypes = [ctypes.c_char_p, ctypes.c_int] + c_ssl.RAND_pseudo_bytes.restype = ctypes.c_int def ping(): @@ -64,6 +78,12 @@ def exercise_importer(n): return simple_pkg.a.subtract_one_add_two(n) +skipIfUnsupported = unittest2.skipIf( + condition=(not mitogen.fork.FORK_SUPPORTED), + reason="mitogen.fork unsupported on this platform" +) + + class ForkTest(testlib.RouterMixin, testlib.TestCase): def test_okay(self): context = self.router.fork() @@ -74,6 +94,10 @@ class ForkTest(testlib.RouterMixin, testlib.TestCase): context = self.router.fork() self.assertNotEqual(context.call(random_random), random_random()) + @unittest2.skipIf( + condition=LIBSSL_PATH is None or ctypes is None, + reason='cant test libssl on this platform', + ) def test_ssl_module_diverges(self): # Ensure generator state is initialized. RAND_pseudo_bytes() @@ -93,6 +117,8 @@ class ForkTest(testlib.RouterMixin, testlib.TestCase): context = self.router.fork(on_start=on_start) self.assertEquals(123, recv.get().unpickle()) +ForkTest = skipIfUnsupported(ForkTest) + class DoubleChildTest(testlib.RouterMixin, testlib.TestCase): def test_okay(self): @@ -115,6 +141,8 @@ class DoubleChildTest(testlib.RouterMixin, testlib.TestCase): c2 = self.router.fork(name='c2', via=c1) self.assertEqual(2, c2.call(exercise_importer, 1)) +DoubleChildTest = skipIfUnsupported(DoubleChildTest) + if __name__ == '__main__': unittest2.main()