diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 27b109fe..cf539dfe 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -505,16 +505,24 @@ The primary reason for using :py:mod:`cPickle` is that it is computationally efficient, and avoids including a potentially large body of serialization code in the bootstrap. -The pickler will instantiate only built-in types and one of 3 constructor -functions, to support unpickling :py:class:`CallError +The pickler will, by default, instantiate only built-in types and one of 3 +constructor functions, to support unpickling :py:class:`CallError `, :py:class:`mitogen.core.Sender`,and -:py:class:`Context `. +:py:class:`Context `. If you want to allow the deserialization +of arbitrary types to, for example, allow passing remote function call arguments of an +arbitrary type, you can use :py:func:`mitogen.core.set_pickle_whitelist` to set a +list of allowable patterns that match against a global's +:code:`[module].[func]` string. The choice of Pickle is one area to be revisited later. All accounts suggest it cannot be used securely, however few of those accounts appear to be expert, and none mention any additional attacks that would not be prevented by using a restrictive class whitelist. +In the future, pickled data could include an HMAC that is based upon a +preshared key (specified by the parent during child boot) to reduce the risk +of malicioius tampering. + The IO Multiplexer ------------------ diff --git a/mitogen/core.py b/mitogen/core.py index 152f456e..07a30160 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -46,6 +46,7 @@ import logging import os import pickle as py_pickle import pstats +import re import signal import socket import struct @@ -769,6 +770,45 @@ else: _Unpickler = pickle.Unpickler +#: A list of compiled regex patterns which allow end-users to selectively opt +#: into deserializing certain globals. +_PICKLE_GLOBAL_WHITELIST_PATTERNS = None +_PICKLE_GLOBAL_WHITELIST = None + + +def set_pickle_whitelist(pattern_strings): + """ + Specify regex patterns that control allowable global unpickling functions. + + `pattern_strings` is sequence of pattern strings that will be fed into + `re.compile` and then used to authenticate pickle calls. In order for a + non-trivially typed message to unpickle, one of these patterns must + match against a complete [module].[function] string. + """ + if not isinstance(pattern_strings, (tuple, list, set)): + pattern_strings = (pattern_strings,) + + global _PICKLE_GLOBAL_WHITELIST + global _PICKLE_GLOBAL_WHITELIST_PATTERNS + + _PICKLE_GLOBAL_WHITELIST = pattern_strings + _PICKLE_GLOBAL_WHITELIST_PATTERNS = [] + + for patt_str in pattern_strings: + if not patt_str.endswith('$'): + patt_str += '$' + _PICKLE_GLOBAL_WHITELIST_PATTERNS.append(re.compile(patt_str)) + + +def _test_pickle_whitelist_accept(module, func): + if not _PICKLE_GLOBAL_WHITELIST_PATTERNS: + return False + + test_str = "{}.{}".format(module, func) + return bool(any( + patt.match(test_str) for patt in _PICKLE_GLOBAL_WHITELIST_PATTERNS)) + + class Message(object): """ Messages are the fundamental unit of communication, comprising fields from @@ -868,7 +908,14 @@ class Message(object): return BytesType elif SimpleNamespace and module == 'types' and func == 'SimpleNamespace': return SimpleNamespace - raise StreamError('cannot unpickle %r/%r', module, func) + elif _test_pickle_whitelist_accept(module, func): + try: + return getattr(import_module(module), func) + except AttributeError as e: + LOG.info(str(e)) + raise StreamError( + 'cannot unpickle %r/%r - try using `set_pickle_whitelist`', + module, func) @property def is_dead(self): @@ -968,8 +1015,8 @@ class Message(object): # Must occur off the broker thread. try: obj = unpickler.load() - except: - LOG.error('raw pickle was: %r', self.data) + except Exception as e: + LOG.error('raw pickle was: %r (exc: %r)', self.data, e) raise self._unpickled = obj except (TypeError, ValueError): @@ -3845,6 +3892,9 @@ class ExternalContext(object): Router.max_message_size = self.config['max_message_size'] if self.config['profiling']: enable_profiling() + if self.config['pickle_whitelist_patterns']: + set_pickle_whitelist(self.config['pickle_whitelist_patterns']) + self.broker = Broker(activate_compat=False) self.router = Router(self.broker) self.router.debug = self.config.get('debug', False) diff --git a/mitogen/parent.py b/mitogen/parent.py index 32aa3cb6..c78c3840 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -639,7 +639,7 @@ class TimerList(object): def get_timeout(self): """ Return the floating point seconds until the next event is due. - + :returns: Floating point delay, or 0.0, or :data:`None` if no events are scheduled. @@ -1504,6 +1504,7 @@ class Connection(object): 'blacklist': self._router.get_module_blacklist(), 'max_message_size': self.options.max_message_size, 'version': mitogen.__version__, + 'pickle_whitelist_patterns': mitogen.core._PICKLE_GLOBAL_WHITELIST, } def get_preamble(self): diff --git a/tests/call_function_test.py b/tests/call_function_test.py index 1e838bda..43384aea 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -77,9 +77,13 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): lambda: self.local.call(func_with_bad_return_value)) self.assertEqual( exc.args[0], - "cannot unpickle '%s'/'CrazyType'" % (__name__,), + "cannot unpickle '%s'/'CrazyType' - try using `set_pickle_whitelist`" % (__name__,), ) + mitogen.core.set_pickle_whitelist(r'.*CrazyType') + self.assertIsInstance(self.local.call(func_with_bad_return_value), CrazyType) + mitogen.core.set_pickle_whitelist([]) + def test_aborted_on_local_context_disconnect(self): stream = self.router._stream_by_id[self.local.context_id] self.broker.stop_receive(stream)