diff --git a/docs/changelog.rst b/docs/changelog.rst index b08614c7..e3c2dbcf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,13 @@ In progress (unreleased) * :gh:issue:`1415` :mod:`mitogen`: Put fallbacks & polyfills into ``if sys.version_info`` blocks +* :gh:issue:`1423` tests: Group and unify naming of connection benchmarks +* :gh:issue:`1424` tests: Parameterize connection benchmarks +* :gh:issue:`1424` tests: Standardise output of connection benchmarks +* :gh:issue:`1424` tests: Parameterize throughput benchmark +* :gh:issue:`1424` tests: Parameterize large message benchmark +* :gh:issue:`1424` :mod:`mitogen`: Consolidate all ``range`` and ``xrange`` + polyfills into :attr:`mitogen.core.range` v0.3.37 (2026-01-08) diff --git a/mitogen/core.py b/mitogen/core.py index 666905ca..3bd23b35 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -227,6 +227,7 @@ if sys.version_info >= (3, 0): BufferType = lambda buf, start: memoryview(buf)[start:] integer_types = (int,) iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values + range = range else: import cPickle as pickle import thread @@ -238,6 +239,7 @@ else: UnicodeType = unicode integer_types = (int, long) iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues + range = xrange AnyTextType = (BytesType, UnicodeType) diff --git a/mitogen/minify.py b/mitogen/minify.py index 09fdc4eb..8f78b2ab 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -104,7 +104,7 @@ def strip_docstrings(tokens): elif typ == tokenize.NEWLINE: stack.append(t) start_line, end_line = stack[0][2][0], stack[-1][3][0]+1 - for i in range(start_line, end_line): + for i in mitogen.core.range(start_line, end_line): yield tokenize.NL, '\n', (i, 0), (i,1), '\n' for t in stack: if t[0] in (tokenize.DEDENT, tokenize.INDENT): diff --git a/mitogen/parent.py b/mitogen/parent.py index 721e3c0e..9d1a0ce3 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -80,7 +80,6 @@ except IOError: if sys.version_info >= (3, 0): - xrange = range closure_attr = '__closure__' IM_SELF_ATTR = '__self__' else: @@ -1729,7 +1728,7 @@ class ChildIdAllocator(object): def __init__(self, router): self.router = router self.lock = threading.Lock() - self.it = iter(xrange(0)) + self.it = iter(mitogen.core.range(0)) def allocate(self): """ @@ -1753,7 +1752,7 @@ class ChildIdAllocator(object): start, end = master.send_await( mitogen.core.Message(dst_id=0, handle=mitogen.core.ALLOCATE_ID) ) - self.it = iter(xrange(start, end)) + self.it = iter(mitogen.core.range(start, end)) finally: self.lock.release() diff --git a/mitogen/select.py b/mitogen/select.py index 2d87574f..d719f78c 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -233,7 +233,7 @@ class Select(object): # the underlying receivers. We handle the possibility of receivers # marked notified yet empty inside Select.get(), so this should be # robust. - for _ in range(recv.size()): + for _ in mitogen.core.range(recv.size()): self._put(recv) not_present_msg = 'Instance is not a member of this Select' diff --git a/mitogen/service.py b/mitogen/service.py index f82d18ad..b852f44d 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -523,7 +523,7 @@ class Pool(object): self.add(service) self._py_24_25_compat() self._threads = [] - for x in range(size): + for x in mitogen.core.range(size): name = 'mitogen.Pool.%04x.%d' % (id(self) & 0xffff, x,) thread = threading.Thread( name=name, diff --git a/tests/bench/connection/fork_lifecycle.py b/tests/bench/connection/fork_lifecycle.py new file mode 100644 index 00000000..6491c69b --- /dev/null +++ b/tests/bench/connection/fork_lifecycle.py @@ -0,0 +1,27 @@ +""" +Measure latency of .fork() setup/teardown. +""" + +import mitogen +import mitogen.core + + +@mitogen.main() +def main(router): + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.add_option( + '-i', '--iterations', type=int, metavar='N', default=200, + help='Number of iterations (default %default)') + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + + t0 = mitogen.core.now() + for x in mitogen.core.range(opts.iterations): + t = mitogen.core.now() + ctx = router.fork(debug=opts.debug) + ctx.shutdown(wait=True) + + t1 = mitogen.core.now() + mean = (t1 - t0) / opts.iterations + print('++ iterations %d, mean %.03f ms' % (opts.iterations, 1e3 * mean)) diff --git a/tests/bench/connection/fork_roundtrip.py b/tests/bench/connection/fork_roundtrip.py new file mode 100644 index 00000000..ac5be2d5 --- /dev/null +++ b/tests/bench/connection/fork_roundtrip.py @@ -0,0 +1,34 @@ +""" +Measure latency of local RPC. +""" + +import mitogen.core +import mitogen.utils +import ansible_mitogen.affinity + +mitogen.utils.setup_gil() +ansible_mitogen.affinity.policy.assign_worker() + + +def do_nothing(): + pass + +@mitogen.main() +def main(router): + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.add_option( + '-i', '--iterations', type=int, metavar='N', default=20000, + help='Number of iterations (default %default)') + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + + f = router.fork(debug=opts.debug) + f.call(do_nothing) + t0 = mitogen.core.now() + for x in mitogen.core.range(opts.iterations): + f.call(do_nothing) + + t1 = mitogen.core.now() + mean = (t1 - t0) / opts.iterations + print('++ iterations %d, mean %.03f us' % (opts.iterations, 1e6 * mean)) diff --git a/tests/bench/connection/local_lifecycle.py b/tests/bench/connection/local_lifecycle.py new file mode 100644 index 00000000..1c6bd07b --- /dev/null +++ b/tests/bench/connection/local_lifecycle.py @@ -0,0 +1,33 @@ +""" +Measure latency of .local() setup. +""" + +import mitogen +import mitogen.core +import mitogen.utils +import ansible_mitogen.affinity + + +mitogen.utils.setup_gil() +#ansible_mitogen.affinity.policy.assign_worker() + + +@mitogen.main() +def main(router): + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.add_option( + '-i', '--iterations', type=int, metavar='N', default=100, + help='Number of iterations (default %default)') + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + + t0 = mitogen.core.now() + for x in mitogen.core.range(opts.iterations): + t = mitogen.core.now() + f = router.local(debug=opts.debug) + tt = mitogen.core.now() + + t1 = mitogen.core.now() + mean = (t1 - t0) / opts.iterations + print('++ iterations %d, mean %.03f ms' % (opts.iterations, 1e3 * mean)) diff --git a/tests/bench/connection/ssh_roundtrip.py b/tests/bench/connection/ssh_roundtrip.py new file mode 100644 index 00000000..6afe7da5 --- /dev/null +++ b/tests/bench/connection/ssh_roundtrip.py @@ -0,0 +1,46 @@ +""" +Measure latency of SSH RPC. +""" + +import getpass + +import mitogen.core +import mitogen.utils +import ansible_mitogen.affinity + +mitogen.utils.setup_gil() +ansible_mitogen.affinity.policy.assign_worker() + + +def do_nothing(): + pass + +@mitogen.main() +def main(router): + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.add_option( + '-p', '--python', metavar='CMD', default='python3', + help='Remote python path (default %default)') + parser.add_option( + '-u', '--user', metavar='S', default=getpass.getuser(), + help='Remote username (default %default)') + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + + f = router.ssh( + hostname=args[0], python_path=opts.python, username=opts.user, + debug=opts.debug, + ) + f.call(do_nothing) + t0 = mitogen.core.now() + end = mitogen.core.now() + 5.0 + i = 0 + while mitogen.core.now() < end: + f.call(do_nothing) + i += 1 + + t1 = mitogen.core.now() + iterations = i + 1 + mean = (t1 - t0) / iterations + print('++ iterations %d, mean %.03f ms' % (iterations, 1e3 * mean)) diff --git a/tests/bench/fork.py b/tests/bench/fork.py deleted file mode 100644 index d77f4437..00000000 --- a/tests/bench/fork.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -Measure latency of .fork() setup/teardown. -""" - -import mitogen -import mitogen.core - -try: - xrange -except NameError: - xrange = range - - -@mitogen.main() -def main(router): - t0 = mitogen.core.now() - for x in xrange(200): - t = mitogen.core.now() - ctx = router.fork() - ctx.shutdown(wait=True) - print('++ %d' % 1000 * ((mitogen.core.now() - t0) / (1.0+x))) diff --git a/tests/bench/large_messages.py b/tests/bench/large_messages.py index 8593a33a..823548d4 100644 --- a/tests/bench/large_messages.py +++ b/tests/bench/large_messages.py @@ -1,4 +1,4 @@ -# Verify _receive_one() quadratic behaviour fixed. +'Measure throughput of messages.' import mitogen import mitogen.core @@ -6,20 +6,27 @@ import mitogen.core @mitogen.main() def main(router): - c = router.fork() + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.add_option( + '-i', '--iterations', type=int, metavar='N', default=10, + help='Number of iterations (default %default)') + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + + c = router.fork(debug=opts.debug) n = 1048576 * 127 s = ' ' * n - print('bytes in %.2fMiB string...' % (n/1048576.0),) t0 = mitogen.core.now() - for x in range(10): + for x in mitogen.core.range(opts.iterations): tt0 = mitogen.core.now() assert n == c.call(len, s) - print('took %dms' % (1000 * (mitogen.core.now() - tt0),)) + t1 = mitogen.core.now() - print('total %dms / %dms avg / %.2fMiB/sec' % ( - 1000 * (t1 - t0), - (1000 * (t1 - t0)) / (x + 1), - ((n * (x + 1)) / (t1 - t0)) / 1048576.0, - )) + mean = (t1 - t0) / opts.iterations + transferred_size = n * opts.iterations + transfer_rate = transferred_size / (t1 - t0) + print('++ iterations %d, mean %.03f ms, rate %.03f MiB/s' + % (opts.iterations, 1e3 * mean, transfer_rate / 2**20)) diff --git a/tests/bench/latch_roundtrip.py b/tests/bench/latch_roundtrip.py index cbc48374..b28fd523 100644 --- a/tests/bench/latch_roundtrip.py +++ b/tests/bench/latch_roundtrip.py @@ -15,7 +15,7 @@ X = 20000 def flip_flop(ready, inp, out): ready.put(None) - for x in xrange(X): + for x in mitogen.core.range(X): inp.get() out.put(None) diff --git a/tests/bench/local.py b/tests/bench/local.py deleted file mode 100644 index 0653a0de..00000000 --- a/tests/bench/local.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Measure latency of .local() setup. -""" - -import mitogen -import mitogen.core -import mitogen.utils -import ansible_mitogen.affinity - - -mitogen.utils.setup_gil() -#ansible_mitogen.affinity.policy.assign_worker() - - -@mitogen.main() -def main(router): - t0 = mitogen.core.now() - for x in range(100): - t = mitogen.core.now() - f = router.local()# debug=True) - tt = mitogen.core.now() - print(x, 1000 * (tt - t)) - print('%.03f ms' % (1000 * (mitogen.core.now() - t0) / (1.0 + x))) diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py deleted file mode 100644 index 774dfc5b..00000000 --- a/tests/bench/roundtrip.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Measure latency of local RPC. -""" - -import mitogen.core -import mitogen.utils -import ansible_mitogen.affinity - -mitogen.utils.setup_gil() -ansible_mitogen.affinity.policy.assign_worker() - -try: - xrange -except NameError: - xrange = range - -def do_nothing(): - pass - -@mitogen.main() -def main(router): - f = router.fork() - f.call(do_nothing) - t0 = mitogen.core.now() - for x in xrange(20000): - f.call(do_nothing) - print('++', int(1e6 * ((mitogen.core.now() - t0) / (1.0+x))), 'usec') diff --git a/tests/bench/ssh-roundtrip.py b/tests/bench/ssh-roundtrip.py deleted file mode 100644 index c5f14574..00000000 --- a/tests/bench/ssh-roundtrip.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Measure latency of SSH RPC. -""" - -import sys - -import mitogen.core -import mitogen.utils -import ansible_mitogen.affinity - -mitogen.utils.setup_gil() -ansible_mitogen.affinity.policy.assign_worker() - -try: - xrange -except NameError: - xrange = range - -def do_nothing(): - pass - -@mitogen.main() -def main(router): - f = router.ssh(hostname=sys.argv[1]) - f.call(do_nothing) - t0 = mitogen.core.now() - end = mitogen.core.now() + 5.0 - i = 0 - while mitogen.core.now() < end: - f.call(do_nothing) - i += 1 - t1 = mitogen.core.now() - - print('++', float(1e3 * (t1 - t0) / (1.0+i)), 'ms') diff --git a/tests/bench/throughput.py b/tests/bench/throughput.py index 7d67d158..d5019c95 100644 --- a/tests/bench/throughput.py +++ b/tests/bench/throughput.py @@ -1,5 +1,8 @@ -# Verify throughput over sudo and SSH at various compression levels. +''' +Measure file service throughput over local, sudo, and (un)compressed SSH. +''' +import getpass import os import tempfile @@ -43,6 +46,23 @@ def run_test(router, fp, s, context): @mitogen.main() def main(router): + import optparse + parser = optparse.OptionParser(description=__doc__) + + parser.add_option( + '--ssh-python', metavar='CMD', default='python3', + help='Remote python path (default %default)') + parser.add_option( + '--ssh-user', metavar='S', default=getpass.getuser(), + help='Remote username (default %default)') + + parser.add_option( + '--sudo-user', metavar='S', default='root', + help='Sudo username (default %default)') + + parser.add_option('--debug', action='store_true') + opts, args = parser.parse_args() + ansible_mitogen.affinity.policy.assign_muxprocess() bigfile = tempfile.NamedTemporaryFile() @@ -53,19 +73,29 @@ def main(router): file_service.register(bigfile.name) pool.add(file_service) try: - context = router.local() + context = router.local(debug=opts.debug) run_test(router, bigfile, 'local()', context) context.shutdown(wait=True) - context = router.sudo() + context = router.sudo(username=opts.sudo_user, debug=opts.debug) run_test(router, bigfile, 'sudo()', context) context.shutdown(wait=True) - context = router.ssh(hostname='localhost', compression=False) + context = router.ssh( + hostname=args[0], + python_path=opts.ssh_python, + username=opts.ssh_user, + compression=False, debug=opts.debug, + ) run_test(router, bigfile, 'ssh(compression=False)', context) context.shutdown(wait=True) - context = router.ssh(hostname='localhost', compression=True) + context = router.ssh( + hostname=args[0], + python_path=opts.ssh_python, + username=opts.ssh_user, + compression=True, debug=opts.debug, + ) run_test(router, bigfile, 'ssh(compression=True)', context) context.shutdown(wait=True) finally: diff --git a/tests/soak/cpu_load.py b/tests/soak/cpu_load.py index cfc65896..f97bf867 100644 --- a/tests/soak/cpu_load.py +++ b/tests/soak/cpu_load.py @@ -8,6 +8,8 @@ import ctypes import multiprocessing import os +import mitogen.core + LIBC = ctypes.CDLL('libc.so.6') sched_yield = LIBC.sched_yield @@ -19,7 +21,7 @@ def burn(): (ord(b) << 8) | (ord(c) << 0)) / 1.6) print(n) - for x in xrange(n): pass + for x in mitogen.core.range(n): pass sched_yield() mul = 1.5