From efdd82d1ab1d7e298de7c19b9a2508ba73d1d539 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 18 Apr 2024 14:35:32 +0100 Subject: [PATCH] mitogen: Streamline Poller classes and Latch.poller_class selection This - Clarifies and corrects docstrings and comments based on investigation for #957 - Removes unused `Poller*._repr` attributes - Eliminates some uses of `getattr()` - Introduces `mitogen.parent.POLLERS` & `mitogen.parent.POLLER_LIGHTWEIGHT` Preamble size change ``` @@ -1,7 +1,7 @@ SSH command size: 759 -Bootstrap (mitogen.core) size: 17862 (17.44KiB) +Bootstrap (mitogen.core) size: 17934 (17.51KiB) Original Minimized Compressed -mitogen.parent 98171 95.9KiB 50569 49.4KiB 51.5% 12742 12.4KiB 13.0% +mitogen.parent 96979 94.7KiB 49844 48.7KiB 51.4% 12697 12.4KiB 13.1% mitogen.fork 8436 8.2KiB 4130 4.0KiB 49.0% 1648 1.6KiB 19.5% mitogen.ssh 10892 10.6KiB 6952 6.8KiB 63.8% 2113 2.1KiB 19.4% ``` --- mitogen/core.py | 26 +++++++++----------------- mitogen/parent.py | 35 +++++++++++------------------------ 2 files changed, 20 insertions(+), 41 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index cd02012f..cdfbbcde 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2523,8 +2523,7 @@ class Poller(object): """ A poller manages OS file descriptors the user is waiting to become available for IO. The :meth:`poll` method blocks the calling thread - until one or more become ready. The default implementation is based on - :func:`select.poll`. + until one or more become ready. Each descriptor has an associated `data` element, which is unique for each readiness type, and defaults to being the same as the file descriptor. The @@ -2546,19 +2545,13 @@ class Poller(object): a resource leak. Pollers may only be used by one thread at a time. + + This implementation uses :func:`select.select` for wider platform support. + That is considered an implementation detail. Previous versions have used + :func:`select.poll`. Future versions may decide at runtime. """ SUPPORTED = True - # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has - # no upper FD limit, it is suitable for use with Latch, which must handle - # FDs larger than select's limit during many-host runs. We want this - # because poll() requires no setup and teardown: just a single system call, - # which is important because Latch.get() creates a Poller on each - # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in - # this scenario. If select() must return in future, it is important - # Latch.poller_class is set from parent.py to point to the industrial - # strength poller for the OS, otherwise Latch will fail randomly. - #: Increments on every poll(). Used to version _rfds and _wfds. _generation = 1 @@ -2681,11 +2674,10 @@ class Latch(object): See :ref:`waking-sleeping-threads` for further discussion. """ - #: The :class:`Poller` implementation to use for waiting. Since the poller - #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller` - #: if it is available, or :class:`mitogen.core.Poller` otherwise, since - #: these implementations require no system calls to create, configure or - #: destroy. + #: The :class:`Poller` implementation to use. Instances are short lived so + #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise + #: :class:`mitogen.core.Poller`. They don't need syscalls to create, + #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`. poller_class = Poller #: If not :data:`None`, a function invoked as `notify(latch)` after a diff --git a/mitogen/parent.py b/mitogen/parent.py index 29bcf66d..4b96dcf4 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -745,8 +745,7 @@ def _upgrade_broker(broker): broker.timers = TimerList() LOG.debug('upgraded %r with %r (new: %d readers, %d writers; ' 'old: %d readers, %d writers)', old, new, - len(new.readers), len(new.writers), - len(old.readers), len(old.writers)) + len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds)) @mitogen.core.takes_econtext @@ -902,22 +901,18 @@ class CallSpec(object): class PollPoller(mitogen.core.Poller): """ Poller based on the POSIX :linux:man2:`poll` interface. Not available on - some versions of OS X, otherwise it is the preferred poller for small FD - counts, as there is no setup/teardown/configuration system call overhead. + some Python/OS X combinations. Otherwise the preferred poller for small + FD counts; or if many pollers are created, used once, then closed. + There there is no setup/teardown/configuration system call overhead. """ SUPPORTED = hasattr(select, 'poll') - _repr = 'PollPoller()' + _readmask = SUPPORTED and select.POLLIN | select.POLLHUP def __init__(self): super(PollPoller, self).__init__() self._pollobj = select.poll() # TODO: no proof we dont need writemask too - _readmask = ( - getattr(select, 'POLLIN', 0) | - getattr(select, 'POLLHUP', 0) - ) - def _update(self, fd): mask = (((fd in self._rfds) and self._readmask) | ((fd in self._wfds) and select.POLLOUT)) @@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller): Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface. """ SUPPORTED = hasattr(select, 'kqueue') - _repr = 'KqueuePoller()' def __init__(self): super(KqueuePoller, self).__init__() @@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller): Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') - _repr = 'EpollPoller()' + _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP def __init__(self): super(EpollPoller, self).__init__() @@ -1077,9 +1071,6 @@ class EpollPoller(mitogen.core.Poller): self._wfds.pop(fd, None) self._control(fd) - _inmask = (getattr(select, 'EPOLLIN', 0) | - getattr(select, 'EPOLLHUP', 0)) - def _poll(self, timeout): the_timeout = -1 if timeout is not None: @@ -1100,18 +1091,14 @@ class EpollPoller(mitogen.core.Poller): yield data -# 2.4 and 2.5 only had select.select() and select.poll(). -for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: - if _klass.SUPPORTED: - PREFERRED_POLLER = _klass +POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller) +PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED) + # For processes that start many threads or connections, it's possible Latch # will also get high-numbered FDs, and so select() becomes useless there too. -# So swap in our favourite poller. -if PollPoller.SUPPORTED: - mitogen.core.Latch.poller_class = PollPoller -else: - mitogen.core.Latch.poller_class = PREFERRED_POLLER +POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER +mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT class LineLoggingProtocolMixin(object):