diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py
index 63caa88a..3a41a43d 100644
--- a/ansible_mitogen/process.py
+++ b/ansible_mitogen/process.py
@@ -281,11 +281,11 @@ def get_cpu_count(default=None):
class Broker(mitogen.master.Broker):
"""
- WorkerProcess maintains at most 2 file descriptors, therefore does not need
+ WorkerProcess maintains fewer file descriptors, therefore does not need
the exuberant syscall expense of EpollPoller, so override it and restore
the poll() poller.
"""
- poller_class = mitogen.core.Poller
+ poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
class Binding(object):
diff --git a/docs/changelog.rst b/docs/changelog.rst
index bcf6d01e..407a8c78 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -22,6 +22,8 @@ Unreleased
----------
* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
+* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
+ "ValueError: filedescriptor out of range in select()"
v0.3.7 (2024-04-08)
diff --git a/docs/contributors.rst b/docs/contributors.rst
index 207e4d7b..ed7fef11 100644
--- a/docs/contributors.rst
+++ b/docs/contributors.rst
@@ -126,11 +126,13 @@ sponsorship and outstanding future-thinking of its early adopters.
jgadling
John F Wall — Making Ansible Great with Massive Parallelism
KennethC
+ Luca Berruti
Lewis Bellwood — Happy to be apart of a great project.
luto
Mayeu a.k.a Matthieu Maury
@nathanhruby
Orion Poplawski
+ Philippe Kueck
Ramy
Scott Vokes
Tom Eichhorn
diff --git a/mitogen/core.py b/mitogen/core.py
index cd02012f..cdfbbcde 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
@@ -2523,8 +2523,7 @@ class Poller(object):
"""
A poller manages OS file descriptors the user is waiting to become
available for IO. The :meth:`poll` method blocks the calling thread
- until one or more become ready. The default implementation is based on
- :func:`select.poll`.
+ until one or more become ready.
Each descriptor has an associated `data` element, which is unique for each
readiness type, and defaults to being the same as the file descriptor. The
@@ -2546,19 +2545,13 @@ class Poller(object):
a resource leak.
Pollers may only be used by one thread at a time.
+
+ This implementation uses :func:`select.select` for wider platform support.
+ That is considered an implementation detail. Previous versions have used
+ :func:`select.poll`. Future versions may decide at runtime.
"""
SUPPORTED = True
- # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
- # no upper FD limit, it is suitable for use with Latch, which must handle
- # FDs larger than select's limit during many-host runs. We want this
- # because poll() requires no setup and teardown: just a single system call,
- # which is important because Latch.get() creates a Poller on each
- # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in
- # this scenario. If select() must return in future, it is important
- # Latch.poller_class is set from parent.py to point to the industrial
- # strength poller for the OS, otherwise Latch will fail randomly.
-
#: Increments on every poll(). Used to version _rfds and _wfds.
_generation = 1
@@ -2681,11 +2674,10 @@ class Latch(object):
See :ref:`waking-sleeping-threads` for further discussion.
"""
- #: The :class:`Poller` implementation to use for waiting. Since the poller
- #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller`
- #: if it is available, or :class:`mitogen.core.Poller` otherwise, since
- #: these implementations require no system calls to create, configure or
- #: destroy.
+ #: The :class:`Poller` implementation to use. Instances are short lived so
+ #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise
+ #: :class:`mitogen.core.Poller`. They don't need syscalls to create,
+ #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`.
poller_class = Poller
#: If not :data:`None`, a function invoked as `notify(latch)` after a
diff --git a/mitogen/parent.py b/mitogen/parent.py
index 29bcf66d..4b96dcf4 100644
--- a/mitogen/parent.py
+++ b/mitogen/parent.py
@@ -745,8 +745,7 @@ def _upgrade_broker(broker):
broker.timers = TimerList()
LOG.debug('upgraded %r with %r (new: %d readers, %d writers; '
'old: %d readers, %d writers)', old, new,
- len(new.readers), len(new.writers),
- len(old.readers), len(old.writers))
+ len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds))
@mitogen.core.takes_econtext
@@ -902,22 +901,18 @@ class CallSpec(object):
class PollPoller(mitogen.core.Poller):
"""
Poller based on the POSIX :linux:man2:`poll` interface. Not available on
- some versions of OS X, otherwise it is the preferred poller for small FD
- counts, as there is no setup/teardown/configuration system call overhead.
+ some Python/OS X combinations. Otherwise the preferred poller for small
+ FD counts; or if many pollers are created, used once, then closed.
+ There there is no setup/teardown/configuration system call overhead.
"""
SUPPORTED = hasattr(select, 'poll')
- _repr = 'PollPoller()'
+ _readmask = SUPPORTED and select.POLLIN | select.POLLHUP
def __init__(self):
super(PollPoller, self).__init__()
self._pollobj = select.poll()
# TODO: no proof we dont need writemask too
- _readmask = (
- getattr(select, 'POLLIN', 0) |
- getattr(select, 'POLLHUP', 0)
- )
-
def _update(self, fd):
mask = (((fd in self._rfds) and self._readmask) |
((fd in self._wfds) and select.POLLOUT))
@@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller):
Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
"""
SUPPORTED = hasattr(select, 'kqueue')
- _repr = 'KqueuePoller()'
def __init__(self):
super(KqueuePoller, self).__init__()
@@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller):
Poller based on the Linux :linux:man7:`epoll` interface.
"""
SUPPORTED = hasattr(select, 'epoll')
- _repr = 'EpollPoller()'
+ _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP
def __init__(self):
super(EpollPoller, self).__init__()
@@ -1077,9 +1071,6 @@ class EpollPoller(mitogen.core.Poller):
self._wfds.pop(fd, None)
self._control(fd)
- _inmask = (getattr(select, 'EPOLLIN', 0) |
- getattr(select, 'EPOLLHUP', 0))
-
def _poll(self, timeout):
the_timeout = -1
if timeout is not None:
@@ -1100,18 +1091,14 @@ class EpollPoller(mitogen.core.Poller):
yield data
-# 2.4 and 2.5 only had select.select() and select.poll().
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
- if _klass.SUPPORTED:
- PREFERRED_POLLER = _klass
+POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller)
+PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED)
+
# For processes that start many threads or connections, it's possible Latch
# will also get high-numbered FDs, and so select() becomes useless there too.
-# So swap in our favourite poller.
-if PollPoller.SUPPORTED:
- mitogen.core.Latch.poller_class = PollPoller
-else:
- mitogen.core.Latch.poller_class = PREFERRED_POLLER
+POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER
+mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT
class LineLoggingProtocolMixin(object):