Use a consistent RLIMIT_NOFILE for ansible-test.

test/runner/lib/ansible_util.py

@@ -4,6 +4,10 @@ from __future__ import absolute_import, print_function
 
 import os
 
+from lib.constants import (
+    SOFT_RLIMIT_NOFILE,
+)
+
 from lib.util import (
     common_environment,
     ApplicationError,
@@ -40,7 +44,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         raise ApplicationError('Configuration not found: %s' % ansible_config)
 
     ansible = dict(
-        ANSIBLE_PYTHON_MODULE_RLIMIT_NOFILE='1024',
+        ANSIBLE_PYTHON_MODULE_RLIMIT_NOFILE=str(SOFT_RLIMIT_NOFILE),
         ANSIBLE_FORCE_COLOR='%s' % 'true' if args.color and color else 'false',
         ANSIBLE_DEPRECATION_WARNINGS='false',
         ANSIBLE_HOST_KEY_CHECKING='false',

test/runner/lib/cli.py

@@ -4,9 +4,14 @@ from __future__ import absolute_import, print_function
 
 import errno
 import os
-import resource
 import sys
 
+# This import should occur as early as possible.
+# It must occur before subprocess has been imported anywhere in the current process.
+from lib.init import (
+    CURRENT_RLIMIT_NOFILE,
+)
+
 from lib.util import (
     ApplicationError,
     display,
@@ -14,6 +19,7 @@ from lib.util import (
     get_docker_completion,
     generate_pip_command,
     read_lines_without_comments,
+    MAXFD,
 )
 
 from lib.delegation import (
@@ -90,16 +96,8 @@ def main():
         display.info_stderr = (isinstance(config, SanityConfig) and config.lint) or (isinstance(config, IntegrationConfig) and config.list_targets)
         check_startup()
 
-        # to achieve a consistent nofile ulimit, set to 16k here, this can affect performance in subprocess.Popen when
+        display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE,), verbosity=2)
-        # being called with close_fds=True on Python (8x the time on some environments)
+        display.info('MAXFD: %d' % MAXFD, verbosity=2)
-        nofile_limit = 16 * 1024
-        current_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-        new_limit = (nofile_limit, nofile_limit)
-        if current_limit > new_limit:
-            display.info('RLIMIT_NOFILE: %s -> %s' % (current_limit, new_limit), verbosity=2)
-            resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_limit, nofile_limit))
-        else:
-            display.info('RLIMIT_NOFILE: %s' % (current_limit, ), verbosity=2)
 
         try:
             args.func(config)

test/runner/lib/constants.py

@@ -0,0 +1,5 @@
+"""Constants used by ansible-test. Imports should not be used in this file."""
+
+# Setting a low soft RLIMIT_NOFILE value will improve the performance of subprocess.Popen on Python 2.x when close_fds=True.
+# This will affect all Python subprocesses. It will also affect the current Python process if set before subprocess is imported for the first time.
+SOFT_RLIMIT_NOFILE = 1024

test/runner/lib/init.py

@@ -0,0 +1,14 @@
+"""Early initialization for ansible-test before most other imports have been performed."""
+
+import resource
+
+from lib.constants import (
+    SOFT_RLIMIT_NOFILE,
+)
+
+CURRENT_RLIMIT_NOFILE = resource.getrlimit(resource.RLIMIT_NOFILE)
+DESIRED_RLIMIT_NOFILE = (SOFT_RLIMIT_NOFILE, CURRENT_RLIMIT_NOFILE[1])
+
+if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE:
+    resource.setrlimit(resource.RLIMIT_NOFILE, DESIRED_RLIMIT_NOFILE)
+    CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE

test/runner/lib/util.py

@@ -41,6 +41,11 @@ except ImportError:
 DOCKER_COMPLETION = {}
 COVERAGE_PATHS = {}  # type: dict[str, str]
 
+try:
+    MAXFD = subprocess.MAXFD
+except AttributeError:
+    MAXFD = -1
+
 
 def get_docker_completion():
     """