From eeb7150f2485a4ff465b124de02e512ea8410251 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 04:20:04 +0100 Subject: [PATCH] issue #549: increase open file limit automatically if possible While catching every possible case where "open file limit exceeded" is not possible, we can at least increase the soft limit to the available hard limit without any user effort. Do this in Ansible top-level process, even though we probably only need it in the MuxProcess. It seems there is no reason this could hurt --- ansible_mitogen/process.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 1adc464f..85ced27f 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -32,6 +32,7 @@ import errno import logging import multiprocessing import os +import resource import signal import socket import sys @@ -237,9 +238,27 @@ def _setup_responder(responder): ) +def increase_open_file_limit(): + """ + #549: in order to reduce the possibility of hitting an open files limit, + increase :data:`resource.RLIMIT_NOFILE` from its soft limit to its hard + limit, if they differ. + + It is common that a low soft limit is configured by default, where the hard + limit is much higher. + """ + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < hard: + LOG.debug('raising soft open file limit from %d to %d', soft, hard) + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + else: + LOG.debug('cannot increase open file limit; existing limit is %d', hard) + + def common_setup(enable_affinity=True, _init_logging=True): save_pid('controller') ansible_mitogen.logging.set_process_name('top') + if enable_affinity: ansible_mitogen.affinity.policy.assign_controller() @@ -255,6 +274,7 @@ def common_setup(enable_affinity=True, _init_logging=True): mitogen.core.enable_profiling() MuxProcess.cls_original_env = dict(os.environ) + increase_open_file_limit() def get_cpu_count(default=None):