issue #159: make LRU size configurable.

ansible_mitogen/services.py

@@ -39,9 +39,10 @@ when a child has completed a job.
 
 from __future__ import absolute_import
 import logging
-import sys
+import os
 import os.path
 import pprint
+import sys
 import threading
 import zlib
 
@@ -73,7 +74,7 @@ class ContextService(mitogen.service.Service):
     """
     handle = 500
     max_message_size = 1000
-    max_contexts = 20
+    max_interpreters = int(os.getenv('MITOGEN_MAX_INTERPRETERS', '20'))
 
     def __init__(self, *args, **kwargs):
         super(ContextService, self).__init__(*args, **kwargs)
@@ -127,7 +128,7 @@ class ContextService(mitogen.service.Service):
             return
 
         lru = self._lru_by_via.setdefault(via, [])
-        if len(lru) < self.max_contexts:
+        if len(lru) < self.max_interpreters:
             lru.append(new_context)
             return
 

docs/ansible.rst

@@ -369,11 +369,10 @@ future.
 Interpreter Recycling
 ~~~~~~~~~~~~~~~~~~~~~
 
-To prevent accidental DoS, the extension stops creating persistent interpreters
+The extension stops limits the number of persistent interpreters in use. When
-after the 20th interpreter has been created. Instead the most recently created
+the limit is reached, the youngest interpreter is terminated before starting a
-interpreter is shut down to make room for any new interpreter. This is to avoid
+new interpreter, preventing situations like below from triggering memory
-situations like below from triggering memory exhaustion by spawning a huge
+exhaustion.
-number of interpreters.
 
 .. code-block:: yaml
 
@@ -392,15 +391,18 @@ number of interpreters.
             dest: "~{{item}}/.bashrc"
           with_items: "{{user_directory}}"
 
-The recycling behaviour does not occur for direct connections from the
+This recycling does not occur for direct connections from the controller, and
-controller, and it is keyed on a per-host basis, i.e. up to 20 interpreters may
+it is keyed on a per-target basis, i.e. up to 20 interpreters may exist for
-exist for each directly connected target host.
+each directly connected target.
 
-The newest interpreter is chosen to avoid recycling useful accounts, like
+The youngest interpreter is chosen to preserve useful accounts, like "root" or
-"root" or "postgresql" that tend to appear early in a run, however it is simple
+"postgresql" that tend to appear early in a run, however it is simple to
-to construct a playbook that defeats this strategy. A future version will key
+construct a playbook that defeats this strategy. A future version will key
-interpreters on the identity of the task, file and/or playbook that created
+interpreters on the identity of their creating task, file and/or playbook,
-them, avoiding the recycling of useful accounts in every scenario.
+avoiding useful account recycling in every scenario.
+
+To raise or lower the limit from 20, set the ``MITOGEN_MAX_INTERPRETERS``
+environment variable to a new value.
 
 
 Runtime Patches

tests/ansible/integration/context_service/lru_one_target.yml

@@ -2,6 +2,10 @@
 
 - hosts: all
   any_errors_fatal: true
+  vars:
+    max_interps: "{{lookup('env', 'MITOGEN_MAX_INTERPRETERS')}}"
+    ubound: "{{max_interps|int + 1}}"
+
   tasks:
     - name: integration/context_service/lru_one_target.yml
       assert:
@@ -12,7 +16,7 @@
       become: true
       vars:
         ansible_become_user: "mitogen__user{{item}}"
-      with_sequence: start=1 end=21
+      with_sequence: start=1 end={{ubound}}
       register: first_run
 
     - name: Reuse them
@@ -20,14 +24,16 @@
       become: true
       vars:
         ansible_become_user: "mitogen__user{{item}}"
-      with_sequence: start=1 end=21
+      with_sequence: start=1 end={{ubound}}
       register: second_run
 
     - assert:
         that:
         - first_run.results[item|int].pid == second_run.results[item|int].pid
-      with_items: start=0 end=20
+      with_items: start=0 end={{max_interps}}
+      when: is_mitogen
 
     - assert:
         that:
         - first_run.results[-1].pid != second_run.results[-1].pid
+      when: is_mitogen

tests/ansible/run_ansible_playbook.sh

@@ -3,6 +3,7 @@
 
 # Used by delegate_to.yml to ensure "sudo -E" preserves environment.
 export I_WAS_PRESERVED=1
+export MITOGEN_MAX_INTERPRETERS=3
 
 if [ "${ANSIBLE_STRATEGY:0:7}" = "mitogen" ]
 then