Merge pull request #1013 from moreati/issue1011-blacklist-msg

mitogen: Clarify blacklisted import error message

ansible_mitogen/mixins.py

@@ -470,18 +470,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         # chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
         # which is required by Ansible's discover_interpreter function
         if self._mitogen_discovering_interpreter:
-            possible_pythons = [
-                '/usr/bin/python',
-                'python3',
-                'python3.7',
-                'python3.6',
-                'python3.5',
-                'python2.7',
-                'python2.6',
-                '/usr/libexec/platform-python',
-                '/usr/bin/python3',
-                'python'
-            ]
+            possible_pythons = self._mitogen_interpreter_candidates
         else:
             # not used, just adding a filler value
             possible_pythons = ['python']
@@ -492,12 +481,15 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                 rc, stdout, stderr = self._connection.exec_command(
                     cmd, in_data, sudoable, mitogen_chdir=chdir,
                 )
-            # TODO: what exception is thrown?
-            except:
+            except BaseException as exc:
                 # we've reached the last python attempted and failed
                 if possible_python == possible_pythons[-1]:
                     raise
                 else:
+                    LOG.debug(
+                        '%r._low_level_execute_command: candidate=%r ignored: %s, %r',
+                        self, possible_python, type(exc), exc,
+                    )
                     continue
 
         stdout_text = to_text(stdout, errors=encoding_errors)

ansible_mitogen/target.py

@@ -40,7 +40,6 @@ import errno
 import grp
 import json
 import logging
-import operator
 import os
 import pty
 import pwd
@@ -66,8 +65,6 @@ if not sys.modules.get(str('__main__')):
 
 import ansible.module_utils.json_utils
 
-from ansible.module_utils.six.moves import reduce
-
 import ansible_mitogen.runner
 
 
@@ -718,7 +715,9 @@ def apply_mode_spec(spec, mode):
             mask = CHMOD_MASKS[ch]
             bits = CHMOD_BITS[ch]
             cur_perm_bits = mode & mask
-            new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0)
+            new_perm_bits = 0
+            for perm in perms:
+                new_perm_bits |= bits[perm]
             mode &= ~mask
             if op == '=':
                 mode |= new_perm_bits

ansible_mitogen/transport_config.py

@@ -73,13 +73,21 @@ import ansible.utils.unsafe_proxy
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils.parsing.convert_bool import boolean
 
+import ansible_mitogen.utils
 import mitogen.core
 
 
 LOG = logging.getLogger(__name__)
 
+if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+    _FALLBACK_INTERPRETER = ansible.executor.interpreter_discovery._FALLBACK_INTERPRETER
+elif ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+    _FALLBACK_INTERPRETER = u'/usr/bin/python3'
+else:
+    _FALLBACK_INTERPRETER = u'/usr/bin/python'
 
-def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python):
+
+def run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python):
     """
     Triggers ansible python interpreter discovery if requested.
     Caches this value the same way Ansible does it.
@@ -107,8 +115,11 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # blow away the discovered_interpreter_config cache and rediscover
             del task_vars['ansible_facts'][discovered_interpreter_config]
 
-        if discovered_interpreter_config not in task_vars['ansible_facts']:
+        try:
+            s = task_vars[u'ansible_facts'][discovered_interpreter_config]
+        except KeyError:
             action._mitogen_discovering_interpreter = True
+            action._mitogen_interpreter_candidates = candidates
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
             s = ansible.executor.interpreter_discovery.discover_interpreter(
@@ -121,18 +132,17 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # cache discovered interpreter
             task_vars['ansible_facts'][discovered_interpreter_config] = s
             action._connection.has_pipelining = False
-        else:
-            s = task_vars['ansible_facts'][discovered_interpreter_config]
 
         # propagate discovered interpreter as fact
         action._discovered_interpreter_key = discovered_interpreter_config
         action._discovered_interpreter = s
 
     action._mitogen_discovering_interpreter = False
+    action._mitogen_interpreter_candidates = None
     return s
 
 
-def parse_python_path(s, task_vars, action, rediscover_python):
+def parse_python_path(s, candidates, task_vars, action, rediscover_python):
     """
     Given the string set for ansible_python_interpeter, parse it using shell
     syntax and return an appropriate argument vector. If the value detected is
@@ -143,10 +153,9 @@ def parse_python_path(s, task_vars, action, rediscover_python):
         # if python_path doesn't exist, default to `auto` and attempt to discover it
         s = 'auto'
 
-    s = run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python)
-    # if unable to determine python_path, fallback to '/usr/bin/python'
+    s = run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python)
     if not s:
-        s = '/usr/bin/python'
+        s = _FALLBACK_INTERPRETER
 
     return ansible.utils.shlex.shlex_split(s)
 
@@ -510,6 +519,9 @@ class PlayContextSpec(Spec):
         interpreter_python = C.config.get_config_value(
             'INTERPRETER_PYTHON', variables=variables,
         )
+        interpreter_python_fallback = C.config.get_config_value(
+            'INTERPRETER_PYTHON_FALLBACK', variables=variables,
+        )
 
         if '{{' in interpreter_python or '{%' in interpreter_python:
             templar = self._connection.templar
@@ -517,6 +529,7 @@ class PlayContextSpec(Spec):
 
         return parse_python_path(
             interpreter_python,
+            candidates=interpreter_python_fallback,
             task_vars=self._task_vars,
             action=self._action,
             rediscover_python=rediscover_python)
@@ -732,11 +745,12 @@ class MitogenViaSpec(Spec):
 
     def python_path(self, rediscover_python=False):
         s = self._host_vars.get('ansible_python_interpreter')
-        # #511, #536: executor/module_common.py::_get_shebang() hard-wires
-        # "/usr/bin/python" as the default interpreter path if no other
-        # interpreter is specified.
+        interpreter_python_fallback = self._host_vars.get(
+            'ansible_interpreter_python_fallback', [],
+        )
         return parse_python_path(
             s,
+            candidates=interpreter_python_fallback,
             task_vars=self._task_vars,
             action=self._action,
             rediscover_python=rediscover_python)

docs/ansible_detailed.rst

@@ -145,6 +145,8 @@ Noteworthy Differences
   +-----------------+ 3.11 - 3.14     |
   | 12              |                 |
   +-----------------+-----------------+
+  | 13              | 3.12 - 3.14     |
+  +-----------------+-----------------+
 
   Verify your installation is running one of these versions by checking
   ``ansible --version`` output.

docs/changelog.rst

@@ -18,6 +18,24 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+In progress (unreleased)
+------------------------
+
+* :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility
+* :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of
+  ``ansible.module_utils.six``
+* :gh:issue:`1354` docs: Document Ansible 13 (ansible-core 2.20) support
+* :gh:issue:`1354` :mod:`mitogen`: Clarify error message when a module
+  request would be refused by allow or deny listing
+
+
+v0.3.35 (2025-12-01)
+--------------------
+
+* :gh:issue:`1132` :mod:`ansible_mitogen` During intrepreter discovery use
+  Ansible ``INTERPRETER_PYTHON_FALLBACK`` config as list of candidates
+
+
 v0.3.34 (2025-11-27)
 --------------------
 
@@ -430,7 +448,7 @@ v0.3.1 (unreleased)
 * :gh:issue:`878` Kubectl connector fixed with Ansible 2.10 and above
 
 
-v0.3.0 (2021-10-28)
+v0.3.0 (2021-11-24)
 -------------------
 
 This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+.
@@ -443,7 +461,7 @@ This release separates itself from the v0.2.X releases. Ansible's API changed to
 * :gh:issue:`847` Removed historic Continuous Integration reverse shell
 
 
-v0.2.10 (2021-10-28)
+v0.2.10 (2021-11-24)
 --------------------
 
 * :gh:issue:`597` mitogen does not support Ansible 2.8 Python interpreter detection
@@ -458,7 +476,6 @@ v0.2.10 (2021-10-28)
   :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
 * :gh:issue:`775` Test with Python 3.9
 * :gh:issue:`775` Add msvcrt to the default module deny list
-* :gh:issue:`847` Removed historic Continuous Integration reverse shell
 
 
 v0.2.9 (2019-11-02)

docs/index.rst

@@ -332,12 +332,16 @@ a large fleet of machines, or to alert the parent of unexpected state changes.
 Compatibility
 #############
 
-Mitogen is compatible with **Python 2.4** released November 2004, making it
+``mitogen.*`` is compatible with Python 2.4 - 2.7 and 3.6 onward; making it
 suitable for managing a fleet of potentially ancient corporate hardware, such
 as Red Hat Enterprise Linux 5, released in 2007.
 
-Every combination of Python 3.x/2.x parent and child should be possible,
-however at present only Python 2.4, 2.6, 2.7 and 3.6 are tested automatically.
+Every combination of Python 3.x/2.x parent and child should be possible.
+Automated testing cannot cover every combination, automated testing tries to
+cover the extemities (e.g. Python 3.14 parent -> Python 2.4 child).
+
+``ansible_mitogen.*`` is compatible with Python 2.7 and 3.6 onward; making it
+suitable for Ansible 2.10 onward.
 
 
 Zero Dependencies

docs/svg-boxify.py

@@ -1,13 +0,0 @@
-
-# Add viewBox attr to SVGs lacking it, so IE scales properly.
-
-import lxml.etree
-import glob
-
-
-for name in glob.glob('images/*.svg') + glob.glob('images/ansible/*.svg'):
-    doc = lxml.etree.parse(open(name))
-    svg = doc.getroot()
-    if 'viewBox' not in svg.attrib:
-        svg.attrib['viewBox'] = '0 0 %(width)s %(height)s' % svg.attrib
-        open(name, 'w').write(lxml.etree.tostring(svg, xml_declaration=True, encoding='UTF-8'))

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 34)
+__version__ = (0, 3, 36, 'dev')
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -541,6 +541,7 @@ def is_blacklisted_import(importer, fullname):
     any packages have been whitelisted and `fullname` is not part of one.
 
     NB:
+      - The default whitelist is `['']` which matches any module name.
       - If a package is on both lists, then it is treated as blacklisted.
       - If any package is whitelisted, then all non-whitelisted packages are
         treated as blacklisted.
@@ -1536,9 +1537,8 @@ class Importer(object):
         return importlib.machinery.ModuleSpec(fullname, loader=self)
 
     blacklisted_msg = (
-        '%r is present in the Mitogen importer blacklist, therefore this '
-        'context will not attempt to request it from the master, as the '
-        'request will always be refused.'
+        'A %r request would be refused by the Mitogen master. The module is '
+        'on the deny list (blacklist) or not on the allow list (whitelist).'
     )
     pkg_resources_msg = (
         'pkg_resources is prohibited from importing __main__, as it causes '

setup.py

@@ -82,7 +82,7 @@ setup(
     license = 'BSD-3-Clause',
     url = 'https://github.com/mitogen-hq/mitogen/',
     packages = find_packages(exclude=['tests', 'examples']),
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
+    python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
     zip_safe = False,
     classifiers = [
         'Environment :: Console',
@@ -91,6 +91,9 @@ setup(
         'Operating System :: MacOS :: MacOS X',
         'Operating System :: POSIX',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2.4',
+        'Programming Language :: Python :: 2.5',
+        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',

tests/image_prep/hosts.ini

@@ -43,6 +43,7 @@ centos7
 centos8
 debian9
 debian10
+ubuntu1604
 ubuntu1804
 
 [ansible_11]

tests/responder_test.py

@@ -198,21 +198,3 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase):
         self.assertEqual(2+os_fork, self.router.responder.good_load_module_count)
         self.assertLess(10000, self.router.responder.good_load_module_size)
         self.assertGreater(40000, self.router.responder.good_load_module_size)
-
-
-class BlacklistTest(testlib.TestCase):
-    @unittest.skip('implement me')
-    def test_whitelist_no_blacklist(self):
-        assert 0
-
-    @unittest.skip('implement me')
-    def test_whitelist_has_blacklist(self):
-        assert 0
-
-    @unittest.skip('implement me')
-    def test_blacklist_no_whitelist(self):
-        assert 0
-
-    @unittest.skip('implement me')
-    def test_blacklist_has_whitelist(self):
-        assert 0

tox.ini

@@ -63,7 +63,7 @@ envlist =
     py{27,36}-m_ans-ans{2.10,3,4}
     py{311}-m_ans-ans{2.10,3-5}
     py{313}-m_ans-ans{6-9}
-    py{314}-m_ans-ans{10-12}
+    py{314}-m_ans-ans{10-13}
     py{27,36,314}-m_mtg
     report,