diff --git a/.github/ISSUE_TEMPLATE/bug-0.2.md b/.github/ISSUE_TEMPLATE/bug-0.2.md deleted file mode 100644 index 1fd9672f..00000000 --- a/.github/ISSUE_TEMPLATE/bug-0.2.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Mitogen 0.2.x bug report -about: Report a bug in Mitogen 0.2.x (for Ansible 2.5, 2.6, 2.7, 2.8, or 2.9) -title: '' -labels: affects-0.2, bug -assignees: '' - ---- - -Please drag-drop large logs as text file attachments. - -Feel free to write an issue in your preferred format, however if in doubt, use -the following checklist as a guide for what to include. - -* Which version of Ansible are you running? -* Is your version of Ansible patched in any way? -* Are you running with any custom modules, or `module_utils` loaded? - -* Have you tried the latest master version from Git? -* Do you have some idea of what the underlying problem may be? - https://mitogen.networkgenomics.com/ansible_detailed.html#common-problems has - instructions to help figure out the likely cause and how to gather relevant - logs. -* Mention your host and target OS and versions -* Mention your host and target Python versions -* If reporting a performance issue, mention the number of targets and a rough - description of your workload (lots of copies, lots of tiny file edits, etc.) -* If reporting a crash or hang in Ansible, please rerun with -vvv and include - 200 lines of output around the point of the error, along with a full copy of - any traceback or error text in the log. Beware "-vvv" may include secret - data! Edit as necessary before posting. -* If reporting any kind of problem with Ansible, please include the Ansible - version along with output of "ansible-config dump --only-changed". diff --git a/.github/ISSUE_TEMPLATE/bug-0.3.md b/.github/ISSUE_TEMPLATE/bug-0.3.md deleted file mode 100644 index 0280198b..00000000 --- a/.github/ISSUE_TEMPLATE/bug-0.3.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Mitogen 0.3.x bug report -about: Report a bug in Mitogen 0.3.x (for Ansible 2.10.x) -title: '' -labels: affects-0.3, bug -assignees: '' - ---- - -Please drag-drop large logs as text file attachments. - -Feel free to write an issue in your preferred format, however if in doubt, use -the following checklist as a guide for what to include. - -* Which version of Ansible are you running? -* Is your version of Ansible patched in any way? -* Are you running with any custom modules, or `module_utils` loaded? - -* Have you tried the latest master version from Git? -* Do you have some idea of what the underlying problem may be? - https://mitogen.networkgenomics.com/ansible_detailed.html#common-problems has - instructions to help figure out the likely cause and how to gather relevant - logs. -* Mention your host and target OS and versions -* Mention your host and target Python versions -* If reporting a performance issue, mention the number of targets and a rough - description of your workload (lots of copies, lots of tiny file edits, etc.) -* If reporting a crash or hang in Ansible, please rerun with -vvv and include - 200 lines of output around the point of the error, along with a full copy of - any traceback or error text in the log. Beware "-vvv" may include secret - data! Edit as necessary before posting. -* If reporting any kind of problem with Ansible, please include the Ansible - version along with output of "ansible-config dump --only-changed". diff --git a/.github/ISSUE_TEMPLATE/bug-0.3.yml b/.github/ISSUE_TEMPLATE/bug-0.3.yml new file mode 100644 index 00000000..1898b7cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-0.3.yml @@ -0,0 +1,62 @@ +name: Bug report +description: Report a bug in Mitogen 0.3.x (for Ansible 2.10 and above) +labels: + - affects-0.3 +type: bug + +body: + - type: textarea + attributes: + label: Description + description: > + When does the problem occur? + What happens after? + How is this different? + Did it previously behave as expected? + placeholder: | + When I do X, Y happens, but I was expecting Z because ... + Before version 1.2.3 it worked as expected. + validations: + required: true + + - type: input + attributes: + label: Mitogen version + placeholder: 0.3.31, 0.3.3-9+deb12u1 + validations: + required: true + + - type: input + attributes: + label: Ansible version (if applicable) + placeholder: 2.18.11 + + - type: textarea + attributes: + label: OS and environment + description: > + What operating system version(s), Python version(s), etc. are you using? + placeholder: | + Controller (master): Debian 13, Python 3.14 + Targets (slaves): Ubuntu 20.04/Python 2.7, RHEL 10, ... + + - type: textarea + attributes: + label: Steps to reproduce + description: > + Instructions, code, or playbook(s) recreate the beahviour + value: | + Steps: + 1. Set config `foo = 42` in somefile.cfg + 2. Run the following Python or Playbook with `cmd --option bar ...` + + ``` + Code or playbook here + ``` + + - type: textarea + attributes: + label: Anything else + description: > + Include any other details you think might be relevant or helpful. + Examples might include logs, unusual settings, environment variables, ... diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index a8e91c02..00000000 --- a/.lgtm.yml +++ /dev/null @@ -1,10 +0,0 @@ -path_classifiers: - library: - - "mitogen/compat" - - "ansible_mitogen/compat" -queries: - # Mitogen 2.4 compatibility trips this query everywhere, so just disable it - - exclude: py/unreachable-statement - - exclude: py/should-use-with - # mitogen.core.b() trips this query everywhere, so just disable it - - exclude: py/import-and-import-from diff --git a/README.md b/README.md index c3cd9a87..26e033e2 100644 --- a/README.md +++ b/README.md @@ -7,5 +7,3 @@ Please see the documentation. ![](https://i.imgur.com/eBM6LhJ.gif) - -[![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 9597e3ee..f753b6b5 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,18 +49,6 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 19) - -NEW_VERSION_MSG = ( - "Your Ansible version (%s) is too recent. The most recent version\n" - "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" - "release notes to see if a new version is available, otherwise\n" - "subscribe to the corresponding GitHub issue to be notified when\n" - "support becomes available.\n" - "\n" - " https://mitogen.rtfd.io/en/latest/changelog.html\n" - " https://github.com/mitogen-hq/mitogen/issues/\n" -) OLD_VERSION_MSG = ( "Your version of Ansible (%s) is too old. The oldest version supported by " "Mitogen for Ansible is %s." @@ -78,11 +66,6 @@ def assert_supported_release(): OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) ) - if v[:2] > ANSIBLE_VERSION_MAX: - raise ansible.errors.AnsibleError( - NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX) - ) - # this is the first file our strategy plugins import, so we need to check this here # in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user diff --git a/docs/changelog.rst b/docs/changelog.rst index d39f471a..ad540c93 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,15 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.32 (2025-11-21) +-------------------- + +* :gh:issue:`1243` :mod:`mitogen`: Pass first stage, context name, & preamble + size as seperate **argv** arguments +* :gh:issue:`1218` :mod:`ansible_mitogen`: Remove maximum Ansible version check +* :gh:issue:`1260` CI: Remove integration of retired lgtm.com + + v0.3.31 (2025-11-05) -------------------- diff --git a/docs/howitworks.rst b/docs/howitworks.rst index d7606b11..ae1910f8 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -27,14 +27,13 @@ Python Command Line ################### The Python command line sent to the host is a :mod:`zlib`-compressed [#f2]_ and -base64-encoded copy of the :py:meth:`mitogen.master.Stream._first_stage` -function, which has been carefully optimized to reduce its size. Prior to -compression and encoding, ``CONTEXT_NAME`` is replaced with the desired context -name in the function's source code. +base64-encoded copy of :py:meth:`mitogen.parent.Connection._first_stage`, +which is carefully written to maximize it compatibility and minimize its size. +A simplified illustration of the bootstrap command is .. code:: - python -c 'exec "xxx".decode("base64").decode("zlib")' + python -c 'exec(sys.argv[1].decode("base64").decode("zlib"))' ... The command-line arranges for the Python interpreter to decode the base64'd component, decompress it and execute it as Python code. Base64 is used since @@ -71,8 +70,8 @@ of the large base64-encoded first stage parameter, and to replace **argv[0]** with something descriptive. After configuring its ``stdin`` to point to the read end of the pipe, the -parent half of the fork re-executes Python, with **argv[0]** taken from the -``CONTEXT_NAME`` variable earlier substituted into its source code. As no +fork parent re-executes Python with **argv[0]** composed of the Python +interpreter path and a remote name supplied by the Mitogen parent. As no arguments are provided to this new execution of Python, and since ``stdin`` is connected to a pipe (whose write end is connected to the first stage), the Python interpreter begins reading source code to execute from the pipe diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 2fd7ce9d..f4ca00a2 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 31) +__version__ = (0, 3, 32) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/parent.py b/mitogen/parent.py index 1a23df18..6e30b1c6 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1396,10 +1396,6 @@ class Connection(object): # with a custom argv. # * Optimized for minimum byte count after minification & compression. # The script preamble_size.py measures this. - # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with - # their respective values. - # * CONTEXT_NAME must be prefixed with the name of the Python binary in - # order to allow virtualenvs to detect their install prefix. # # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible # module test_echo_module, used by the integration tests. @@ -1435,20 +1431,21 @@ class Connection(object): os.close(r) os.close(W) os.close(w) - if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7' - if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1' + if os.uname()[0]+os.uname()[2][:2]+sys.executable=='Darwin19/usr/bin/python':sys.executable+='2.7' + if os.uname()[0]+os.uname()[2][:2]+sys.version[:3]=='Darwin202.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1' + if os.uname()[0]+os.uname()[2][:2]+sys.version[:3]=='Darwin212.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1' os.environ['ARGV0']=sys.executable - os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') + os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) C=''.encode() - while PREAMBLE_COMPRESSED_LEN-len(C)and select.select([0],[],[]):C+=os.read(0,PREAMBLE_COMPRESSED_LEN-len(C)) + while int(sys.argv[3])-len(C)and select.select([0],[],[]):C+=os.read(0,int(sys.argv[3])-len(C)) C=zlib.decompress(C) - fp=os.fdopen(W,'wb',0) - fp.write(C) - fp.close() - fp=os.fdopen(w,'wb',0) - fp.write(C) - fp.close() + f=os.fdopen(W,'wb',0) + f.write(C) + f.close() + f=os.fdopen(w,'wb',0) + f.write(C) + f.close() os.write(1,'MITO001\n'.encode()) os.close(2) @@ -1469,11 +1466,10 @@ class Connection(object): source = inspect.getsource(self._first_stage) source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:])) source = source.replace(' ', ' ') - source = source.replace('CONTEXT_NAME', self.options.remote_name) - preamble_compressed = self.get_preamble() - source = source.replace('PREAMBLE_COMPRESSED_LEN', - str(len(preamble_compressed))) - compressed = zlib.compress(source.encode(), 9) + compressor = zlib.compressobj( + zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, + ) + compressed = compressor.compress(source.encode()) + compressor.flush() encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) # Just enough to decode, decompress, and exec the first stage. @@ -1484,7 +1480,10 @@ class Connection(object): '-c', 'import sys;sys.path=[p for p in sys.path if p];' 'import binascii,os,select,zlib;' - 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), + 'exec(zlib.decompress(binascii.a2b_base64(sys.argv[1]),-15))', + encoded.decode(), + self.options.remote_name, + str(len(self.get_preamble())), ] def get_econtext_config(self): diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index e06f453f..2576ec14 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -26,7 +26,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # preamble from stdin, then execute it. # This test attaches /dev/zero to stdin to create a specific failure - # 1. Fork child reads PREAMBLE_COMPRESSED_LEN bytes of junk (all `\0`) + # 1. Fork child reads bytes of NUL (`b'\0'`) # 2. Fork child crashes (trying to decompress the junk data) # 3. Fork child's file descriptors (write pipes) are closed by the OS # 4. Fork parent does `dup(, )` and `exec()`