diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index ff06c0c5..99294c1f 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -32,6 +32,15 @@ Stable names for PluginLoader instances across Ansible versions. from __future__ import absolute_import +__all__ = [ + 'action_loader', + 'connection_loader', + 'module_loader', + 'module_utils_loader', + 'shell_loader', + 'strategy_loader', +] + try: from ansible.plugins.loader import action_loader from ansible.plugins.loader import connection_loader diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 93b72f3f..503e9bb7 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -32,7 +32,6 @@ import logging import multiprocessing import os import resource -import signal import socket import sys diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ce5d335e..5cf171b6 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -37,7 +37,6 @@ how to build arguments for it, preseed related data, etc. """ import atexit -import codecs import imp import os import re diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 4e8dcd27..c3bbad59 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -82,8 +82,7 @@ Installation
- Releases occur frequently and often include important fixes. Subscribe - to the mitogen-announce list to stay updated. + Get notified of new releases and important fixes.

diff --git a/docs/changelog.rst b/docs/changelog.rst index 63266113..37cab2e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -72,15 +72,21 @@ Mitogen for Ansible detected, to work around a broken heuristic in popular SELinux policies that prevents inheriting ``AF_UNIX`` sockets across privilege domains. -* `#467 `_: an incompatibility +* `#467 `_: an incompatibility running Mitogen under Molecule was resolved. -* `#547 `_: fix a serious deadlock - possible during initialization of any task executed by forking, such as - ``async`` tasks, tasks using custom :mod:`ansible.module_utils`, +* `#547 `_, + `#598 `_: fix a serious deadlock + possible while initializing the service pool of any child, such as during + connection, ``async`` tasks, tasks using custom :mod:`module_utils`, ``mitogen_task_isolation: fork`` modules, and those present on an internal blacklist of misbehaving modules. + This deadlock is relatively easy hit, has been present since 0.2.0, and is + likely to have impacted many users. For new connections it could manifest as + a *Connection timed out* error, for forked tasks it could manifest as a + timeout or an apparent hang. + * `#549 `_: the open file descriptor limit for the Ansible process is increased to the available hard limit. It is common for distributions to ship with a much higher hard limit than their @@ -114,7 +120,8 @@ Mitogen for Ansible is more accurately inferred for `meta: reset_connection`, the `synchronize` module, and for any action plug-ins that establish additional connections. -* `#605 `_: fix a deadlock managing a +* `#598 `_, + `#605 `_: fix a deadlock managing a shared counter used for load balancing. * `#615 `_: streaming file transfer @@ -224,6 +231,7 @@ bug reports, testing, features and fixes in this release contributed by `Stefane Fermigier `_, `Szabó Dániel Ernő `_, `Ulrich Schreiner `_, +`yen `_, `Yuki Nishida `_, `@alexhexabeam `_, `@DavidVentura `_, diff --git a/docs/howitworks.rst b/docs/howitworks.rst index e5fdee2f..20c4f948 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -673,8 +673,12 @@ code occurring after the first conditional that looks like a standard if __name__ == '__main__': run_some_code() -This is a hack, but it's the least annoying hack I've found for the problem -yet. +To further avoid accidental execution, Mitogen will refuse to serve +:mod:`__main__` to children if no execution guard is found, as it is common +that no guard is present during early script prototyping. + +These are hacks, but they are the safest and least annoying found to solve the +problem. Avoiding Negative Imports diff --git a/docs/requirements.txt b/docs/requirements.txt index 85f30a2e..3c4674fd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -Sphinx==2.1.2 -sphinxcontrib-programoutput==0.14 -alabaster==0.7.10 +Sphinx==2.1.2; python_version > '3.0' +sphinxcontrib-programoutput==0.14; python_version > '3.0' +alabaster==0.7.10; python_version > '3.0' diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index c1b17032..55b272d9 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -245,7 +245,7 @@ def main(router): if sys.platform == 'darwin': kwargs['volname'] = '%s (Mitogen)' % (sys.argv[1],) - f = fuse.FUSE( + fuse.FUSE( operations=Operations(sys.argv[1]), mountpoint=sys.argv[2], foreground=True, diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 00000000..09349570 --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,416 @@ +# ########################################################################################## +# # Customize file classifications. # +# # Results from files under any classifier will be excluded from LGTM # +# # statistics. # +# ########################################################################################## +# +# ########################################################################################## +# # Use the `path_classifiers` block to define changes to the default classification of # +# # files. # +# ########################################################################################## +# +path_classifiers: +# docs: +# # Identify the top-level file called `generate_javadoc.py` as documentation-related. +# - generate_javadoc.py +# test: +# # Override LGTM’s default classification of test files by excluding all files. +# - exclude: / +# # Classify all files in the top-level directories test/ and testsuites/ as test code. +# - test +# - testsuites +# # Classify all files with suffix `.test` as test code. +# # Note: use only forward slash / as a path separator. +# # Use ** to indicate an arbitrary parent path. +# # Use * to indicate any sequence of characters excluding /. +# # Always enclose the expression in double quotes if it includes *. +# - "**/*.test" +# # Refine the classifications above by excluding files in test/util/. +# - exclude: test/util +# # The default behavior is to tag all files created during the +# # build as `generated`. Results are hidden for generated code. You can tag +# # further files as being generated by adding them to the `generated` section. +# generated: +# # Exclude all `*.c` files under the `ui/` directory from classification as +# # generated code. +# # By default, all files not checked into the repository are considered to be +# # 'generated'. +# - exclude: ui/**.c +# # The default behavior is to tag library code as `library`. Results are hidden +# # for library code. You can tag further files as being library code by adding them +# # to the `library` section. + library: + - "mitogen/compat" +# # The default behavior is to tag template files as `template`. Results are hidden +# # for template files. You can tag further files as being template files by adding +# # them to the `template` section. +# template: +# # Define your own category, for example: 'some_custom_category'. +# some_custom_category: +# # Classify all files in the top-level directory tools/ (or the top-level file +# # called tools). +# - tools +# +# ######################################################################################### +# # Use the `queries` block to change the default display of query results. # +# ######################################################################################### +# +queries: + # Mitogen 2.4 compatibility trips this query everywhere, so just disable it + - exclude: py/unreachable-statement + - exclude: py/should-use-with + # mitogen.core.b() trips this query everywhere, so just disable it + - exclude: py/import-and-import-from +# # Specifically hide the results of two queries. +# - exclude: cpp/use-of-goto +# - exclude: java/equals-on-unrelated-types +# # Hide the results of all queries. +# - exclude: "*" +# # Refine by including the `java/command-line-injection` query. +# - include: java/command-line-injection +# # Include all queries tagged ‘security’ and ‘correctness', and with a severity of +# # ‘error’. +# - include: +# tags: +# - "security" +# - "correctness" +# severity: "error" +# +# ######################################################################################### +# # Define changes to the default code extraction process. # +# # Each block configures the extraction of a single language, and modifies actions in a # +# # named step. Every named step includes automatic default actions, # +# # except for the ‘prepare’ step. The steps are performed in the following sequence: # +# # prepare # +# # after_prepare # +# # configure (C/C++ only) # +# # python_setup (Python only) # +# # before_index # +# # index # +########################################################################################### +# +# ######################################################################################### +# # Environment variables available to the steps: # +# ######################################################################################### +# +# # LGTM_SRC +# # The root of the source tree. +# # LGTM_WORKSPACE +# # An existing (initially empty) folder outside the source tree. +# # Used for temporary download and setup commands. +# +# ######################################################################################### +# # Use the extraction block to define changes to the default code extraction process # +# # for one or more languages. The settings for each language are defined in a child # +# # block, with one or more steps. # +# ######################################################################################### +# +# extraction: +# # Define settings for C/C++ analysis +# ##################################### +# cpp: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # The `packages` section is valid for LGTM.com only. It names Ubuntu packages to +# # be installed. +# packages: +# - libxml2-dev +# - libxslt1-dev +# # Add an `after-prepare` step if you need to run commands after the prepare step. +# # Each command should be listed on a separate line. +# # This step is useful for C/C++ analysis where you want to prepare the environment +# # for the `configure` step without changing the default behavior for that step. +# after_prepare: +# - mkdir -p $LGTM_WORKSPACE/latest-gcc-symlinks +# - ln -s /usr/bin/g++-6 $LGTM_WORKSPACE/latest-gcc-symlinks/g++ +# - ln -s /usr/bin/gcc-6 $LGTM_WORKSPACE/latest-gcc-symlinks/gcc +# - export PATH=$LGTM_WORKSPACE/latest-gcc-symlinks:$PATH +# - export GNU_MAKE=make +# - export GIT=true +# # The `configure` step generates build configuration files which the `index` step +# # then uses to build the codebase. +# configure: +# command: +# - ./prepare_deps +# # Optional step. You should add a `before_index` step if you need to run commands +# # before the `index` step. +# before_index: +# - export BOOST_DIR=$LGTM_SRC/boost +# - export GTEST_DIR=$LGTM_SRC/googletest +# - export HUNSPELL_DIR=$LGTM_SRC/hunspell +# - export CRYPTOPP_DIR=$LGTM_SRC/cryptopp +# # The `index` step builds the code and extracts information during the build +# # process. +# index: +# # Override the autobuild process by specifying a list of custom build commands +# # to use instead. +# build_command: +# - $GNU_MAKE -j2 -s +# # Specify that all project or solution files should be used for extraction. +# # Default: false. +# all_solutions: true +# # Specify a list of one or more project or solution files for extraction. +# # Default: LGTM chooses the file closest to the root of the repository (this may +# # fail if there are multiple candidates). +# solution: +# - myProject.sln +# # Specify MSBuild settings +# msbuild: +# # Specify a list of additional arguments to MSBuild. Default: empty. +# arguments: /p:Platform=x64 /p:Configuration=Release +# # Specify the MSBuild configuration to use, for example, debug or release. +# # Default: read from the solution file or files. +# configuration: +# # Specify the platform to target, for example: x86, x64, or Any CPU. +# # Default: read from the solution file or files. +# platform: +# # Specify the MSBuild target. Default: rebuild. +# target: +# # Specify whether or not to perform a NuGet restore for extraction. Default: true. +# nuget_restore: false +# # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom +# # build commands (build_command). For example: +# # 10 for Visual Studio 2010 +# # 12 for Visual Studio 2012 +# # 14 for Visual Studio 2015 +# # 15 for Visual Studio 2017 +# # Default: read from project files. +# vstools_version: 10 +# +# # Define settings for C# analysis +# ################################## +# csharp: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # Add an `after-prepare` step if you need to run commands after the `prepare` step. +# # Each command should be listed on a separate line. +# after_prepare: +# - export PATH=$LGTM_WORKSPACE/tools:$PATH +# # The `index` step builds the code and extracts information during the build +# # process. +# index: +# # Specify that all project or solution files should be used for extraction. +# # Default: false. +# all_solutions: true +# # Specify a list of one or more project or solution files for extraction. +# # Default: LGTM chooses the file closest to the root of the repository (this may +# # fail if there are multiple candidates). +# solution: +# - myProject.sln +# # Override the autobuild process by specifying a list of custom build commands +# # to use instead. +# build_command: +# # By default, LGTM analyzes the code by building it. You can override this, +# # and tell LGTM not to build the code. Beware that this can lead +# # to less accurate results. +# buildless: true +# # Specify .NET Core settings. +# dotnet: +# # Specify additional arguments to `dotnet build`. +# # Default: empty. +# arguments: +# # Specify the version of .NET Core SDK to use. +# # Default: The version installed on the build machine. +# version: 2.1 +# # Specify MSBuild settings. +# msbuild: +# # Specify a list of additional arguments to MSBuild. Default: empty. +# arguments: /P:WarningLevel=2 +# # Specify the MSBuild configuration to use, for example, debug or release. +# # Default: read from the solution file or files. +# configuration: release +# # Specify the platform to target, for example: x86, x64, or Any CPU. +# # Default: read from the solution file or files. +# platform: x86 +# # Specify the MSBuild target. Default: rebuild. +# target: notest +# # Specify whether or not to perform a NuGet restore for extraction. Default: true. +# nuget_restore: false +# # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom +# # build commands (build_command). For example: +# # 10 for Visual Studio 2010 +# # 12 for Visual Studio 2012 +# # 14 for Visual Studio 2015 +# # 15 for Visual Studio 2017 +# # Default: read from project files +# vstools_version: 10 +# # Specify additional options for the extractor, +# # for example --fast to perform a faster extraction that produces a smaller +# # database. +# extractor: +# +# # Define settings for COBOL analysis +# ##################################### +# cobol: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # Add an `after-prepare` step if you need to run commands after the prepare step. +# # Each command should be listed on a separate line. +# after_prepare: +# - export PATH=$LGTM_WORKSPACE/tools:$PATH +# index: +# # Specify the source file format: fixed, free, or variable. +# # Default: fixed. +# format: "variable" +# # Specify the number of spaces in a tab. Default: 4. +# tab_length: 2 +# # Specify whether preprocessing is applied before extraction or not. +# # Default: true (preprocessing on). +# preprocessing: false +# # Specify file extensions that LGTM will identify as belonging to source files. +# # Default: files with a .CBL, .cbl, .COB or .cob extension are considered +# # source files. +# source_globs: +# - "**.cbl" +# - "**.CBL" +# - "**.cob" +# - "**.cobol" +# - "**.COB" +# # Specify file extensions that LGTM will identify as belonging to library files. +# # Default: files with a .CPY, .cpy, .COPY or .copy extension. +# library_globs: +# - "**.CPY" +# - "**.COPY" +# +# # Define settings for Java analysis +# #################################### +# java: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # Add an `after-prepare` step if you need to run commands after the prepare step. +# # Each command should be listed on a separate line. +# after_prepare: +# - export PATH=$LGTM_WORKSPACE/tools:$PATH +# # The `index` step extracts information from the files in the codebase. +# index: +# # Specify Gradle settings. +# gradle: +# # Specify the required Gradle version. +# # Default: determined automatically. +# version: 4.4 +# # Override the autobuild process by specifying a list of custom build commands +# # to use instead. +# build_command: ./compile-all.sh +# # Specify the Java version required to build the project. +# java_version: 11 +# # Specify Maven settings. +# maven: +# # Specify the path (absolute or relative) of a Maven settings file to use. +# # Default: Maven uses a settings file in the default location, if it exists. +# settings_file: /opt/share/settings.xml +# # Specify the path of a Maven toolchains file. +# # Default: Maven uses a toolchains file in the default location, if it exists. +# toolchains_file: /opt/share/toolchains.xml +# # Specify the required Maven version. +# # Default: the Maven version is determined automatically, where feasible. +# version: 3.5.2 +# # Specify how XML files should be extracted: +# # ALL = extract all XML files. +# # DEFAULT = only extract XML files named `AndroidManifest.xml`. +# # DISABLED = do not extract any XML files. +# xml_mode: ALL +# +# # Define settings for JavaScript analysis +# ########################################## +# javascript: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # Add an `after-prepare` step if you need to run commands after the prepare step. +# # Each command should be listed on a separate line. +# after_prepare: +# - export PATH=$LGTM_WORKSPACE/tools:$PATH +# # The `index` step extracts information from the files in the codebase. +# index: +# # Specify a list of files and folders to extract. +# # Default: The project root directory. +# include: +# - src/js +# # Specify a list of files and folders to exclude from extraction. +# exclude: +# - thirdparty/lib +# - node_modules +# # You can add additional file types for LGTM to extract, by mapping file +# # extensions (including the leading dot) to file types. The usual +# # include/exclude patterns apply, so, for example, `.jsm` files under +# # `thirdparty/lib` and `node_modules` will not be extracted. +# filetypes: +# ".jsm": "js" +# ".tmpl": "html" +# # Specify a list of glob patterns to include/exclude files from extraction; this +# # is applied on top of the include/exclude paths from above; patterns are +# # processed in the same way as for path classifiers above. +# # Default: include all files with known extensions (such as .js, .ts and .html), +# # but exclude files ending in `-min.js` or `.min.js`. +# filters: +# # exclude any node_modules folders anywhere. +# - exclude: "**/node_modules/*" +# # exclude any *.ts files anywhere. +# - exclude: "**/*.ts" +# # but include *.ts files under src/js/typescript. +# - include: "src/js/typescript/**/*.ts" +# # Specify the type of project: module or script. +# # Default: type is automatically detected. +# source_type: module +# # Specify how TypeScript files should be extracted: +# # none = exclude all TypeScript files. +# # basic = extract syntactic information from TypeScript files. +# # full = extract syntactic and type information from TypeScript files. +# # Default: full. +# typescript: basic +# # By default, LGTM doesn't extract any XML files. You can override this by +# # using the `xml_mode` property and setting it to `all`. +# xml_mode: all +# +# # Define settings for Python analysis +# ###################################### +# python: +# # The `prepare` step exists for customization on LGTM.com only. +# prepare: +# # The `packages` section is valid for LGTM.com only. It names packages to +# # be installed. +# packages: libpng-dev +# # This step is useful for Python analysis where you want to prepare the +# # environment for the `python_setup` step without changing the default behavior +# # for that step. +# after_prepare: +# - export PATH=$LGTM_WORKSPACE/tools:$PATH +# # This sets up the Python interpreter and virtual environment, ready for the +# # `index` step to extract the codebase. +# python_setup: +# # Specify packages that should NOT be installed despite being mentioned in the +# # requirements.txt file. +# # Default: no package marked for exclusion. +# exclude_requirements: +# - pywin32 +# # Specify a list of pip packages to install. +# # If any of these packages cannot be installed, the extraction will fail. +# requirements: +# - Pillow +# # Specify a list of requirements text files to use to set up the environment, +# # or false for none. Default: any requirements.txt, test-requirements.txt, +# # and similarly named files identified in the codebase are used. +# requirements_files: +# - required-packages.txt +# # Specify a setup.py file to use to set up the environment, or false for none. +# # Default: any setup.py files identified in the codebase are used in preference +# # to any requirements text files. +# setup_py: new-setup.py +# # Override the version of the Python interpreter used for setup and extraction +# # Default: Python 3 if no version is explicitly specified, and if there are no +# # commits to the repository before January 1, 2017. Otherwise Python 2. +# version: 3 +# # Optional step. You should add a `before_index` step if you need to run commands +# # before the `index` step. +# before_index: +# - antlr4 -Dlanguage=Python3 Grammar.g4 +# # The `index` step extracts information from the files in the codebase. +# index: +# # Specify a list of files and folders to exclude from extraction. +# # Default: Git submodules and Subversion externals. +# exclude: +# - legacy-implementation +# - thirdparty/libs +# filters: +# - exclude: "**/documentation/examples/snippets/*.py" +# - include: "**/documentation/examples/test_application/*" +# include: diff --git a/mitogen/master.py b/mitogen/master.py index b4eb6643..11ef2b00 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -47,7 +47,6 @@ import re import string import sys import threading -import time import types import zlib diff --git a/mitogen/parent.py b/mitogen/parent.py index 983df829..bc4bfc0d 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -41,7 +41,6 @@ import getpass import heapq import inspect import logging -import logging import os import re import signal @@ -52,7 +51,6 @@ import sys import termios import textwrap import threading -import time import zlib # Absolute imports for <2.5. diff --git a/mitogen/service.py b/mitogen/service.py index 8882b30b..168f0140 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -37,7 +37,6 @@ import pwd import stat import sys import threading -import time import mitogen.core import mitogen.select diff --git a/tests/ansible/gcloud/mitogen-load-testing.tf b/tests/ansible/gcloud/mitogen-load-testing.tf index 076722ce..9bab03d4 100644 --- a/tests/ansible/gcloud/mitogen-load-testing.tf +++ b/tests/ansible/gcloud/mitogen-load-testing.tf @@ -2,6 +2,10 @@ variable "node-count" { default = 0 } +variable "preemptible" { + default = true +} + variable "big" { default = false } @@ -93,7 +97,7 @@ resource "google_compute_instance_template" "node" { machine_type = "custom-1-1024" scheduling { - preemptible = true + preemptible = "${var.preemptible}" automatic_restart = false } diff --git a/tests/ansible/integration/async/runner_one_job.yml b/tests/ansible/integration/async/runner_one_job.yml index a576d4ce..871d672f 100644 --- a/tests/ansible/integration/async/runner_one_job.yml +++ b/tests/ansible/integration/async/runner_one_job.yml @@ -9,9 +9,10 @@ # Verify output of a single async job. - name: start 2 second op + # Sleep after writing; see https://github.com/ansible/ansible/issues/51393 shell: | + echo alldone; sleep 1; - echo alldone async: 1000 poll: 0 register: job1 @@ -40,9 +41,9 @@ # ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44 - | (ansible_version.full >= '2.8' and - result1.cmd == "sleep 1;\necho alldone\n") or + result1.cmd == "echo alldone;\nsleep 1;\n") or (ansible_version.full < '2.8' and - result1.cmd == "sleep 1;\n echo alldone") + result1.cmd == "echo alldone;\n sleep 1;") - result1.delta|length == 14 - result1.start|length == 26 - result1.finished == 1 diff --git a/tests/router_test.py b/tests/router_test.py index a9ad5ae1..1cde016d 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -434,12 +434,21 @@ class ShutdownTest(testlib.RouterMixin, testlib.TestCase): l1 = self.router.local() pid = l1.call(os.getpid) - conn = self.router.stream_by_id(l1.context_id).conn + strm = self.router.stream_by_id(l1.context_id) exitted = mitogen.core.Latch() - mitogen.core.listen(conn.proc, 'exit', exitted.put) + + # It is possible for Process 'exit' signal to fire immediately during + # processing of Stream 'disconnect' signal, so we must wait for both, + # otherwise ChannelError below will return 'respondent context has + # disconnected' rather than 'no route', because RouteMonitor hasn't run + # yet and the Receiver caught Context 'disconnect' signal instead of a + # dead message. + mitogen.core.listen(strm.conn.proc, 'exit', exitted.put) + mitogen.core.listen(strm, 'disconnect', exitted.put) l1.shutdown(wait=False) exitted.get() + exitted.get() e = self.assertRaises(OSError, lambda: os.waitpid(pid, 0))