From 9dce5fe05617e408b4c727a6f2619019818373ac Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 10 Jan 2021 20:20:45 +0000 Subject: [PATCH 1/4] tests: Correct DISTRO vs DISTROS usage in CI configurations The environment variable DISTRO is used to set the target Docker image used when running the Mitogen unit tests. DISTROS is used for the Ansible integration tests. VER sets the version of Ansible that is installed on the controller. When MODE=Mitogen - there is no need to set VER, because nothing that installs Ansible - it does not make sense to set DISTROS. --- .ci/azure-pipelines.yml | 4 ---- .ci/ci_lib.py | 3 +++ .travis.yml | 6 +++--- tests/doas_test.py | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index d436f175..d181e5e4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -17,7 +17,6 @@ jobs: Mito27_27: python.version: '2.7' MODE: mitogen - VER: 2.10.0 # TODO: test python3, python3 tests are broken Ans210_27: python.version: '2.7' @@ -46,7 +45,6 @@ jobs: python.version: '2.7' MODE: mitogen DISTRO: debian - VER: 2.10.0 #MitoPy27CentOS6_26: #python.version: '2.7' @@ -57,13 +55,11 @@ jobs: python.version: '3.6' MODE: mitogen DISTRO: centos6 - VER: 2.10.0 Mito37Debian_27: python.version: '3.7' MODE: mitogen DISTRO: debian - VER: 2.10.0 #Py26CentOS7: #python.version: '2.7' diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f735f6a1..82ebc120 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -149,9 +149,12 @@ class Fold(object): os.environ.setdefault('ANSIBLE_STRATEGY', os.environ.get('STRATEGY', 'mitogen_linear')) +# Ignoreed when MODE=mitogen ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +# Used only when MODE=mitogen DISTRO = os.environ.get('DISTRO', 'debian') +# Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) BASE_PORT = 2200 diff --git a/.travis.yml b/.travis.yml index aafb4413..d4f3ef7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,11 +72,11 @@ matrix: #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 - python: "3.6" - env: MODE=mitogen DISTROS=centos7 VER=2.10.0 + env: MODE=mitogen DISTRO=centos7 # 2.6 -> 2.7 # - python: "2.6" - # env: MODE=mitogen DISTROS=centos7 VER=2.10.0 + # env: MODE=mitogen DISTRO=centos7 # 2.6 -> 3.5 # - python: "2.6" - # env: MODE=mitogen DISTROS=debian-py3 VER=2.10.0 + # env: MODE=mitogen DISTRO=debian-py3 # 3.6 -> 2.6 -- moved to Azure diff --git a/tests/doas_test.py b/tests/doas_test.py index 16b0b9da..d1266e2e 100644 --- a/tests/doas_test.py +++ b/tests/doas_test.py @@ -28,7 +28,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals('1', context.call(os.getenv, 'THIS_IS_STUB_DOAS')) -# TODO: https://github.com/dw/mitogen/issues/694 they are flaky on python 2.6 MODE=mitogen DISTRO=centos7 +# TODO: https://github.com/dw/mitogen/issues/694 they are flaky on python 2.6 MODE=mitogen DISTROS=centos7 # class DoasTest(testlib.DockerMixin, testlib.TestCase): # # Only mitogen/debian-test has doas. # mitogen_test_distro = 'debian' From 234dde5fc1394d4ac6f855426e7794c97cc53f91 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Sun, 7 Feb 2021 13:09:17 -0800 Subject: [PATCH 2/4] check Ansible version before loaders are loaded --- ansible_mitogen/loaders.py | 67 +++++++++++++++++++++++++++++-------- ansible_mitogen/planner.py | 10 ------ ansible_mitogen/strategy.py | 48 +------------------------- docs/changelog.rst | 1 + 4 files changed, 55 insertions(+), 71 deletions(-) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 00a89b74..c00915d5 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -31,6 +31,7 @@ Stable names for PluginLoader instances across Ansible versions. """ from __future__ import absolute_import +import distutils.version __all__ = [ 'action_loader', @@ -41,21 +42,59 @@ __all__ = [ 'strategy_loader', ] -try: - from ansible.plugins.loader import action_loader - from ansible.plugins.loader import connection_loader - from ansible.plugins.loader import module_loader - from ansible.plugins.loader import module_utils_loader - from ansible.plugins.loader import shell_loader - from ansible.plugins.loader import strategy_loader -except ImportError: # Ansible <2.4 - from ansible.plugins import action_loader - from ansible.plugins import connection_loader - from ansible.plugins import module_loader - from ansible.plugins import module_utils_loader - from ansible.plugins import shell_loader - from ansible.plugins import strategy_loader +import ansible +ANSIBLE_VERSION_MIN = (2, 10) +ANSIBLE_VERSION_MAX = (2, 10) + +NEW_VERSION_MSG = ( + "Your Ansible version (%s) is too recent. The most recent version\n" + "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" + "release notes to see if a new version is available, otherwise\n" + "subscribe to the corresponding GitHub issue to be notified when\n" + "support becomes available.\n" + "\n" + " https://mitogen.rtfd.io/en/latest/changelog.html\n" + " https://github.com/mitogen-hq/mitogen/issues/\n" +) +OLD_VERSION_MSG = ( + "Your version of Ansible (%s) is too old. The oldest version supported by " + "Mitogen for Ansible is %s." +) + + +def assert_supported_release(): + """ + Throw AnsibleError with a descriptive message in case of being loaded into + an unsupported Ansible release. + """ + v = ansible.__version__ + if not isinstance(v, tuple): + v = tuple(distutils.version.LooseVersion(v).version) + + if v[:2] < ANSIBLE_VERSION_MIN: + raise ansible.errors.AnsibleError( + OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) + ) + + if v[:2] > ANSIBLE_VERSION_MAX: + raise ansible.errors.AnsibleError( + NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) + ) + + +# this is the first file our strategy plugins import, so we need to check this here +# in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user +# is trying to load an old Ansible version, we'll fail and error gracefully +assert_supported_release() + + +from ansible.plugins.loader import action_loader +from ansible.plugins.loader import connection_loader +from ansible.plugins.loader import module_loader +from ansible.plugins.loader import module_utils_loader +from ansible.plugins.loader import shell_loader +from ansible.plugins.loader import strategy_loader # These are original, unwrapped implementations action_loader__get = action_loader.get diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index faf8d197..c0913a3e 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -436,26 +436,16 @@ def py_modname_from_path(name, path): Fetch the logical name of a new-style module as it might appear in :data:`sys.modules` of the target's Python interpreter. - * For Ansible <2.7, this is an unpackaged module named like - "ansible_module_%s". - - * For Ansible <2.9, this is an unpackaged module named like - "ansible.modules.%s" - * Since Ansible 2.9, modules appearing within a package have the original package hierarchy approximated on the target, enabling relative imports to function correctly. For example, "ansible.modules.system.setup". """ - # 2.9+ if _get_ansible_module_fqn: try: return _get_ansible_module_fqn(path) except ValueError: pass - if ansible.__version__ < '2.7': - return 'ansible_module_' + name - return 'ansible.modules.' + name diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index ddc0a5b6..792cfada 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -27,7 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import -import distutils.version import os import signal import threading @@ -43,52 +42,8 @@ import ansible_mitogen.loaders import ansible_mitogen.mixins import ansible_mitogen.process -import ansible import ansible.executor.process.worker - -try: - # 2.8+ has a standardized "unset" object. - from ansible.utils.sentinel import Sentinel -except ImportError: - Sentinel = None - -ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 10) - -NEW_VERSION_MSG = ( - "Your Ansible version (%s) is too recent. The most recent version\n" - "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" - "release notes to see if a new version is available, otherwise\n" - "subscribe to the corresponding GitHub issue to be notified when\n" - "support becomes available.\n" - "\n" - " https://mitogen.rtfd.io/en/latest/changelog.html\n" - " https://github.com/dw/mitogen/issues/\n" -) -OLD_VERSION_MSG = ( - "Your version of Ansible (%s) is too old. The oldest version supported by " - "Mitogen for Ansible is %s." -) - - -def _assert_supported_release(): - """ - Throw AnsibleError with a descriptive message in case of being loaded into - an unsupported Ansible release. - """ - v = ansible.__version__ - if not isinstance(v, tuple): - v = tuple(distutils.version.LooseVersion(v).version) - - if v[:2] < ANSIBLE_VERSION_MIN: - raise ansible.errors.AnsibleError( - OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) - ) - - if v[:2] > ANSIBLE_VERSION_MAX: - raise ansible.errors.AnsibleError( - NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) - ) +from ansible.utils.sentinel import Sentinel def _patch_awx_callback(): @@ -351,7 +306,6 @@ class StrategyMixin(object): Wrap :meth:`run` to ensure requisite infrastructure and modifications are configured for the duration of the call. """ - _assert_supported_release() wrappers = AnsibleWrappers() self._worker_model = self._get_worker_model() ansible_mitogen.process.set_worker_model(self._worker_model) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99c30798..21d5d1a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ v0.3.0 (unreleased) This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. `See here for details `_. +* :gh:issue:`770` better check for supported Ansible version * :gh:issue:`731` ansible 2.10 support * :gh:issue:`652` support for ansible collections import hook From e67e4b83b4801c797d9e2080714e78b7b24765d0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 8 Feb 2021 22:15:18 +0000 Subject: [PATCH 3/4] tests: Fix throttling of Docker pulls fixes #809 --- .ci/ansible_install.py | 6 +++--- .ci/ci_lib.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index bb659f8a..08f04356 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -16,9 +16,9 @@ batches = [ ] ] -batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1'] +batches.append(ci_lib.throttle( + 'docker pull %s' % (ci_lib.image_for_distro(distro),) for distro in ci_lib.DISTROS -) +)) ci_lib.run_batches(batches) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index cf5406d7..524337a9 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -129,6 +129,20 @@ def combine(batch): )) +def throttle(batch, pause=1): + """ + Add pauses between commands in a batch + + >>> throttle(['echo foo', 'echo bar', 'echo baz']) + ['echo foo', 'sleep 1', 'echo bar', 'sleep 1', 'echo baz'] + """ + def _with_pause(batch, pause): + for cmd in batch: + yield cmd + yield 'sleep %i' % (pause,) + return list(_with_pause(batch, pause))[:-1] + + def run_batches(batches): """ Run shell commands grouped into batches, showing an execution trace. From 48e6da4f4404e6b3f5983d3ffe6718c21b7058c3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 7 Feb 2021 13:40:52 +0000 Subject: [PATCH 4/4] chore: Add issue templates for 0.2 vs 0.3 releases These should automatically tag bug reports, reducing triage effort. --- .github/ISSUE_TEMPLATE/bug-0.2.md | 33 +++++++++++++++++++ .../bug-0.3.md} | 8 +++++ 2 files changed, 41 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-0.2.md rename .github/{ISSUE_TEMPLATE.md => ISSUE_TEMPLATE/bug-0.3.md} (89%) diff --git a/.github/ISSUE_TEMPLATE/bug-0.2.md b/.github/ISSUE_TEMPLATE/bug-0.2.md new file mode 100644 index 00000000..1fd9672f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-0.2.md @@ -0,0 +1,33 @@ +--- +name: Mitogen 0.2.x bug report +about: Report a bug in Mitogen 0.2.x (for Ansible 2.5, 2.6, 2.7, 2.8, or 2.9) +title: '' +labels: affects-0.2, bug +assignees: '' + +--- + +Please drag-drop large logs as text file attachments. + +Feel free to write an issue in your preferred format, however if in doubt, use +the following checklist as a guide for what to include. + +* Which version of Ansible are you running? +* Is your version of Ansible patched in any way? +* Are you running with any custom modules, or `module_utils` loaded? + +* Have you tried the latest master version from Git? +* Do you have some idea of what the underlying problem may be? + https://mitogen.networkgenomics.com/ansible_detailed.html#common-problems has + instructions to help figure out the likely cause and how to gather relevant + logs. +* Mention your host and target OS and versions +* Mention your host and target Python versions +* If reporting a performance issue, mention the number of targets and a rough + description of your workload (lots of copies, lots of tiny file edits, etc.) +* If reporting a crash or hang in Ansible, please rerun with -vvv and include + 200 lines of output around the point of the error, along with a full copy of + any traceback or error text in the log. Beware "-vvv" may include secret + data! Edit as necessary before posting. +* If reporting any kind of problem with Ansible, please include the Ansible + version along with output of "ansible-config dump --only-changed". diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug-0.3.md similarity index 89% rename from .github/ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/bug-0.3.md index 46df53df..0280198b 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/bug-0.3.md @@ -1,3 +1,11 @@ +--- +name: Mitogen 0.3.x bug report +about: Report a bug in Mitogen 0.3.x (for Ansible 2.10.x) +title: '' +labels: affects-0.3, bug +assignees: '' + +--- Please drag-drop large logs as text file attachments.