From da3772cbffd99fd6ca09fff366709726dc513743 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 19:02:25 +0000 Subject: [PATCH 1/8] Increment version to 0.3.1.dev0 --- docs/changelog.rst | 11 ++++++++--- mitogen/__init__.py | 2 +- setup.py | 4 +++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e7045161..75917494 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,8 +17,13 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -v0.3.0 (unreleased) --------------------- + +v0.3.1.dev0 (unreleased) +------------------------ + + +v0.3.0 (2021-11-24) +------------------- This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. `See here for details `_. @@ -30,7 +35,7 @@ This release separates itself from the v0.2.X releases. Ansible's API changed to * :gh:issue:`847` Removed historic Continuous Integration reverse shell -v0.2.10 (unreleased) +v0.2.10 (2021-11-24) -------------------- * :gh:issue:`597` mitogen does not support Ansible 2.8 Python interpreter detection diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 9e709d7d..8df8bfe2 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 0, 'rc', 1) +__version__ = (0, 3, 1, 'dev0') #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/setup.py b/setup.py index bd105147..8d3c75df 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import ast import os from setuptools import find_packages, setup @@ -37,7 +38,8 @@ def grep_version(): for line in fp: if line.startswith('__version__'): _, _, s = line.partition('=') - return '%i.%i.%i%s%i' % eval(s) + parts = ast.literal_eval(s.strip()) + return '.'.join(str(part) for part in parts) def long_description(): From 3ba1625a99f489fdd2a16c0f50f425b468f37405 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 19:22:37 +0000 Subject: [PATCH 2/8] ci: Reduce verbosity of Ansible integration tests --- .ci/ansible_tests.py | 2 +- .ci/azure-pipelines.yml | 5 +++++ .ci/localhost_ansible_tests.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index b2aa3199..665e2c60 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -70,7 +70,7 @@ with ci_lib.Fold('job_setup'): with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') try: - run('./run_ansible_playbook.py %s -i "%s" -vvv %s', + run('./run_ansible_playbook.py %s -i "%s" %s', playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) except: pause_if_interactive() diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index c22dcf6c..bde7fa87 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -3,6 +3,11 @@ # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: # https://docs.microsoft.com/azure/devops/pipelines/languages/python +# User defined variables are also injected as environment variables +# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables +#variables: + #ANSIBLE_VERBOSITY: 3 + jobs: - job: Mac diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 6d7bef0d..13c77d96 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -46,11 +46,11 @@ with ci_lib.Fold('machine_prep'): if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) - run("ansible-playbook -c local -i localhost, _user_accounts.yml -vvv") + run("ansible-playbook -c local -i localhost, _user_accounts.yml") with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') - run('./run_ansible_playbook.py %s -l target %s -vvv', + run('./run_ansible_playbook.py %s -l target %s', playbook, ' '.join(sys.argv[1:])) From 39d02e9d5932cc0cdbb718ea19e89eec75d1947d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 19:28:00 +0000 Subject: [PATCH 3/8] ci: Don't install pycparser and idna seperately AFAICT no longer necessary. These install fine as part of requirements.txt. --- .ci/ansible_install.py | 4 ---- .ci/debops_common_install.py | 3 --- .ci/localhost_ansible_install.py | 5 ----- .ci/mitogen_install.py | 1 - tests/requirements.txt | 1 + 5 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 7b5d5657..63dbe563 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -4,10 +4,6 @@ import ci_lib batches = [ [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml - 'pip install "pycparser<2.19" "idna<2.7"', 'pip install ' '-r tests/requirements.txt ' '-r tests/ansible/requirements.txt', diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 62519994..50d556cc 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -7,9 +7,6 @@ ci_lib.DISTROS = ['debian'] ci_lib.run_batches([ [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - 'pip install "pycparser<2.19"', 'pip install -qqq "debops[ansible]==2.1.2" "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION), ], [ diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py index dba07053..e70bcf18 100755 --- a/.ci/localhost_ansible_install.py +++ b/.ci/localhost_ansible_install.py @@ -4,11 +4,6 @@ import ci_lib batches = [ [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml - # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. - 'pip install "pycparser<2.19" "idna<2.7" virtualenv', 'pip install ' '-r tests/requirements.txt ' '-r tests/ansible/requirements.txt', diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index d51c2f17..ab9bb2f2 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -4,7 +4,6 @@ import ci_lib batches = [ [ - 'pip install "pycparser<2.19" "idna<2.7"', 'pip install -r tests/requirements.txt', ] ] diff --git a/tests/requirements.txt b/tests/requirements.txt index 21ef8166..a82ff815 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -16,3 +16,4 @@ urllib3[secure]==1.23; python_version < '2.7' urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' # Last idna compatible with Python 2.6 was idna 2.7. idna==2.7; python_version < '2.7' +virtualenv==20.10.0 From da536e8ae1190573cd78d73de6b3b8610c2d23b4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 21:50:01 +0000 Subject: [PATCH 4/8] Fix stdlib typos that would cause NameError or AttributeError exceptions --- mitogen/profiler.py | 2 +- tests/testlib.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/profiler.py b/mitogen/profiler.py index bbf6086a..512a593e 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -90,7 +90,7 @@ def merge_stats(outpath, inpaths): break time.sleep(0.2) - stats.dump_stats(outpath) + pstats.dump_stats(outpath) def generate_stats(outpath, tmpdir): diff --git a/tests/testlib.py b/tests/testlib.py index 019b35d7..d40ce573 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -144,7 +144,7 @@ def wait_for_port( if not pattern: # Success: We connected & there's no banner check to perform. - sock.shutdown(socket.SHUTD_RDWR) + sock.shutdown(socket.SHUT_RDWR) sock.close() return From 9201761348e57625417881100a54f4bda4521367 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 21:51:53 +0000 Subject: [PATCH 5/8] ci: Format ansible-playbook output as yaml --- tests/ansible/ansible.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 59752492..2fe01ee3 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -5,7 +5,9 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy inventory_plugins = lib/inventory action_plugins = lib/action callback_plugins = lib/callback -stdout_callback = nice_stdout +stdout_callback = yaml +stdout_whitelist = + yaml vars_plugins = lib/vars library = lib/modules filter_plugins = lib/filters From 1bdf6294e4e15d06fb3e2541bb3c3e815365aa85 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 21:52:32 +0000 Subject: [PATCH 6/8] ci: Print playbook role execution times --- tests/ansible/ansible.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 2fe01ee3..dfc2858e 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -7,6 +7,8 @@ action_plugins = lib/action callback_plugins = lib/callback stdout_callback = yaml stdout_whitelist = + profile_roles, + timer, yaml vars_plugins = lib/vars library = lib/modules @@ -33,6 +35,9 @@ timeout = 10 # On Travis, paramiko check fails due to host key checking enabled. host_key_checking = False +[callback_profile_tasks] +task_output_limit = 10 + [ssh_connection] ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s pipelining = True From da0262cc180b89659936ccd91ff2bf43bfb5589f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 21:54:48 +0000 Subject: [PATCH 7/8] ci: Upgrade faulthandler to non-yanked release --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index a82ff815..c125a28f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,11 +1,11 @@ psutil==5.4.8 coverage==4.5.1 Django==1.6.11 # Last version supporting 2.6. +faulthandler==3.2; python_version < '3.3' mock==2.0.0 pytz==2018.5 cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect pycparser==2.18 # Last version supporting 2.6. -faulthandler==3.1; python_version < '3.3' # used by testlib pytest-catchlog==1.2.2 pytest==3.1.2 timeoutcontext==1.2.0 From e76eefb8be7cce03d3002c2c69889924ede4a80d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Nov 2021 22:00:33 +0000 Subject: [PATCH 8/8] Fix miscellaneous spelling/formatting --- .ci/ci_lib.py | 2 +- docs/howitworks.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 513ac98c..428d3ab9 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -167,7 +167,7 @@ def get_output(s, *args, **kwargs): def exists_in_path(progname): """ - Return True if proganme exists in $PATH. + Return True if progname exists in $PATH. >>> exists_in_path('echo') True diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 05c097e5..27b109fe 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -813,7 +813,7 @@ executes under the runtime importer lock, ensuring :py:keyword:`import` statements executing in local threads are serialized. .. note:: - + In Python 2, :py:exc:`ImportError` is raised when :py:keyword:`import` is attempted while the runtime import lock is held by another thread, therefore imports must be serialized by only attempting them from the main