From 717f1092e38e4255c776125bfa823abff8562704 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 6 Aug 2024 14:53:40 -0700 Subject: [PATCH] ansible-test - Update venv management and sanity requirements (#83729) * ansible-test - Update venv management * Upgrade from pip 24.0 to 24.2 * Omit `wheel` and `setuptools` from ansible-test managed virtual environments * Drop pre-release hacks * Update mypy requirements * Freeze sanity test requirements * Update sanity test configuration * Update sanity ignores --- .../ansible-test-sanity-test-requirements.yml | 2 ++ .../fragments/ansible-test-venv-bootstrap.yml | 3 +++ hacking/update-sanity-requirements.py | 1 - .../_data/requirements/constraints.txt | 2 -- .../_data/requirements/sanity.ansible-doc.txt | 6 ++--- .../_data/requirements/sanity.changelog.txt | 8 +++---- .../requirements/sanity.import.plugin.txt | 4 ++-- .../_data/requirements/sanity.import.txt | 2 +- .../sanity.integration-aliases.txt | 2 +- .../_data/requirements/sanity.mypy.in | 3 ++- .../_data/requirements/sanity.mypy.txt | 24 +++++++++---------- .../_data/requirements/sanity.pep8.txt | 2 +- .../_data/requirements/sanity.pylint.txt | 12 ++++------ .../requirements/sanity.runtime-metadata.txt | 4 ++-- .../requirements/sanity.validate-modules.txt | 6 ++--- .../_internal/python_requirements.py | 19 ++++----------- .../controller/sanity/mypy/ansible-core.ini | 3 ++- .../controller/sanity/mypy/ansible-test.ini | 3 ++- .../sanity/pylint/config/collection.cfg | 1 + .../sanity/pylint/config/default.cfg | 1 + .../_util/target/setup/requirements.py | 8 +++++++ .../deprecated-config.requirements.txt | 4 ++-- .../code-smell/package-data.requirements.in | 2 -- .../code-smell/package-data.requirements.txt | 17 ++++++------- .../code-smell/pymarkdown.requirements.txt | 6 ++--- .../update-bundled.requirements.txt | 2 +- test/sanity/ignore.txt | 7 ++++++ 27 files changed, 80 insertions(+), 74 deletions(-) create mode 100644 changelogs/fragments/ansible-test-sanity-test-requirements.yml create mode 100644 changelogs/fragments/ansible-test-venv-bootstrap.yml diff --git a/changelogs/fragments/ansible-test-sanity-test-requirements.yml b/changelogs/fragments/ansible-test-sanity-test-requirements.yml new file mode 100644 index 00000000000..2bfd645e903 --- /dev/null +++ b/changelogs/fragments/ansible-test-sanity-test-requirements.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Updated the frozen requirements for all sanity tests. diff --git a/changelogs/fragments/ansible-test-venv-bootstrap.yml b/changelogs/fragments/ansible-test-venv-bootstrap.yml new file mode 100644 index 00000000000..3f6d8aa2b8d --- /dev/null +++ b/changelogs/fragments/ansible-test-venv-bootstrap.yml @@ -0,0 +1,3 @@ +minor_changes: + - ansible-test - Virtual environments created by ansible-test no longer include the ``wheel`` or ``setuptools`` packages. + - ansible-test - Upgrade ``pip`` used in ansible-test managed virtual environments from version 24.0 to 24.2. diff --git a/hacking/update-sanity-requirements.py b/hacking/update-sanity-requirements.py index 997d6dbf87a..aaaa803cde8 100755 --- a/hacking/update-sanity-requirements.py +++ b/hacking/update-sanity-requirements.py @@ -52,7 +52,6 @@ class SanityTest: if pip_freeze.stdout: raise Exception(f'Initial virtual environment is not empty:\n{pip_freeze.stdout}') - subprocess.run(pip + ['install', 'wheel'], env=env, check=True) # make bdist_wheel available during pip install subprocess.run(pip + ['install', '-r', self.source_path], env=env, check=True) freeze_options = ['--all'] diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index 40b84a1b1d4..c86de074596 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -8,5 +8,3 @@ requests-ntlm >= 1.1.0 # message encryption support requests-credssp >= 0.1.0 # message encryption support mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...) pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option -cffi == 1.17.0rc1 ; python_version >= '3.13' # temporary hack to support Python 3.13 -pyyaml == 6.0.2rc1 ; python_version >= '3.13' # temporary hack to support Python 3.13 diff --git a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt index 60ef86f66b8..a8b0ef3aec7 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt @@ -1,5 +1,5 @@ # edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 -packaging==24.0 -PyYAML==6.0.1 +packaging==24.1 +PyYAML==6.0.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt index cc5b635cb55..95aa188bd49 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt @@ -1,9 +1,9 @@ # edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog -antsibull-changelog==0.26.0 +antsibull-changelog==0.29.0 docutils==0.18.1 -packaging==24.0 -PyYAML==6.0.1 +packaging==24.1 +PyYAML==6.0.2 rstcheck==5.0.0 semantic-version==2.10.0 types-docutils==0.18.3 -typing_extensions==4.10.0 +typing_extensions==4.12.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt index 9116ed98c39..7d49234e591 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt @@ -1,4 +1,4 @@ # edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 -PyYAML==6.0.1 +PyYAML==6.0.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.import.txt b/test/lib/ansible_test/_data/requirements/sanity.import.txt index 4d9d4f530b0..3ea630668ed 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.import.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.import.txt @@ -1,2 +1,2 @@ # edit "sanity.import.in" and generate with: hacking/update-sanity-requirements.py --test import -PyYAML==6.0.1 +PyYAML==6.0.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt index 17d60b6faa9..3c7dd80db84 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt @@ -1,2 +1,2 @@ # edit "sanity.integration-aliases.in" and generate with: hacking/update-sanity-requirements.py --test integration-aliases -PyYAML==6.0.1 +PyYAML==6.0.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.mypy.in b/test/lib/ansible_test/_data/requirements/sanity.mypy.in index f01ae948d89..073513bdf8b 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.mypy.in +++ b/test/lib/ansible_test/_data/requirements/sanity.mypy.in @@ -2,9 +2,10 @@ mypy cryptography # type stubs not published separately jinja2 # type stubs not published separately packaging # type stubs not published separately +tomli # type stubs not published separately, required for toml inventory plugin types-backports types-paramiko types-pyyaml types-requests -types-setuptools +types-setuptools # required for the pkg_resources import in the pip module types-toml diff --git a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt index 651aea85961..a1a1bb08cf9 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt @@ -1,18 +1,18 @@ # edit "sanity.mypy.in" and generate with: hacking/update-sanity-requirements.py --test mypy -cffi==1.16.0 -cryptography==42.0.5 -Jinja2==3.1.3 +cffi==1.17.0 +cryptography==43.0.0 +Jinja2==3.1.4 MarkupSafe==2.1.5 -mypy==1.9.0 +mypy==1.11.1 mypy-extensions==1.0.0 -packaging==24.0 -pycparser==2.21 +packaging==24.1 +pycparser==2.22 tomli==2.0.1 types-backports==0.1.3 -types-paramiko==3.4.0.20240311 -types-PyYAML==6.0.12.20240311 -types-requests==2.31.0.20240311 -types-setuptools==69.2.0.20240317 +types-paramiko==3.4.0.20240423 +types-PyYAML==6.0.12.20240724 +types-requests==2.32.0.20240712 +types-setuptools==71.1.0.20240806 types-toml==0.10.8.20240310 -typing_extensions==4.10.0 -urllib3==2.2.1 +typing_extensions==4.12.2 +urllib3==2.2.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt index 51d2b644cda..6ed5e503b28 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt @@ -1,2 +1,2 @@ # edit "sanity.pep8.in" and generate with: hacking/update-sanity-requirements.py --test pep8 -pycodestyle==2.11.1 +pycodestyle==2.12.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt index b6bdec5e81d..a81d013253f 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt @@ -1,11 +1,9 @@ # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint -astroid==3.1.0 +astroid==3.2.4 dill==0.3.8 isort==5.13.2 mccabe==0.7.0 -platformdirs==4.2.0 -pylint==3.1.0 -PyYAML==6.0.1 -tomli==2.0.1 -tomlkit==0.12.4 -typing_extensions==4.10.0 +platformdirs==4.2.2 +pylint==3.2.6 +PyYAML==6.0.2 +tomlkit==0.13.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt index 8e6e2cebc91..e4b2449e5d5 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt @@ -1,3 +1,3 @@ # edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata -PyYAML==6.0.1 -voluptuous==0.14.2 +PyYAML==6.0.2 +voluptuous==0.15.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt index fba0da1d4aa..9a882275608 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt @@ -1,6 +1,6 @@ # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules antsibull-docs-parser==1.0.0 -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 -PyYAML==6.0.1 -voluptuous==0.14.2 +PyYAML==6.0.2 +voluptuous==0.15.2 diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 46d9df68e08..aaa60789849 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -112,6 +112,8 @@ class PipBootstrap(PipCommand): pip_version: str packages: list[str] + setuptools: bool + wheel: bool # Entry Points @@ -177,6 +179,8 @@ def collect_bootstrap(python: PythonConfig) -> list[PipCommand]: bootstrap = PipBootstrap( pip_version=pip_version, packages=packages, + setuptools=False, + wheel=False, ) return [bootstrap] @@ -218,17 +222,6 @@ def collect_requirements( # removing them reduces the size of environments cached in containers uninstall_packages = list(get_venv_packages(python)) - if not minimize: - # installed packages may have run-time dependencies on setuptools - uninstall_packages.remove('setuptools') - - # hack to allow the package-data sanity test to keep wheel in the venv - install_commands = [command for command in commands if isinstance(command, PipInstall)] - install_wheel = any(install.has_package('wheel') for install in install_commands) - - if install_wheel: - uninstall_packages.remove('wheel') - commands.extend(collect_uninstall(packages=uninstall_packages)) return commands @@ -412,9 +405,7 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]: # See: https://github.com/ansible/base-test-container/blob/main/files/installer.py default_packages = dict( - pip='24.0', - setuptools='70.0.0', - wheel='0.43.0', + pip='24.2', ) override_packages: dict[str, dict[str, str]] = { diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini index 0251f674b51..0d2208be2dd 100644 --- a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini @@ -8,7 +8,8 @@ strict_optional = False # There are ~70 errors reported in ansible-core when checking attributes. # Until the number of occurrences are reduced, it's better to disable the check. -disable_error_code = attr-defined +# The safe-super rule is disabled because it reports false positives on methods which return None. +disable_error_code = attr-defined,safe-super [mypy-ansible.module_utils.six.moves.*] ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini index 6be35724a42..8b7a8ab8c5f 100644 --- a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini @@ -9,7 +9,8 @@ strict_optional = False # There are ~13 type-abstract errors reported in ansible-test. # This is due to assumptions mypy makes about Type and abstract types. # See: https://discuss.python.org/t/add-abstracttype-to-the-typing-module/21996/13 -disable_error_code = type-abstract +# The safe-super rule is disabled because it reports false positives on methods which return None. +disable_error_code = type-abstract,safe-super [mypy-argcomplete] ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg index bb6c3e4d6a8..778bf77fd85 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg @@ -69,6 +69,7 @@ disable= pointless-statement, pointless-string-statement, possibly-unused-variable, + possibly-used-before-assignment, protected-access, raise-missing-from, # Python 2.x does not support raise from redefined-argument-from-local, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg index 6264948c285..d41185214e2 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg @@ -63,6 +63,7 @@ disable= not-an-iterable, not-callable, possibly-unused-variable, + possibly-used-before-assignment, protected-access, raise-missing-from, # Python 2.x does not support raise from redefined-argument-from-local, diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py index 5847ac731e5..da2555f4307 100644 --- a/test/lib/ansible_test/_util/target/setup/requirements.py +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -66,6 +66,8 @@ def bootstrap(pip, options): # type: (str, t.Dict[str, t.Any]) -> None """Bootstrap pip and related packages in an empty virtual environment.""" pip_version = options['pip_version'] packages = options['packages'] + setuptools = options['setuptools'] + wheel = options['wheel'] url = 'https://ci-files.testing.ansible.com/ansible-test/get-pip-%s.py' % pip_version cache_path = os.path.expanduser('~/.ansible/test/cache/get_pip_%s.py' % pip_version.replace(".", "_")) @@ -101,6 +103,12 @@ https://github.com/ansible/ansible/issues/77304 options = common_pip_options() options.extend(packages) + if not setuptools: + options.append('--no-setuptools') + + if not wheel: + options.append('--no-wheel') + command = [sys.executable, pip] + options execute_command(command, env=env) diff --git a/test/sanity/code-smell/deprecated-config.requirements.txt b/test/sanity/code-smell/deprecated-config.requirements.txt index 42c1825adf2..21f88654002 100644 --- a/test/sanity/code-smell/deprecated-config.requirements.txt +++ b/test/sanity/code-smell/deprecated-config.requirements.txt @@ -1,4 +1,4 @@ # edit "deprecated-config.requirements.in" and generate with: hacking/update-sanity-requirements.py --test deprecated-config -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 -PyYAML==6.0.1 +PyYAML==6.0.2 diff --git a/test/sanity/code-smell/package-data.requirements.in b/test/sanity/code-smell/package-data.requirements.in index 81b58bcf713..493ddd4bda4 100644 --- a/test/sanity/code-smell/package-data.requirements.in +++ b/test/sanity/code-smell/package-data.requirements.in @@ -1,8 +1,6 @@ build # required to build sdist -wheel # required to build wheel jinja2 pyyaml resolvelib < 1.1.0 rstcheck < 6 # newer versions have too many dependencies antsibull-changelog -setuptools == 66.1.0 # minimum supported setuptools diff --git a/test/sanity/code-smell/package-data.requirements.txt b/test/sanity/code-smell/package-data.requirements.txt index 4faee339caa..136fe084e0a 100644 --- a/test/sanity/code-smell/package-data.requirements.txt +++ b/test/sanity/code-smell/package-data.requirements.txt @@ -1,17 +1,14 @@ # edit "package-data.requirements.in" and generate with: hacking/update-sanity-requirements.py --test package-data -antsibull-changelog==0.26.0 -build==1.1.1 +antsibull-changelog==0.29.0 +build==1.2.1 docutils==0.18.1 -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 -packaging==24.0 -pyproject_hooks==1.0.0 -PyYAML==6.0.1 +packaging==24.1 +pyproject_hooks==1.1.0 +PyYAML==6.0.2 resolvelib==1.0.1 rstcheck==5.0.0 semantic-version==2.10.0 -setuptools==66.1.0 -tomli==2.0.1 types-docutils==0.18.3 -typing_extensions==4.10.0 -wheel==0.43.0 +typing_extensions==4.12.2 diff --git a/test/sanity/code-smell/pymarkdown.requirements.txt b/test/sanity/code-smell/pymarkdown.requirements.txt index c1571c9cd68..cf8007962e5 100644 --- a/test/sanity/code-smell/pymarkdown.requirements.txt +++ b/test/sanity/code-smell/pymarkdown.requirements.txt @@ -1,9 +1,9 @@ # edit "pymarkdown.requirements.in" and generate with: hacking/update-sanity-requirements.py --test pymarkdown application_properties==0.8.2 Columnar==1.4.1 -pymarkdownlnt==0.9.18 -PyYAML==6.0.1 +pymarkdownlnt==0.9.22 +PyYAML==6.0.2 tomli==2.0.1 toolz==0.12.1 -typing_extensions==4.10.0 +typing_extensions==4.12.2 wcwidth==0.2.13 diff --git a/test/sanity/code-smell/update-bundled.requirements.txt b/test/sanity/code-smell/update-bundled.requirements.txt index 76bf9c5854d..347c56f7ef5 100644 --- a/test/sanity/code-smell/update-bundled.requirements.txt +++ b/test/sanity/code-smell/update-bundled.requirements.txt @@ -1,2 +1,2 @@ # edit "update-bundled.requirements.in" and generate with: hacking/update-sanity-requirements.py --test update-bundled -packaging==24.0 +packaging==24.1 diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 45389cef7c0..9a1328eff28 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -160,3 +160,10 @@ test/integration/targets/find/files/hello_world.gbk no-smart-quotes test/integration/targets/find/files/hello_world.gbk no-unwanted-characters lib/ansible/plugins/action/__init__.py pylint:ansible-deprecated-version # 2.18 deprecation lib/ansible/template/__init__.py pylint:ansible-deprecated-version # 2.18 deprecation +lib/ansible/module_utils/facts/hardware/aix.py pylint:used-before-assignment +lib/ansible/modules/rpm_key.py pylint:used-before-assignment +lib/ansible/modules/service.py pylint:used-before-assignment +lib/ansible/modules/user.py pylint:used-before-assignment +lib/ansible/plugins/action/copy.py pylint:undefined-variable +test/integration/targets/module_utils/library/test_optional.py pylint:used-before-assignment +test/support/windows-integration/plugins/action/win_copy.py pylint:undefined-variable