diff --git a/changelogs/fragments/80488-pip-pkg-resources.yml b/changelogs/fragments/80488-pip-pkg-resources.yml new file mode 100644 index 00000000000..7e6d4393a0a --- /dev/null +++ b/changelogs/fragments/80488-pip-pkg-resources.yml @@ -0,0 +1,3 @@ +bugfixes: +- pip module - Update module to prefer use of the python ``packaging`` and ``importlib.metadata`` modules due to ``pkg_resources`` being deprecated + (https://github.com/ansible/ansible/issues/80488) diff --git a/lib/ansible/modules/pip.py b/lib/ansible/modules/pip.py index 9a3daf61b80..fa105bbef21 100644 --- a/lib/ansible/modules/pip.py +++ b/lib/ansible/modules/pip.py @@ -134,7 +134,7 @@ notes: requirements: - pip - virtualenv -- setuptools +- setuptools or packaging author: - Matt Wright (@mattupstate) ''' @@ -275,14 +275,23 @@ import traceback from ansible.module_utils.compat.version import LooseVersion -SETUPTOOLS_IMP_ERR = None +PACKAGING_IMP_ERR = None +HAS_PACKAGING = False +HAS_SETUPTOOLS = False try: - from pkg_resources import Requirement - - HAS_SETUPTOOLS = True -except ImportError: - HAS_SETUPTOOLS = False - SETUPTOOLS_IMP_ERR = traceback.format_exc() + from packaging.requirements import Requirement as parse_requirement + HAS_PACKAGING = True +except Exception: + # This is catching a generic Exception, due to packaging on EL7 raising a TypeError on import + HAS_PACKAGING = False + PACKAGING_IMP_ERR = traceback.format_exc() + try: + from pkg_resources import Requirement + parse_requirement = Requirement.parse # type: ignore[misc,assignment] + del Requirement + HAS_SETUPTOOLS = True + except ImportError: + pass from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.basic import AnsibleModule, is_executable, missing_required_lib @@ -293,8 +302,16 @@ from ansible.module_utils.six import PY3 #: Python one-liners to be run at the command line that will determine the # installed version for these special libraries. These are libraries that # don't end up in the output of pip freeze. -_SPECIAL_PACKAGE_CHECKERS = {'setuptools': 'import setuptools; print(setuptools.__version__)', - 'pip': 'import pkg_resources; print(pkg_resources.get_distribution("pip").version)'} +_SPECIAL_PACKAGE_CHECKERS = { + 'importlib': { + 'setuptools': 'from importlib.metadata import version; print(version("setuptools"))', + 'pip': 'from importlib.metadata import version; print(version("pip"))', + }, + 'pkg_resources': { + 'setuptools': 'import setuptools; print(setuptools.__version__)', + 'pip': 'import pkg_resources; print(pkg_resources.get_distribution("pip").version)', + } +} _VCS_RE = re.compile(r'(svn|git|hg|bzr)\+') @@ -503,7 +520,7 @@ def _fail(module, cmd, out, err): module.fail_json(cmd=cmd, msg=msg) -def _get_package_info(module, package, env=None): +def _get_package_info(module, package, python_bin=None): """This is only needed for special packages which do not show up in pip freeze pip and setuptools fall into this category. @@ -511,20 +528,19 @@ def _get_package_info(module, package, env=None): :returns: a string containing the version number if the package is installed. None if the package is not installed. """ - if env: - opt_dirs = ['%s/bin' % env] - else: - opt_dirs = [] - python_bin = module.get_bin_path('python', False, opt_dirs) - if python_bin is None: + return + + discovery_mechanism = 'pkg_resources' + importlib_rc = module.run_command([python_bin, '-c', 'import importlib.metadata'])[0] + if importlib_rc == 0: + discovery_mechanism = 'importlib' + + rc, out, err = module.run_command([python_bin, '-c', _SPECIAL_PACKAGE_CHECKERS[discovery_mechanism][package]]) + if rc: formatted_dep = None else: - rc, out, err = module.run_command([python_bin, '-c', _SPECIAL_PACKAGE_CHECKERS[package]]) - if rc: - formatted_dep = None - else: - formatted_dep = '%s==%s' % (package, out.strip()) + formatted_dep = '%s==%s' % (package, out.strip()) return formatted_dep @@ -602,13 +618,15 @@ class Package: separator = '==' if version_string[0].isdigit() else ' ' name_string = separator.join((name_string, version_string)) try: - self._requirement = Requirement.parse(name_string) + self._requirement = parse_requirement(name_string) # old pkg_resource will replace 'setuptools' with 'distribute' when it's already installed - if self._requirement.project_name == "distribute" and "setuptools" in name_string: + project_name = Package.canonicalize_name( + getattr(self._requirement, 'name', None) or getattr(self._requirement, 'project_name', None) + ) + if project_name == "distribute" and "setuptools" in name_string: self.package_name = "setuptools" - self._requirement.project_name = "setuptools" else: - self.package_name = Package.canonicalize_name(self._requirement.project_name) + self.package_name = project_name self._plain_package = True except ValueError as e: pass @@ -616,7 +634,7 @@ class Package: @property def has_version_specifier(self): if self._plain_package: - return bool(self._requirement.specs) + return bool(getattr(self._requirement, 'specifier', None) or getattr(self._requirement, 'specs', None)) return False def is_satisfied_by(self, version_to_test): @@ -672,9 +690,9 @@ def main(): supports_check_mode=True, ) - if not HAS_SETUPTOOLS: - module.fail_json(msg=missing_required_lib("setuptools"), - exception=SETUPTOOLS_IMP_ERR) + if not HAS_SETUPTOOLS and not HAS_PACKAGING: + module.fail_json(msg=missing_required_lib("packaging"), + exception=PACKAGING_IMP_ERR) state = module.params['state'] name = module.params['name'] @@ -714,6 +732,9 @@ def main(): if not os.path.exists(os.path.join(env, 'bin', 'activate')): venv_created = True out, err = setup_virtualenv(module, env, chdir, out, err) + py_bin = os.path.join(env, 'bin', 'python') + else: + py_bin = module.params['executable'] or sys.executable pip = _get_pip(module, env, module.params['executable']) @@ -796,7 +817,7 @@ def main(): # So we need to get those via a specialcase for pkg in ('setuptools', 'pip'): if pkg in name: - formatted_dep = _get_package_info(module, pkg, env) + formatted_dep = _get_package_info(module, pkg, py_bin) if formatted_dep is not None: pkg_list.append(formatted_dep) out += '%s\n' % formatted_dep diff --git a/test/integration/targets/pip/tasks/main.yml b/test/integration/targets/pip/tasks/main.yml index 66992fd017e..a3770702238 100644 --- a/test/integration/targets/pip/tasks/main.yml +++ b/test/integration/targets/pip/tasks/main.yml @@ -40,6 +40,9 @@ extra_args: "-c {{ remote_constraints }}" - include_tasks: pip.yml + + - include_tasks: no_setuptools.yml + when: ansible_python.version_info[:2] >= [3, 8] always: - name: platform specific cleanup include_tasks: "{{ cleanup_filename }}" diff --git a/test/integration/targets/pip/tasks/no_setuptools.yml b/test/integration/targets/pip/tasks/no_setuptools.yml new file mode 100644 index 00000000000..695605e8104 --- /dev/null +++ b/test/integration/targets/pip/tasks/no_setuptools.yml @@ -0,0 +1,48 @@ +- name: Get coverage version + pip: + name: coverage + check_mode: true + register: pip_coverage + +- name: create a virtualenv for use without setuptools + pip: + name: + - packaging + # coverage is needed when ansible-test is invoked with --coverage + # and using a custom ansible_python_interpreter below + - '{{ pip_coverage.stdout_lines|select("match", "coverage==")|first }}' + virtualenv: "{{ remote_tmp_dir }}/no_setuptools" + +- name: Remove setuptools + pip: + name: + - setuptools + - pkg_resources # This shouldn't be a thing, but ubuntu 20.04... + virtualenv: "{{ remote_tmp_dir }}/no_setuptools" + state: absent + +- name: Ensure pkg_resources is gone + command: "{{ remote_tmp_dir }}/no_setuptools/bin/python -c 'import pkg_resources'" + register: result + failed_when: result.rc == 0 + +- vars: + ansible_python_interpreter: "{{ remote_tmp_dir }}/no_setuptools/bin/python" + block: + - name: Checkmode install pip + pip: + name: pip + virtualenv: "{{ remote_tmp_dir }}/no_setuptools" + check_mode: true + register: pip_check_mode + + - assert: + that: + - pip_check_mode.stdout is contains "pip==" + - pip_check_mode.stdout is not contains "setuptools==" + + - name: Install fallible + pip: + name: fallible==0.0.1a2 + virtualenv: "{{ remote_tmp_dir }}/no_setuptools" + register: fallible_install