Support packaging and importlib.metadata for pip module (#80881)

changelogs/fragments/80488-pip-pkg-resources.yml

@@ -0,0 +1,3 @@
+bugfixes:
+- pip module - Update module to prefer use of the python ``packaging`` and ``importlib.metadata`` modules due to ``pkg_resources`` being deprecated
+  (https://github.com/ansible/ansible/issues/80488)

lib/ansible/modules/pip.py

@@ -134,7 +134,7 @@ notes:
 requirements:
 - pip
 - virtualenv
-- setuptools
+- setuptools or packaging
 author:
 - Matt Wright (@mattupstate)
 '''
@@ -275,14 +275,23 @@ import traceback
 
 from ansible.module_utils.compat.version import LooseVersion
 
-SETUPTOOLS_IMP_ERR = None
+PACKAGING_IMP_ERR = None
+HAS_PACKAGING = False
+HAS_SETUPTOOLS = False
 try:
-    from pkg_resources import Requirement
+    from packaging.requirements import Requirement as parse_requirement
-
+    HAS_PACKAGING = True
-    HAS_SETUPTOOLS = True
+except Exception:
-except ImportError:
+    # This is catching a generic Exception, due to packaging on EL7 raising a TypeError on import
-    HAS_SETUPTOOLS = False
+    HAS_PACKAGING = False
-    SETUPTOOLS_IMP_ERR = traceback.format_exc()
+    PACKAGING_IMP_ERR = traceback.format_exc()
+    try:
+        from pkg_resources import Requirement
+        parse_requirement = Requirement.parse  # type: ignore[misc,assignment]
+        del Requirement
+        HAS_SETUPTOOLS = True
+    except ImportError:
+        pass
 
 from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.basic import AnsibleModule, is_executable, missing_required_lib
@@ -293,8 +302,16 @@ from ansible.module_utils.six import PY3
 #: Python one-liners to be run at the command line that will determine the
 # installed version for these special libraries.  These are libraries that
 # don't end up in the output of pip freeze.
-_SPECIAL_PACKAGE_CHECKERS = {'setuptools': 'import setuptools; print(setuptools.__version__)',
+_SPECIAL_PACKAGE_CHECKERS = {
-                             'pip': 'import pkg_resources; print(pkg_resources.get_distribution("pip").version)'}
+    'importlib': {
+        'setuptools': 'from importlib.metadata import version; print(version("setuptools"))',
+        'pip': 'from importlib.metadata import version; print(version("pip"))',
+    },
+    'pkg_resources': {
+        'setuptools': 'import setuptools; print(setuptools.__version__)',
+        'pip': 'import pkg_resources; print(pkg_resources.get_distribution("pip").version)',
+    }
+}
 
 _VCS_RE = re.compile(r'(svn|git|hg|bzr)\+')
 
@@ -503,7 +520,7 @@ def _fail(module, cmd, out, err):
     module.fail_json(cmd=cmd, msg=msg)
 
 
-def _get_package_info(module, package, env=None):
+def _get_package_info(module, package, python_bin=None):
     """This is only needed for special packages which do not show up in pip freeze
 
     pip and setuptools fall into this category.
@@ -511,20 +528,19 @@ def _get_package_info(module, package, env=None):
     :returns: a string containing the version number if the package is
         installed.  None if the package is not installed.
     """
-    if env:
-        opt_dirs = ['%s/bin' % env]
-    else:
-        opt_dirs = []
-    python_bin = module.get_bin_path('python', False, opt_dirs)
-
     if python_bin is None:
+        return
+
+    discovery_mechanism = 'pkg_resources'
+    importlib_rc = module.run_command([python_bin, '-c', 'import importlib.metadata'])[0]
+    if importlib_rc == 0:
+        discovery_mechanism = 'importlib'
+
+    rc, out, err = module.run_command([python_bin, '-c', _SPECIAL_PACKAGE_CHECKERS[discovery_mechanism][package]])
+    if rc:
         formatted_dep = None
     else:
-        rc, out, err = module.run_command([python_bin, '-c', _SPECIAL_PACKAGE_CHECKERS[package]])
+        formatted_dep = '%s==%s' % (package, out.strip())
-        if rc:
-            formatted_dep = None
-        else:
-            formatted_dep = '%s==%s' % (package, out.strip())
     return formatted_dep
 
 
@@ -602,13 +618,15 @@ class Package:
             separator = '==' if version_string[0].isdigit() else ' '
             name_string = separator.join((name_string, version_string))
         try:
-            self._requirement = Requirement.parse(name_string)
+            self._requirement = parse_requirement(name_string)
             # old pkg_resource will replace 'setuptools' with 'distribute' when it's already installed
-            if self._requirement.project_name == "distribute" and "setuptools" in name_string:
+            project_name = Package.canonicalize_name(
+                getattr(self._requirement, 'name', None) or getattr(self._requirement, 'project_name', None)
+            )
+            if project_name == "distribute" and "setuptools" in name_string:
                 self.package_name = "setuptools"
-                self._requirement.project_name = "setuptools"
             else:
-                self.package_name = Package.canonicalize_name(self._requirement.project_name)
+                self.package_name = project_name
             self._plain_package = True
         except ValueError as e:
             pass
@@ -616,7 +634,7 @@ class Package:
     @property
     def has_version_specifier(self):
         if self._plain_package:
-            return bool(self._requirement.specs)
+            return bool(getattr(self._requirement, 'specifier', None) or getattr(self._requirement, 'specs', None))
         return False
 
     def is_satisfied_by(self, version_to_test):
@@ -672,9 +690,9 @@ def main():
         supports_check_mode=True,
     )
 
-    if not HAS_SETUPTOOLS:
+    if not HAS_SETUPTOOLS and not HAS_PACKAGING:
-        module.fail_json(msg=missing_required_lib("setuptools"),
+        module.fail_json(msg=missing_required_lib("packaging"),
-                         exception=SETUPTOOLS_IMP_ERR)
+                         exception=PACKAGING_IMP_ERR)
 
     state = module.params['state']
     name = module.params['name']
@@ -714,6 +732,9 @@ def main():
             if not os.path.exists(os.path.join(env, 'bin', 'activate')):
                 venv_created = True
                 out, err = setup_virtualenv(module, env, chdir, out, err)
+            py_bin = os.path.join(env, 'bin', 'python')
+        else:
+            py_bin = module.params['executable'] or sys.executable
 
         pip = _get_pip(module, env, module.params['executable'])
 
@@ -796,7 +817,7 @@ def main():
                     # So we need to get those via a specialcase
                     for pkg in ('setuptools', 'pip'):
                         if pkg in name:
-                            formatted_dep = _get_package_info(module, pkg, env)
+                            formatted_dep = _get_package_info(module, pkg, py_bin)
                             if formatted_dep is not None:
                                 pkg_list.append(formatted_dep)
                                 out += '%s\n' % formatted_dep

test/integration/targets/pip/tasks/main.yml

@@ -40,6 +40,9 @@
         extra_args: "-c {{ remote_constraints }}"
 
     - include_tasks: pip.yml
+
+    - include_tasks: no_setuptools.yml
+      when: ansible_python.version_info[:2] >= [3, 8]
   always:
     - name: platform specific cleanup
       include_tasks: "{{ cleanup_filename }}"

test/integration/targets/pip/tasks/no_setuptools.yml

@@ -0,0 +1,48 @@
+- name: Get coverage version
+  pip:
+    name: coverage
+  check_mode: true
+  register: pip_coverage
+
+- name: create a virtualenv for use without setuptools
+  pip:
+    name:
+      - packaging
+      # coverage is needed when ansible-test is invoked with --coverage
+      # and using a custom ansible_python_interpreter below
+      - '{{ pip_coverage.stdout_lines|select("match", "coverage==")|first }}'
+    virtualenv: "{{ remote_tmp_dir }}/no_setuptools"
+
+- name: Remove setuptools
+  pip:
+    name:
+      - setuptools
+      - pkg_resources  # This shouldn't be a thing, but ubuntu 20.04...
+    virtualenv: "{{ remote_tmp_dir }}/no_setuptools"
+    state: absent
+
+- name: Ensure pkg_resources is gone
+  command: "{{ remote_tmp_dir }}/no_setuptools/bin/python -c 'import pkg_resources'"
+  register: result
+  failed_when: result.rc == 0
+
+- vars:
+    ansible_python_interpreter: "{{ remote_tmp_dir }}/no_setuptools/bin/python"
+  block:
+    - name: Checkmode install pip
+      pip:
+        name: pip
+        virtualenv: "{{ remote_tmp_dir }}/no_setuptools"
+      check_mode: true
+      register: pip_check_mode
+
+    - assert:
+        that:
+          - pip_check_mode.stdout is contains "pip=="
+          - pip_check_mode.stdout is not contains "setuptools=="
+
+    - name: Install fallible
+      pip:
+        name: fallible==0.0.1a2
+        virtualenv: "{{ remote_tmp_dir }}/no_setuptools"
+      register: fallible_install