scan_packages: made adding package managers easier (#49079)

* made adding package managers easier

  added portage support

* moar pkg mgrs and moar info

 - added 'pkg' pkg manager (freebsd)
 - added pip
 - more apt info

* updated clgo

* Updates from feedback

Co-Authored-By: bcoca <bcoca@users.noreply.github.com>

* incorporated more feedback and added docstrings

* moar from feedback

  - made manager list dynamic and names based on class
  - better not found msg
  - made abstract metaclass again
  - test is now init exception
  - module to global
  - better dedupe comments

* more targetted errors/warnings

* added strategy, reordered to conserve priority

* rpm > apt

* move break to top

* fix trate

* piping it

* lines and meta

* refactored common functions

 - moved pip into it's own module
 - cleaned up base clases
 - ensure 'lower' match in package_facts

* missing license

* avoid facts

* update clog

* addressed feedback

* fix clog

* cleanup

* upd

* removed pip as that was removed

* renamed cpan

* added a single line since 2 lines are needed to be

readabnle instead of just 1 line, it is a huge problem otherwise

* fix internal ref

* not intended in this round

* updated as per fb

changelogs/fragments/scan_packages.yml

@@ -0,0 +1,3 @@
+minor_changes:
+    - package_facts, now supports multiple package managers per system.
+      New systems supported include Gentoo's portage with portage-utils installed, as well as FreeBSD's pkg

lib/ansible/module_utils/facts/packages.py

@@ -0,0 +1,83 @@
+# (c) 2018, Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from abc import ABCMeta, abstractmethod
+
+from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.basic import get_all_subclasses
+from ansible.module_utils.common.process import get_bin_path
+
+
+def get_all_pkg_managers():
+
+    return dict([(obj.__name__.lower(), obj) for obj in get_all_subclasses(PkgMgr) if obj not in (CLIMgr, LibMgr)])
+
+
+class PkgMgr(with_metaclass(ABCMeta, object)):
+
+    @abstractmethod
+    def is_available(self):
+        # This method is supposed to return True/False if the package manager is currently installed/usable
+        # It can also 'prep' the required systems in the process of detecting availability
+        pass
+
+    @abstractmethod
+    def list_installed(self):
+        # This method should return a list of installed packages, each list item will be passed to get_package_details
+        pass
+
+    @abstractmethod
+    def get_package_details(self, package):
+        # This takes a 'package' item and returns a dictionary with the package information, name and version are minimal requirements
+        pass
+
+    def get_packages(self):
+        # Take all of the above and return a dictionary of lists of dictionaries (package = list of installed versions)
+
+        installed_packages = {}
+        for package in self.list_installed():
+            package_details = self.get_package_details(package)
+            if 'source' not in package_details:
+                package_details['source'] = self.__class__.__name__.lower()
+            name = package_details['name']
+            if name not in installed_packages:
+                installed_packages[name] = [package_details]
+            else:
+                installed_packages[name].append(package_details)
+        return installed_packages
+
+
+class LibMgr(PkgMgr):
+
+    LIB = None
+
+    def __init__(self):
+
+        self._lib = None
+        super(LibMgr, self).__init__()
+
+    def is_available(self):
+        found = False
+        try:
+            self._lib = __import__(self.LIB)
+            found = True
+        except ImportError:
+            pass
+        return found
+
+
+class CLIMgr(PkgMgr):
+
+    CLI = None
+
+    def __init__(self):
+
+        self._cli = None
+        super(CLIMgr, self).__init__()
+
+    def is_available(self):
+        self._cli = get_bin_path(self.CLI, False)
+        return bool(self._cli)

lib/ansible/modules/packaging/language/pip_package_info.py

@@ -0,0 +1,152 @@
+#!/usr/bin/python
+# (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# started out with AWX's scan_packages module
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+DOCUMENTATION = '''
+module: pip_package_info
+short_description: pip package information
+description:
+  - Return information about installed pip packages
+version_added: "2.8"
+options:
+  clients:
+    description:
+      - A list of the pip executables that will be used to get the packages.
+        They can be supplied with the full path or just the executable name, i.e `pip3.7`.
+    default: ['pip']
+    required: False
+    type: list
+requirements:
+    - The requested pip executables must be installed on the target.
+author:
+  - Matthew Jones (@matburt)
+  - Brian Coca (@bcoca)
+  - Adam Miller (@maxamillion)
+'''
+
+EXAMPLES = '''
+- name: Just get the list from default pip
+  pip_package_info:
+
+- name: get the facts for default pip, pip2 and pip3.6
+  pip_package_info:
+    clients: ['pip', 'pip2', 'pip3.6']
+
+- name: get from specific paths (virtualenvs?)
+  pip_package_info:
+    clients: '/home/me/projec42/python/pip3.5'
+'''
+
+RETURN = '''
+packages:
+  description: a dictionary of installed package data
+  returned: always
+  type: dict
+  contains:
+    python:
+      description: A dictionary with each pip client which then contains a list of dicts with python package information
+      returned: always
+      type: dict
+      sample:
+        "packages": {
+            "pip": {
+                "Babel": [
+                    {
+                        "name": "Babel",
+                        "source": "pip",
+                        "version": "2.6.0"
+                    }
+                ],
+                "Flask": [
+                    {
+                        "name": "Flask",
+                        "source": "pip",
+                        "version": "1.0.2"
+                    }
+                ],
+                "Flask-SQLAlchemy": [
+                    {
+                        "name": "Flask-SQLAlchemy",
+                        "source": "pip",
+                        "version": "2.3.2"
+                    }
+                ],
+                "Jinja2": [
+                    {
+                        "name": "Jinja2",
+                        "source": "pip",
+                        "version": "2.10"
+                    }
+                ],
+            },
+        }
+'''
+import json
+import os
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.facts.packages import CLIMgr
+
+
+class PIP(CLIMgr):
+
+    def __init__(self, pip):
+
+        self.CLI = pip
+
+    def list_installed(self):
+        global module
+        rc, out, err = module.run_command([self._cli, 'list', '-l', '--format=json'])
+        if rc != 0:
+            raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
+        return json.loads(out)
+
+    def get_package_details(self, package):
+        package['source'] = self.CLI
+        return package
+
+
+def main():
+
+    # start work
+    global module
+    module = AnsibleModule(argument_spec=dict(clients={'type': 'list', 'default': ['pip']},), supports_check_mode=True)
+    packages = {}
+    results = {'packages': {}}
+    clients = module.params['clients']
+
+    found = 0
+    for pip in clients:
+
+        if not os.path.basename(pip).startswith('pip'):
+            module.warn('Skipping invalid pip client: %s' % (pip))
+            continue
+        try:
+            pip_mgr = PIP(pip)
+            if pip_mgr.is_available():
+                found += 1
+                packages[pip] = pip_mgr.get_packages()
+        except Exception as e:
+            module.warn('Failed to retrieve packages with %s: %s' % (pip, to_text(e)))
+            continue
+
+    if found == 0:
+        module.fail_json(msg='Unable to use any of the supplied pip clients: %s' % clients)
+
+    # return info
+    results['packages'] = packages
+    module.exit_json(**results)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/modules/packaging/os/package_facts.py

@@ -19,11 +19,24 @@ description:
 options:
   manager:
     description:
-      - The package manager used by the system so we can query the package information
-    default: auto
-    choices: ["auto", "rpm", "apt"]
+      - The package manager used by the system so we can query the package information.
+      - Since 2.8 this is a list and can support multiple package managers per system.
+      - The 'portage' and 'pkg' options were added in version 2.8.
+    default: ['auto']
+    choices: ['auto', 'rpm', 'apt', 'portage', 'pkg']
     required: False
+    type: list
+  strategy:
+    description:
+      - This option controls how the module queres the package managers on the system.
+        C(first) means it will return only informatino for the first supported package manager available.
+        C(all) will return information for all supported and available package managers on the system.
+    choices: ['first', 'all']
+    default: 'first'
+    version_added: "2.8"
 version_added: "2.5"
+requirements:
+    - For 'portage' support it requires the `qlist` utility, which is part of 'app-portage/portage-utils'.
 author:
   - Matthew Jones (@matburt)
   - Brian Coca (@bcoca)
@@ -140,94 +153,162 @@ ansible_facts:
         }
 '''
 
-import sys
-
+from ansible.module_utils._text import to_native, to_text
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_text
-
-
-def rpm_package_list():
-
-    try:
-        import rpm
-    except ImportError:
-        module.fail_json(msg='Unable to use the rpm python bindings, please ensure they are installed under the python the module runs under')
-
-    trans_set = rpm.TransactionSet()
-    installed_packages = {}
-    for package in trans_set.dbMatch():
-        package_details = dict(name=package[rpm.RPMTAG_NAME],
-                               version=package[rpm.RPMTAG_VERSION],
-                               release=package[rpm.RPMTAG_RELEASE],
-                               epoch=package[rpm.RPMTAG_EPOCH],
-                               arch=package[rpm.RPMTAG_ARCH],
-                               source='rpm')
-        if package_details['name'] not in installed_packages:
-            installed_packages[package_details['name']] = [package_details]
-        else:
-            installed_packages[package_details['name']].append(package_details)
-    return installed_packages
-
-
-def apt_package_list():
-
-    try:
-        import apt
-    except ImportError:
-        module.fail_json(msg='Unable to use the apt python bindings, please ensure they are installed under the python the module runs under')
-
-    apt_cache = apt.Cache()
-    installed_packages = {}
-    apt_installed_packages = [pk for pk in apt_cache.keys() if apt_cache[pk].is_installed]
-    for package in apt_installed_packages:
-        ac_pkg = apt_cache[package].installed
-        package_details = dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, source='apt')
-        if package_details['name'] not in installed_packages:
-            installed_packages[package_details['name']] = [package_details]
-        else:
-            installed_packages[package_details['name']].append(package_details)
-    return installed_packages
-
-
-# FIXME: add more listing methods
+from ansible.module_utils.facts.packages import LibMgr, CLIMgr, get_all_pkg_managers
+
+
+class RPM(LibMgr):
+
+    LIB = 'rpm'
+
+    def list_installed(self):
+        return self._lib.TransactionSet().dbMatch()
+
+    def get_package_details(self, package):
+        return dict(name=package[self._lib.RPMTAG_NAME],
+                    version=package[self._lib.RPMTAG_VERSION],
+                    release=package[self._lib.RPMTAG_RELEASE],
+                    epoch=package[self._lib.RPMTAG_EPOCH],
+                    arch=package[self._lib.RPMTAG_ARCH],)
+
+
+class APT(LibMgr):
+
+    LIB = 'apt'
+
+    @property
+    def pkg_cache(self):
+        if self._cache:
+            return self._cache
+
+        self._cache = self._lib.Cache()
+        return self._cache
+
+    def list_installed(self):
+        return [pk for pk in self.pkg_cache.keys() if self.pkg_cache[pk].is_installed]
+
+    def get_package_details(self, package):
+        ac_pkg = self.pkg_cache[package].installed
+        return dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, category=ac_pkg.section, origin=ac_pkg.origins[0].origin)
+
+
+class PKG(CLIMgr):
+
+    CLI = 'pkg'
+    atoms = ['name', 'version', 'origin', 'installed', 'automatic', 'arch', 'category', 'prefix', 'vital']
+
+    def list_installed(self):
+        rc, out, err = module.run_command([self._cli, 'query', "%%%s" % '\t%'.join(['n', 'v', 'R', 't', 'a', 'q', 'o', 'p', 'V'])])
+        if rc != 0 or err:
+            raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
+        return out.splitlines()
+
+    def get_package_details(self, package):
+
+        pkg = dict(zip(self.atoms, package.split('\t')))
+
+        if 'arch' in pkg:
+            try:
+                pkg['arch'] = pkg['arch'].split(':')[2]
+            except IndexError:
+                pass
+
+        if 'automatic' in pkg:
+            pkg['automatic'] = bool(pkg['automatic'])
+
+        if 'category' in pkg:
+            pkg['category'] = pkg['category'].split('/', 1)[0]
+
+        if 'version' in pkg:
+            if ',' in pkg['version']:
+                pkg['version'], pkg['port_epoch'] = pkg['version'].split(',', 1)
+            else:
+                pkg['port_epoch'] = 0
+
+            if '_' in pkg['version']:
+                pkg['version'], pkg['revision'] = pkg['version'].split('_', 1)
+            else:
+                pkg['revision'] = '0'
+
+        if 'vital' in pkg:
+            pkg['vital'] = bool(pkg['vital'])
+
+        return pkg
+
+
+class PORTAGE(CLIMgr):
+
+    CLI = 'qlist'
+    atoms = ['category', 'name', 'version', 'ebuild_revision', 'slots', 'prefixes', 'sufixes']
+
+    def list_installed(self):
+        rc, out, err = module.run_command(' '.join([self._cli, '-Iv', '|', 'xargs', '-n', '1024', 'qatom']), use_unsafe_shell=True)
+        if rc != 0:
+            raise RuntimeError("Unable to list packages rc=%s : %s" % (rc, to_native(err)))
+        return out.splitlines()
+
+    def get_package_details(self, package):
+        return dict(zip(self.atoms, package.split()))
+
+
 def main():
+
+    # get supported pkg managers
+    PKG_MANAGERS = get_all_pkg_managers()
+    PKG_MANAGER_NAMES = [x.lower() for x in PKG_MANAGERS.keys()]
+
+    # start work
     global module
-    module = AnsibleModule(argument_spec=dict(manager=dict()), supports_check_mode=True)
-    manager = module.params['manager']
+    module = AnsibleModule(argument_spec=dict(manager={'type': 'list', 'default': ['auto']},
+                                              strategy={'choices': ['first', 'all'], 'default': 'first'}),
+                           supports_check_mode=True)
     packages = {}
-    results = {}
+    results = {'ansible_facts': {}}
+    managers = [x.lower() for x in module.params['manager']]
+    strategy = module.params['strategy']
+
+    if 'auto' in managers:
+        # keep order from user, we do dedupe below
+        managers.extend(PKG_MANAGER_NAMES)
+        managers.remove('auto')
+
+    unsupported = set(managers).difference(PKG_MANAGER_NAMES)
+    if unsupported:
+        module.fail_json(msg='Unsupported package managers requested: %s' % (', '.join(unsupported)))
 
-    if manager is None or manager == 'auto':
+    found = 0
+    seen = set()
+    for pkgmgr in managers:
 
-        # detect!
-        for manager_lib in ('rpm', 'apt'):
+        if found and strategy == 'first':
+            break
+
+        # dedupe as per above
+        if pkgmgr in seen:
+            continue
+        seen.add(pkgmgr)
+        try:
             try:
-                dummy = __import__(manager_lib)
-                manager = manager_lib
-                break
-            except ImportError:
-                pass
+                # manager throws exception on init (calls self.test) if not usable.
+                manager = PKG_MANAGERS[pkgmgr]()
+                if manager.is_available():
+                    found += 1
+                    packages.update(manager.get_packages())
+            except Exception as e:
+                if pkgmgr in module.params['manager']:
+                    module.warn('Requested package manager %s was not usable by this module: %s' % (pkgmgr, to_text(e)))
+                continue
 
-        # FIXME: add more detection methods
-    try:
-        if manager == "rpm":
-            packages = rpm_package_list()
-        elif manager == "apt":
-            packages = apt_package_list()
-        else:
-            if manager:
-                results['msg'] = 'Unsupported package manager: %s' % manager
-                results['skipped'] = True
-            else:
-                module.fail_json(msg='Could not detect supported package manager')
-    except Exception as e:
-        from traceback import format_tb
-        module.fail_json(msg='Failed to retrieve packages: %s' % to_text(e), exception=format_tb(sys.exc_info()[2]))
-
-    results['ansible_facts'] = {}
-    # Set the facts, this will override the facts in ansible_facts that might
-    # exist from previous runs when using operating system level or distribution
-    # package managers
+        except Exception as e:
+            if pkgmgr in module.params['manager']:
+                module.warn('Failed to retrieve packages with %s: %s' % (pkgmgr, to_text(e)))
+
+    if found == 0:
+        module.fail_json(msg='Could not detect a supported package manager from the following list: %s' % managers)
+
+    # Set the facts, this will override the facts in ansible_facts that might exist from previous runs
+    # when using operating system level or distribution package managers
     results['ansible_facts']['packages'] = packages
 
     module.exit_json(**results)

test/sanity/validate-modules/ignore.txt

@@ -721,7 +721,6 @@ lib/ansible/modules/packaging/os/opkg.py E322
 lib/ansible/modules/packaging/os/opkg.py E324
 lib/ansible/modules/packaging/os/opkg.py E326
 lib/ansible/modules/packaging/os/opkg.py E336
-lib/ansible/modules/packaging/os/package_facts.py E324
 lib/ansible/modules/packaging/os/package_facts.py E326
 lib/ansible/modules/packaging/os/pacman.py E326
 lib/ansible/modules/packaging/os/pacman.py E336