ansible/lib/ansible/module_utils/facts/collector.py

# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# (c) 2017 Red Hat Inc.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from collections import defaultdict

import platform

import ansible.module_utils.compat.typing as t

from ansible.module_utils.facts import timeout


class CycleFoundInFactDeps(Exception):
    '''Indicates there is a cycle in fact collector deps

    If collector-B requires collector-A, and collector-A requires
    collector-B, that is a cycle. In that case, there is no ordering
    that will satisfy B before A and A and before B. That will cause this
    error to be raised.
    '''
    pass


class UnresolvedFactDep(ValueError):
    pass


class CollectorNotFoundError(KeyError):
    pass


class BaseFactCollector:
    _fact_ids = set()  # type: t.Set[str]

    _platform = 'Generic'
    name = None  # type: str | None
    required_facts = set()  # type: t.Set[str]

    def __init__(self, collectors=None, namespace=None):
        '''Base class for things that collect facts.

        'collectors' is an optional list of other FactCollectors for composing.'''
        self.collectors = collectors or []

        # self.namespace is a object with a 'transform' method that transforms
        # the name to indicate the namespace (ie, adds a prefix or suffix).
        self.namespace = namespace

        self.fact_ids = set([self.name])
        self.fact_ids.update(self._fact_ids)

    @classmethod
    def platform_match(cls, platform_info):
        if platform_info.get('system', None) == cls._platform:
            return cls
        return None

    def _transform_name(self, key_name):
        if self.namespace:
            return self.namespace.transform(key_name)
        return key_name

    def _transform_dict_keys(self, fact_dict):
        '''update a dicts keys to use new names as transformed by self._transform_name'''

        for old_key in list(fact_dict.keys()):
            new_key = self._transform_name(old_key)
            # pop the item by old_key and replace it using new_key
            fact_dict[new_key] = fact_dict.pop(old_key)
        return fact_dict

    # TODO/MAYBE: rename to 'collect' and add 'collect_without_namespace'
    def collect_with_namespace(self, module=None, collected_facts=None):
        # collect, then transform the key names if needed
        facts_dict = self.collect(module=module, collected_facts=collected_facts)
        if self.namespace:
            facts_dict = self._transform_dict_keys(facts_dict)
        return facts_dict

    def collect(self, module=None, collected_facts=None):
        '''do the fact collection

        'collected_facts' is a object (a dict, likely) that holds all previously
          facts. This is intended to be used if a FactCollector needs to reference
          another fact (for ex, the system arch) and should not be modified (usually).

          Returns a dict of facts.

          '''
        facts_dict = {}
        return facts_dict


def get_collector_names(valid_subsets=None,
                        minimal_gather_subset=None,
                        gather_subset=None,
                        aliases_map=None,
                        platform_info=None):
    '''return a set of FactCollector names based on gather_subset spec.

    gather_subset is a spec describing which facts to gather.
    valid_subsets is a frozenset of potential matches for gather_subset ('all', 'network') etc
    minimal_gather_subsets is a frozenset of matches to always use, even for gather_subset='!all'
    '''

    # Retrieve module parameters
    gather_subset = gather_subset or ['all']

    # the list of everything that 'all' expands to
    valid_subsets = valid_subsets or frozenset()

    # if provided, minimal_gather_subset is always added, even after all negations
    minimal_gather_subset = minimal_gather_subset or frozenset()

    aliases_map = aliases_map or defaultdict(set)

    # Retrieve all facts elements
    additional_subsets = set()
    exclude_subsets = set()

    # total always starts with the min set, then
    # adds of the additions in gather_subset, then
    # excludes all of the excludes, then add any explicitly
    # requested subsets.
    gather_subset_with_min = ['min']
    gather_subset_with_min.extend(gather_subset)

    # subsets we mention in gather_subset explicitly, except for 'all'/'min'
    explicitly_added = set()

    for subset in gather_subset_with_min:
        subset_id = subset
        if subset_id == 'min':
            additional_subsets.update(minimal_gather_subset)
            continue
        if subset_id == 'all':
            additional_subsets.update(valid_subsets)
            continue
        if subset_id.startswith('!'):
            subset = subset[1:]
            if subset == 'min':
                exclude_subsets.update(minimal_gather_subset)
                continue
            if subset == 'all':
                exclude_subsets.update(valid_subsets - minimal_gather_subset)
                continue
            exclude = True
        else:
            exclude = False

        if exclude:
            # include 'devices', 'dmi' etc for '!hardware'
            exclude_subsets.update(aliases_map.get(subset, set()))
            exclude_subsets.add(subset)
        else:
            # NOTE: this only considers adding an unknown gather subsetup an error. Asking to
            #       exclude an unknown gather subset is ignored.
            if subset_id not in valid_subsets:
                raise TypeError("Bad subset '%s' given to Ansible. gather_subset options allowed: all, %s" %
                                (subset, ", ".join(sorted(valid_subsets))))

            explicitly_added.add(subset)
            additional_subsets.add(subset)

    if not additional_subsets:
        additional_subsets.update(valid_subsets)

    additional_subsets.difference_update(exclude_subsets - explicitly_added)

    return additional_subsets


def find_collectors_for_platform(all_collector_classes, compat_platforms):
    found_collectors = set()
    found_collectors_names = set()

    # start from specific platform, then try generic
    for compat_platform in compat_platforms:
        platform_match = None
        for all_collector_class in all_collector_classes:

            # ask the class if it is compatible with the platform info
            platform_match = all_collector_class.platform_match(compat_platform)

            if not platform_match:
                continue

            primary_name = all_collector_class.name

            if primary_name not in found_collectors_names:
                found_collectors.add(all_collector_class)
                found_collectors_names.add(all_collector_class.name)

    return found_collectors


def build_fact_id_to_collector_map(collectors_for_platform):
    fact_id_to_collector_map = defaultdict(list)
    aliases_map = defaultdict(set)

    for collector_class in collectors_for_platform:
        primary_name = collector_class.name

        fact_id_to_collector_map[primary_name].append(collector_class)

        for fact_id in collector_class._fact_ids:
            fact_id_to_collector_map[fact_id].append(collector_class)
            aliases_map[primary_name].add(fact_id)

    return fact_id_to_collector_map, aliases_map


def select_collector_classes(collector_names, all_fact_subsets):
    seen_collector_classes = set()

    selected_collector_classes = []

    for collector_name in collector_names:
        collector_classes = all_fact_subsets.get(collector_name, [])
        for collector_class in collector_classes:
            if collector_class not in seen_collector_classes:
                selected_collector_classes.append(collector_class)
                seen_collector_classes.add(collector_class)

    return selected_collector_classes


def _get_requires_by_collector_name(collector_name, all_fact_subsets):
    required_facts = set()

    try:
        collector_classes = all_fact_subsets[collector_name]
    except KeyError:
        raise CollectorNotFoundError('Fact collector "%s" not found' % collector_name)
    for collector_class in collector_classes:
        required_facts.update(collector_class.required_facts)
    return required_facts


def find_unresolved_requires(collector_names, all_fact_subsets):
    '''Find any collector names that have unresolved requires

    Returns a list of collector names that correspond to collector
    classes whose .requires_facts() are not in collector_names.
    '''
    unresolved = set()

    for collector_name in collector_names:
        required_facts = _get_requires_by_collector_name(collector_name, all_fact_subsets)
        for required_fact in required_facts:
            if required_fact not in collector_names:
                unresolved.add(required_fact)

    return unresolved


def resolve_requires(unresolved_requires, all_fact_subsets):
    new_names = set()
    failed = []
    for unresolved in unresolved_requires:
        if unresolved in all_fact_subsets:
            new_names.add(unresolved)
        else:
            failed.append(unresolved)

    if failed:
        raise UnresolvedFactDep('unresolved fact dep %s' % ','.join(failed))
    return new_names


def build_dep_data(collector_names, all_fact_subsets):
    dep_map = defaultdict(set)
    for collector_name in collector_names:
        collector_deps = set()
        for collector in all_fact_subsets[collector_name]:
            for dep in collector.required_facts:
                collector_deps.add(dep)
        dep_map[collector_name] = collector_deps
    return dep_map


def tsort(dep_map):
    sorted_list = []

    unsorted_map = dep_map.copy()

    while unsorted_map:
        acyclic = False
        for node, edges in list(unsorted_map.items()):
            for edge in edges:
                if edge in unsorted_map:
                    break
            else:
                acyclic = True
                del unsorted_map[node]
                sorted_list.append((node, edges))

        if not acyclic:
            raise CycleFoundInFactDeps('Unable to tsort deps, there was a cycle in the graph. sorted=%s' % sorted_list)

    return sorted_list


def _solve_deps(collector_names, all_fact_subsets):
    unresolved = collector_names.copy()
    solutions = collector_names.copy()

    while True:
        unresolved = find_unresolved_requires(solutions, all_fact_subsets)
        if unresolved == set():
            break

        new_names = resolve_requires(unresolved, all_fact_subsets)
        solutions.update(new_names)

    return solutions


def collector_classes_from_gather_subset(all_collector_classes=None,
                                         valid_subsets=None,
                                         minimal_gather_subset=None,
                                         gather_subset=None,
                                         gather_timeout=None,
                                         platform_info=None):
    '''return a list of collector classes that match the args'''

    # use gather_name etc to get the list of collectors

    all_collector_classes = all_collector_classes or []

    minimal_gather_subset = minimal_gather_subset or frozenset()

    platform_info = platform_info or {'system': platform.system()}

    gather_timeout = gather_timeout or timeout.DEFAULT_GATHER_TIMEOUT

    # tweak the modules GATHER_TIMEOUT
    timeout.GATHER_TIMEOUT = gather_timeout

    valid_subsets = valid_subsets or frozenset()

    # maps alias names like 'hardware' to the list of names that are part of hardware
    # like 'devices' and 'dmi'
    aliases_map = defaultdict(set)

    compat_platforms = [platform_info, {'system': 'Generic'}]

    collectors_for_platform = find_collectors_for_platform(all_collector_classes, compat_platforms)

    # all_facts_subsets maps the subset name ('hardware') to the class that provides it.

    # TODO: name collisions here? are there facts with the same name as a gather_subset (all, network, hardware, virtual, ohai, facter)
    all_fact_subsets, aliases_map = build_fact_id_to_collector_map(collectors_for_platform)

    all_valid_subsets = frozenset(all_fact_subsets.keys())

    # expand any fact_id/collectorname/gather_subset term ('all', 'env', etc) to the list of names that represents
    collector_names = get_collector_names(valid_subsets=all_valid_subsets,
                                          minimal_gather_subset=minimal_gather_subset,
                                          gather_subset=gather_subset,
                                          aliases_map=aliases_map,
                                          platform_info=platform_info)

    complete_collector_names = _solve_deps(collector_names, all_fact_subsets)

    dep_map = build_dep_data(complete_collector_names, all_fact_subsets)

    ordered_deps = tsort(dep_map)
    ordered_collector_names = [x[0] for x in ordered_deps]

    selected_collector_classes = select_collector_classes(ordered_collector_names,
                                                          all_fact_subsets)

    return selected_collector_classes