Revamp the plugin_formatter doc generator

* Use a template to generate the category lists
* Refactor so that we first extract all of the data that we need to
  build the docs and then give that data to the templates to build with
* Add docs page listing modules ordered by support level

docs/bin/plugin_formatter.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
 # (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
+# (c) 2017 Ansible Project
 #
 # This file is part of Ansible
 #
@@ -17,10 +18,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-import cgi
+
 import datetime
 import glob
 import optparse
@@ -28,9 +29,17 @@ import os
 import re
 import sys
 import warnings
-import yaml
-
 from collections import defaultdict
+try:
+    from html import escape as html_escape
+except ImportError:
+    # Python-3.2 or later
+    import cgi
+
+    def html_escape(text, quote=True):
+        return cgi.escape(text, quote)
+
+import yaml
 from jinja2 import Environment, FileSystemLoader
 from six import iteritems
 
@@ -38,6 +47,7 @@ from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
 from ansible.utils import plugin_docs
 
+
 #####################################################################################
 # constants and paths
 
@@ -82,7 +92,7 @@ def rst_ify(text):
 def html_ify(text):
     ''' convert symbols like I(this is in italics) to valid HTML '''
 
-    t = cgi.escape(text)
+    t = html_escape(text)
     t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
     t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
     t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
@@ -104,25 +114,48 @@ def rst_xline(width, char="="):
     return char * width
 
 
-def write_data(text, options, outputname, module):
+def write_data(text, output_dir, outputname, module=None):
     ''' dumps module output to a file or the screen, as requested '''
 
-    if options.output_dir is not None:
+    if output_dir is not None:
-        fname = os.path.join(options.output_dir, outputname % module)
+        if module:
+            outputname = outputname % module
+        fname = os.path.join(output_dir, outputname)
         fname = fname.replace(".py", "")
-        f = open(fname, 'wb')
+        with open(fname, 'wb') as f:
             f.write(to_bytes(text))
-        f.close()
     else:
         print(text)
 
 
-def list_modules(module_dir, depth=0, limit_to_modules=None):
+def get_module_info(module_dir, limit_to_modules=None, verbose=False):
-    ''' returns a hash of categories, each category being a hash of module names to file paths '''
+    '''
+    Returns information about modules and the categories that they belong to
+
+    :arg module_dir: file system path to the top of the modules directory
+    :kwarg limit_to_modules: If given, this is a list of module names to
+        generate information for.  All other modules will be ignored.
+    :returns: Tuple of two dicts containing module_info, categories, and
+        aliases and a set listing deprecated modules:
+
+        :module_info: mapping of module names to information about them.  The fields of the dict are:
+
+            :path: filesystem path to the module
+            :deprecated: boolean.  True means the module is deprecated otherwise not.
+            :aliases: set of aliases to this module name
+            :metadata: The modules metadata (as recorded in the module)
+            :doc: The documentation structure for the module
+            :examples: The module's examples
+            :returndocs: The module's returndocs
+
+        :categories: maps category names to a dict.  The dict contains at
+            least one key, '_modules' which contains a list of module names in
+            that category.  Any other keys in the dict are subcategories with
+            the same structure.
+    '''
 
     categories = dict()
-    module_info = dict()
+    module_info = defaultdict(dict)
-    aliases = defaultdict(set)
 
     # * windows powershell modules have documentation stubs in python docstring
     #   format (they are not executed) so skip the ps1 format files
@@ -135,41 +168,72 @@ def list_modules(module_dir, depth=0, limit_to_modules=None):
     )
 
     for module_path in files:
+        # Do not list __init__.py files
         if module_path.endswith('__init__.py'):
             continue
+
+        # Do not list blacklisted modules
+        module = os.path.splitext(os.path.basename(module_path))[0]
+        if module in plugin_docs.BLACKLIST['MODULE']:
+            continue
+
+        # If requested, limit module documentation building only to passed-in
+        # modules.
+        if limit_to_modules is not None and module.lower() in limit_to_modules:
+            continue
+
+        deprecated = False
+        if module.startswith("_"):
+            if os.path.islink(module_path):
+                # Handle aliases
+                source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
+                module = module.replace("_", "", 1)
+                aliases = module_info[source].get('aliases', set())
+                aliases.add(module)
+                # In case we just created this via get()'s fallback
+                module_info[source]['aliases'] = aliases
+                continue
+            else:
+                # Handle deprecations
+                module = module.replace("_", "", 1)
+                deprecated = True
+
+        #
+        # Regular module to process
+        #
+
         category = categories
-        mod_path_only = module_path
-        # Start at the second directory because we don't want the "vendor"
 
+        # Start at the second directory because we don't want the "vendor"
         mod_path_only = os.path.dirname(module_path[len(module_dir):])
 
-        # directories (core, extras)
+        # build up the categories that this module belongs to
         for new_cat in mod_path_only.split('/')[1:]:
             if new_cat not in category:
                 category[new_cat] = dict()
+                category[new_cat]['_modules'] = []
             category = category[new_cat]
 
-        module = os.path.splitext(os.path.basename(module_path))[0]
+        category['_modules'].append(module)
-        if module in plugin_docs.BLACKLIST['MODULE']:
-            # Do not list blacklisted modules
-            continue
-        if module.startswith("_") and os.path.islink(module_path):
-            source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
-            module = module.replace("_", "", 1)
-            aliases[source].add(module)
-            continue
 
-        # If requested, limit module documentation building only to passed-in
+        # use ansible core library to parse out doc metadata YAML and plaintext examples
-        # modules.
+        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
-        if limit_to_modules is None or module.lower() in limit_to_modules:
+
-            category[module] = module_path
+        # save all the information
-            module_info[module] = module_path
+        module_info[module] = {'path': module_path,
+                               'deprecated': deprecated,
+                               'aliases': set(),
+                               'metadata': metadata,
+                               'doc': doc,
+                               'examples': examples,
+                               'returndocs': returndocs,
+                               }
 
     # keep module tests out of becoming module docs
     if 'test' in categories:
         del categories['test']
 
-    return module_info, categories, aliases
+    return module_info, categories
 
 
 def generate_parser():
@@ -202,17 +266,21 @@ def jinja2_environment(template_dir, typ):
                       trim_blocks=True)
     env.globals['xline'] = rst_xline
 
+    templates = {}
     if typ == 'rst':
         env.filters['convert_symbols_to_format'] = rst_ify
         env.filters['html_ify'] = html_ify
         env.filters['fmt'] = rst_fmt
         env.filters['xline'] = rst_xline
-        template = env.get_template('plugin.rst.j2')
+        templates['plugin'] = env.get_template('plugin.rst.j2')
+        templates['category_list'] = env.get_template('modules_by_category.rst.j2')
+        templates['support_list'] = env.get_template('modules_by_support.rst.j2')
+        templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_modules.rst.j2')
         outputname = "%s_module.rst"
     else:
         raise Exception("unknown module format type: %s" % typ)
 
-    return env, template, outputname
+    return templates, outputname
 
 
 def too_old(added):
@@ -225,196 +293,156 @@ def too_old(added):
     except ValueError as e:
         warnings.warn("Could not parse %s: %s" % (added, str(e)))
         return False
-    return (added_float < TO_OLD_TO_BE_NOTABLE)
+    return added_float < TO_OLD_TO_BE_NOTABLE
-
-
-def process_module(module, options, env, template, outputname, module_map, aliases):
 
-    fname = module_map[module]
-    if isinstance(fname, dict):
-        return "SKIPPED"
-
-    basename = os.path.basename(fname)
-    deprecated = False
-
-    # ignore files with extensions
-    if not basename.endswith(".py"):
-        return
-    elif module.startswith("_"):
-        if os.path.islink(fname):
-            return  # ignore, its an alias
-        deprecated = True
-        module = module.replace("_", "", 1)
 
+def process_modules(module_map, templates, outputname, output_dir, ansible_version):
+    for module in module_map:
         print("rendering: %s" % module)
 
-    # use ansible core library to parse out doc metadata YAML and plaintext examples
+        fname = module_map[module]['path']
-    doc, examples, returndocs, metadata = plugin_docs.get_docstring(fname, verbose=options.verbose)
 
         # crash if module is missing documentation and not explicitly hidden from docs index
-    if doc is None:
+        if module_map[module]['doc'] is None:
             sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
 
-    if deprecated and 'deprecated' not in doc:
+        # Going to reference this heavily so make a short name to reference it by
-        sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
+        doc = module_map[module]['doc']
 
-    if module in aliases:
+        if module_map[module]['deprecated'] and 'deprecated' not in doc:
-        doc['aliases'] = aliases[module]
+            sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
-
-    all_keys = []
 
         if 'version_added' not in doc:
             sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
 
+        #
+        # The present template gets everything from doc so we spend most of this
+        # function moving data into doc for the template to reference
+        #
+
+        if module_map[module]['aliases']:
+            doc['aliases'] = module_map[module]['aliases']
+
+        # don't show version added information if it's too old to be called out
         added = 0
         if doc['version_added'] == 'historical':
             del doc['version_added']
         else:
             added = doc['version_added']
 
-    # don't show version added information if it's too old to be called out
+        # Strip old version_added for the module
         if too_old(added):
             del doc['version_added']
 
+        option_names = []
+
         if 'options' in doc and doc['options']:
             for (k, v) in iteritems(doc['options']):
-            # don't show version added information if it's too old to be called out
+                # Error out if there's no description
-            if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
-                del doc['options'][k]['version_added']
                 if 'description' not in doc['options'][k]:
                     raise AnsibleError("Missing required description for option %s in %s " % (k, module))
 
+                # Error out if required isn't a boolean (people have been putting
+                # information on when something is required in here.  Those need
+                # to go in the description instead).
                 required_value = doc['options'][k].get('required', False)
                 if not isinstance(required_value, bool):
                     raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
+
+                # Strip old version_added information for options
+                if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
+                    del doc['options'][k]['version_added']
+
+                # Make sure description is a list of lines for later formatting
                 if not isinstance(doc['options'][k]['description'], list):
                     doc['options'][k]['description'] = [doc['options'][k]['description']]
 
-            all_keys.append(k)
+                option_names.append(k)
 
-    all_keys = sorted(all_keys)
+        option_names.sort()
 
-    doc['option_keys'] = all_keys
+        doc['option_keys'] = option_names
         doc['filename'] = fname
         doc['docuri'] = doc['module'].replace('_', '-')
         doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
-    doc['ansible_version'] = options.ansible_version
+        doc['ansible_version'] = ansible_version
-    doc['plainexamples'] = examples  # plain text
+        doc['plainexamples'] = module_map[module]['examples']  # plain text
-    doc['metadata'] = metadata
+        doc['metadata'] = module_map[module]['metadata']
 
-    if returndocs:
+        if module_map[module]['returndocs']:
             try:
-            doc['returndocs'] = yaml.safe_load(returndocs)
+                doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
             except:
-            print("could not load yaml: %s" % returndocs)
+                print("could not load yaml: %s" % module_map[module]['returndocs'])
                 raise
         else:
             doc['returndocs'] = None
 
-    # here is where we build the table of contents...
+        text = templates['plugin'].render(doc)
 
-    try:
+        write_data(text, output_dir, outputname, module)
-        text = template.render(doc)
-    except Exception as e:
-        raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e)))
-    write_data(text, options, outputname, module)
-    return doc['short_description']
 
 
-def print_modules(module, category_file, deprecated, options, env, template, outputname, module_map, aliases):
+def process_categories(mod_info, categories, templates, output_dir, output_name):
-    modstring = module
+    for category in sorted(categories.keys()):
-    if modstring.startswith('_'):
-        modstring = module[1:]
-    modname = modstring
-    if module in deprecated:
-        modstring = to_bytes(modstring) + DEPRECATED
-
-    category_file.write(b"  %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
-
-
-def process_category(category, categories, options, env, template, outputname):
-
-    # FIXME:
-    # We no longer conceptually deal with a mapping of category names to
-    # modules to file paths.  Instead we want several different records:
-    # (1) Mapping of module names to file paths (what's presently used
-    #     as categories['all']
-    # (2) Mapping of category names to lists of module names (what you'd
-    #     presently get from categories[category_name][subcategory_name].keys()
-    # (3) aliases (what's presently in categories['_aliases']
-    #
-    # list_modules() now returns those.  Need to refactor this function and
-    # main to work with them.
-
         module_map = categories[category]
-    module_info = categories['all']
+        category_filename = output_name % category
 
-    aliases = {}
+        print("*** recording category %s in %s ***" % (category, category_filename))
-    if '_aliases' in categories:
-        aliases = categories['_aliases']
-
-    category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
-    category_file = open(category_file_path, "wb")
-    print("*** recording category %s in %s ***" % (category, category_file_path))
 
         # start a new category file
 
         category = category.replace("_", " ")
         category = category.title()
 
-    modules = []
+        subcategories = dict((k, v) for k, v in module_map.items() if k != '_modules')
-    deprecated = []
+        template_data = {'title': category,
-    for module in module_map.keys():
+                         'category': module_map,
-        if isinstance(module_map[module], dict):
+                         'subcategories': subcategories,
-            for mod in (m for m in module_map[module].keys() if m in module_info):
+                         'module_info': mod_info,
-                if mod.startswith("_"):
+                         }
-                    deprecated.append(mod)
+
-        else:
+        text = templates['list_of_CATEGORY_modules'].render(template_data)
-            if module not in module_info:
+        write_data(text, output_dir, category_filename)
-                continue
+
-            if module.startswith("_"):
+
-                deprecated.append(module)
+def process_support_levels(mod_info, templates, output_dir):
-        modules.append(module)
+    supported_by = {'Ansible Core Team': {'slug': 'core_supported',
-
+                                          'modules': [],
-    modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
+                                          'output': 'core_maintained.rst'},
-
+                    'Ansible Network Team': {'slug': 'network_supported',
-    category_header = b"%s Modules" % (to_bytes(category.title()))
+                                             'modules': [],
-    underscores = b"`" * len(category_header)
+                                             'output': 'network_maintained.rst'},
-
+                    'Ansible Partners': {'slug': 'partner_supported',
-    category_file.write(b"""\
+                                         'modules': [],
-%s
+                                         'output': 'partner_maintained.rst'},
-%s
+                    'Ansible Community': {'slug': 'community_supported',
-
+                                          'modules': [],
-.. toctree:: :maxdepth: 1
+                                          'output': 'community_maintained.rst'},
-
+                    }
-""" % (category_header, underscores))
+
-    sections = []
+    # Separate the modules by support_level
-    for module in modules:
+    for module, info in mod_info.items():
-        if module in module_map and isinstance(module_map[module], dict):
+        if info['metadata']['supported_by'] == 'core':
-            sections.append(module)
+            supported_by['Ansible Core Team']['modules'].append(module)
-            continue
+        elif info['metadata']['supported_by'] == 'network':
+            supported_by['Ansible Network Team']['modules'].append(module)
+        elif info['metadata']['supported_by'] == 'certified':
+            supported_by['Ansible Partners']['modules'].append(module)
+        elif info['metadata']['supported_by'] == 'community':
+            supported_by['Ansible Community']['modules'].append(module)
         else:
-            print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
+            raise AnsibleError('Unknown supported_by value: %s' % info['metadata']['supported_by'])
-
-    sections.sort()
-    for section in sections:
-        category_file.write(b"\n%s\n%s\n\n" % (to_bytes(section.replace("_", " ").title()), b'-' * len(section)))
-        category_file.write(b".. toctree:: :maxdepth: 1\n\n")
-
-        section_modules = list(module_map[section].keys())
-        section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
-        # for module in module_map[section]:
-        for module in (m for m in section_modules if m in module_info):
-            print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
 
-    category_file.write(b"""\n\n
+    # Render the module lists
-.. note::
+    for maintainers, data in supported_by.items():
-    - %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
+        template_data = {'maintainers': maintainers,
-       The module documentation details page may explain more about this rationale.
+                         'modules': data['modules'],
-""" % DEPRECATED)
+                         'slug': data['slug'],
-    category_file.close()
+                         'module_info': mod_info,
-
+                         }
-    # TODO: end a new category file
+        text = templates['support_list'].render(template_data)
+        write_data(text, output_dir, data['output'])
 
 
 def validate_options(options):
@@ -435,43 +463,34 @@ def main():
     (options, args) = p.parse_args()
     validate_options(options)
 
-    env, template, outputname = jinja2_environment(options.template_dir, options.type)
+    templates, outputname = jinja2_environment(options.template_dir, options.type)
 
     # Convert passed-in limit_to_modules to None or list of modules.
     if options.limit_to_modules is not None:
         options.limit_to_modules = [s.lower() for s in options.limit_to_modules.split(",")]
 
-    mod_info, categories, aliases = list_modules(options.module_dir, limit_to_modules=options.limit_to_modules)
+    mod_info, categories = get_module_info(options.module_dir, limit_to_modules=options.limit_to_modules,
-    categories['all'] = mod_info
+                                           verbose=options.verbose)
-    categories['_aliases'] = aliases
+
-    category_names = [c for c in categories.keys() if not c.startswith('_')]
+    categories['all'] = {'_modules': mod_info.keys()}
-    category_names.sort()
+
+    # Transform the data
+    if options.type == 'rst':
+        for record in mod_info.values():
+            record['doc']['short_description'] = rst_ify(record['doc']['short_description'])
 
     # Write master category list
-    category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
+    category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
-    with open(category_list_path, "wb") as category_list_file:
+    write_data(category_list_text, options.output_dir, 'modules_by_category.rst')
-        category_list_file.write(b"Module Index\n")
+
-        category_list_file.write(b"============\n")
+    # Render all the individual module pages
-        category_list_file.write(b"\n\n")
+    process_modules(mod_info, templates, outputname, options.output_dir, options.ansible_version)
-        category_list_file.write(b".. toctree::\n")
+
-        category_list_file.write(b"   :maxdepth: 1\n\n")
+    # Render all the categories for modules
-
+    process_categories(mod_info, categories, templates, options.output_dir, "list_of_%s_modules.rst")
-        for category in category_names:
-            category_list_file.write(b"   list_of_%s_modules\n" % to_bytes(category))
-
-    # Import all the docs into memory
-    module_map = mod_info.copy()
-
-    for modname in module_map:
-        result = process_module(modname, options, env, template, outputname, module_map, aliases)
-        if result == 'SKIPPED':
-            del categories['all'][modname]
-        else:
-            categories['all'][modname] = (categories['all'][modname], result)
 
-    # Render all the docs to rst via category pages
+    # Render all the categories for modules
-    for category in category_names:
+    process_support_levels(mod_info, templates, options.output_dir)
-        process_category(category, categories, options, env, template, outputname)
 
 
 if __name__ == '__main__':

docs/docsite/rst/modules_support.rst

@@ -21,12 +21,18 @@ Core
 These are modules that the Ansible Core Team maintains and will always ship with Ansible itself.
 They will also receive slightly higher priority for all requests. Non-core modules are still fully usable.
 
+.. seealso::
+    List of :ref:`Core Supported<core_supported>` modules
+
 Network
 ```````
 
 These modules are supported by the Ansible Network Team in a relationship
 similar to how the Ansible Core Team maintains the Core modules.
 
+.. seealso::
+    List of :ref:`Network Supported<network_supported>` modules
+
 Certified
 `````````
 
@@ -37,6 +43,9 @@ Also, it is strongly recommended (but not presently required) for these types of
 
 These modules are currently shipped with Ansible, but might be shipped separately in the future.
 
+.. seealso::
+    List of :ref:`Certified<partner_supported>` modules
+
 Community
 `````````
 These modules **are not** supported by Core Committers or by companies/partners associated to the module. They are maintained by the community.
@@ -45,6 +54,10 @@ They are still fully usable, but the response rate to issues is purely up to the
 
 These modules are currently shipped with Ansible, but will most likely be shipped separately in the future.
 
+.. seealso::
+    List of Core Supported modules
+    List of :ref:`Community Supported<community_supported>` modules
+
 
 .. seealso::
 

docs/templates/list_of_CATEGORY_modules.rst.j2

@@ -0,0 +1,27 @@
+@{ title }@ Modules
+@{ '`' * title | length }@````````
+
+.. toctree:: :maxdepth: 1
+{% if category['_modules'] %}
+
+{% for module in category['_modules'] | sort %}
+  @{ module }@{% if module_info[module]['deprecated'] %} **(D)**{% endif%} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
+{% endfor %}
+{% endif %}
+
+{% for name, info in subcategories.items() | sort %}
+@{ name.title() }@
+@{ '-' * name | length }@
+
+.. toctree:: :maxdepth: 1
+
+{% for module in info['_modules'] | sort %}
+  @{ module }@{% if module_info[module]['deprecated'] %}**(D)**{% endif%} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
+{% endfor %}
+
+{% endfor %}
+
+.. note::
+    - **(D)**: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
+      The module documentation details page may explain more about this rationale.
+

docs/templates/modules_by_category.rst.j2

@@ -0,0 +1,9 @@
+Module Index
+============
+
+
+.. toctree:: :maxdepth: 1
+
+{% for name in categories %}
+   list_of_@{ name }@_modules
+{% endfor %}

docs/templates/modules_by_support.rst.j2

@@ -0,0 +1,15 @@
+.. _@{ slug }@:
+
+Modules Maintained by the @{ maintainers }@
+``````````````````````````@{ '`' * maintainers | length }@
+
+.. toctree:: :maxdepth: 1
+
+{% for module in modules | sort %}
+  @{ module }@{% if module_info[module]['deprecated'] %}**(D)**{% endif %} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
+{% endfor %}
+
+.. note::
+    - **(D)**: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
+      The module documentation details page may explain more about this rationale.
+