archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Revamp the plugin_formatter doc generator

Browse Source 
* Use a template to generate the category lists
* Refactor so that we first extract all of the data that we need to
  build the docs and then give that data to the templates to build with
* Add docs page listing modules ordered by support level
pull/28265/head
Toshio Kuratomi 7 years ago
parent af2073d057
commit 546187a8af
5 changed files with 324 additions and 241 deletions
Show Stats Download Patch File Download Diff File

501
docs/bin/plugin_formatter.py
Unescape Escape View File

@ -1,6 +1,7 @@
#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
# (c) 2017 Ansible Project
#
# This file is part of Ansible
#
@ -17,10 +18,10 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
from __future__ import absolute_import, division, print_function
__metaclass__ = type
import cgi
import datetime
import glob
import optparse
@ -28,9 +29,17 @@ import os
import re
import sys
import warnings
import yaml
from collections import defaultdict
try:
from html import escape as html_escape
except ImportError:
# Python-3.2 or later
import cgi
def html_escape(text, quote=True):
return cgi.escape(text, quote)
import yaml
from jinja2 import Environment, FileSystemLoader
from six import iteritems
@ -38,6 +47,7 @@ from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.utils import plugin_docs
#####################################################################################
# constants and paths
@ -82,7 +92,7 @@ def rst_ify(text):
def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML '''
t = cgi.escape(text)
t = html_escape(text)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
@ -104,25 +114,48 @@ def rst_xline(width, char="="):
return char * width
def write_data(text, options, outputname, module):
def write_data(text, output_dir, outputname, module=None):
''' dumps module output to a file or the screen, as requested '''
if options.output_dir is not None:
fname = os.path.join(options.output_dir, outputname % module)
if output_dir is not None:
if module:
outputname = outputname % module
fname = os.path.join(output_dir, outputname)
fname = fname.replace(".py", "")
f = open(fname, 'wb')
f.write(to_bytes(text))
f.close()
with open(fname, 'wb') as f:
f.write(to_bytes(text))
else:
print(text)
def list_modules(module_dir, depth=0, limit_to_modules=None):
''' returns a hash of categories, each category being a hash of module names to file paths '''
def get_module_info(module_dir, limit_to_modules=None, verbose=False):
'''
Returns information about modules and the categories that they belong to
:arg module_dir: file system path to the top of the modules directory
:kwarg limit_to_modules: If given, this is a list of module names to
generate information for. All other modules will be ignored.
:returns: Tuple of two dicts containing module_info, categories, and
aliases and a set listing deprecated modules:
:module_info: mapping of module names to information about them. The fields of the dict are:
:path: filesystem path to the module
:deprecated: boolean. True means the module is deprecated otherwise not.
:aliases: set of aliases to this module name
:metadata: The modules metadata (as recorded in the module)
:doc: The documentation structure for the module
:examples: The module's examples
:returndocs: The module's returndocs
:categories: maps category names to a dict. The dict contains at
least one key, '_modules' which contains a list of module names in
that category. Any other keys in the dict are subcategories with
the same structure.
'''
categories = dict()
module_info = dict()
aliases = defaultdict(set)
module_info = defaultdict(dict)
# * windows powershell modules have documentation stubs in python docstring
# format (they are not executed) so skip the ps1 format files
@ -135,41 +168,72 @@ def list_modules(module_dir, depth=0, limit_to_modules=None):
)
for module_path in files:
# Do not list __init__.py files
if module_path.endswith('__init__.py'):
continue
# Do not list blacklisted modules
module = os.path.splitext(os.path.basename(module_path))[0]
if module in plugin_docs.BLACKLIST['MODULE']:
continue
# If requested, limit module documentation building only to passed-in
# modules.
if limit_to_modules is not None and module.lower() in limit_to_modules:
continue
deprecated = False
if module.startswith("_"):
if os.path.islink(module_path):
# Handle aliases
source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
module = module.replace("_", "", 1)
aliases = module_info[source].get('aliases', set())
aliases.add(module)
# In case we just created this via get()'s fallback
module_info[source]['aliases'] = aliases
continue
else:
# Handle deprecations
module = module.replace("_", "", 1)
deprecated = True
#
# Regular module to process
#
category = categories
mod_path_only = module_path
# Start at the second directory because we don't want the "vendor"
# Start at the second directory because we don't want the "vendor"
mod_path_only = os.path.dirname(module_path[len(module_dir):])
# directories (core, extras)
# build up the categories that this module belongs to
for new_cat in mod_path_only.split('/')[1:]:
if new_cat not in category:
category[new_cat] = dict()
category[new_cat]['_modules'] = []
category = category[new_cat]
module = os.path.splitext(os.path.basename(module_path))[0]
if module in plugin_docs.BLACKLIST['MODULE']:
# Do not list blacklisted modules
continue
if module.startswith("_") and os.path.islink(module_path):
source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
module = module.replace("_", "", 1)
aliases[source].add(module)
continue
category['_modules'].append(module)
# If requested, limit module documentation building only to passed-in
# modules.
if limit_to_modules is None or module.lower() in limit_to_modules:
category[module] = module_path
module_info[module] = module_path
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
# save all the information
module_info[module] = {'path': module_path,
'deprecated': deprecated,
'aliases': set(),
'metadata': metadata,
'doc': doc,
'examples': examples,
'returndocs': returndocs,
}
# keep module tests out of becoming module docs
if 'test' in categories:
del categories['test']
return module_info, categories, aliases
return module_info, categories
def generate_parser():
@ -202,17 +266,21 @@ def jinja2_environment(template_dir, typ):
trim_blocks=True)
env.globals['xline'] = rst_xline
templates = {}
if typ == 'rst':
env.filters['convert_symbols_to_format'] = rst_ify
env.filters['html_ify'] = html_ify
env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline
template = env.get_template('plugin.rst.j2')
templates['plugin'] = env.get_template('plugin.rst.j2')
templates['category_list'] = env.get_template('modules_by_category.rst.j2')
templates['support_list'] = env.get_template('modules_by_support.rst.j2')
templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_modules.rst.j2')
outputname = "%s_module.rst"
else:
raise Exception("unknown module format type: %s" % typ)
return env, template, outputname
return templates, outputname
def too_old(added):
@ -225,196 +293,156 @@ def too_old(added):
except ValueError as e:
warnings.warn("Could not parse %s: %s" % (added, str(e)))
return False
return (added_float < TO_OLD_TO_BE_NOTABLE)
def process_module(module, options, env, template, outputname, module_map, aliases):
fname = module_map[module]
if isinstance(fname, dict):
return "SKIPPED"
basename = os.path.basename(fname)
deprecated = False
return added_float < TO_OLD_TO_BE_NOTABLE
# ignore files with extensions
if not basename.endswith(".py"):
return
elif module.startswith("_"):
if os.path.islink(fname):
return # ignore, its an alias
deprecated = True
module = module.replace("_", "", 1)
print("rendering: %s" % module)
def process_modules(module_map, templates, outputname, output_dir, ansible_version):
for module in module_map:
print("rendering: %s" % module)
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples, returndocs, metadata = plugin_docs.get_docstring(fname, verbose=options.verbose)
fname = module_map[module]['path']
# crash if module is missing documentation and not explicitly hidden from docs index
if doc is None:
sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
# crash if module is missing documentation and not explicitly hidden from docs index
if module_map[module]['doc'] is None:
sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
if deprecated and 'deprecated' not in doc:
sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
# Going to reference this heavily so make a short name to reference it by
doc = module_map[module]['doc']
if module in aliases:
doc['aliases'] = aliases[module]
if module_map[module]['deprecated'] and 'deprecated' not in doc:
sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
all_keys = []
if 'version_added' not in doc:
sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
if 'version_added' not in doc:
sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
#
# The present template gets everything from doc so we spend most of this
# function moving data into doc for the template to reference
#
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
added = doc['version_added']
# don't show version added information if it's too old to be called out
if too_old(added):
del doc['version_added']
if 'options' in doc and doc['options']:
for (k, v) in iteritems(doc['options']):
# don't show version added information if it's too old to be called out
if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
del doc['options'][k]['version_added']
if 'description' not in doc['options'][k]:
raise AnsibleError("Missing required description for option %s in %s " % (k, module))
required_value = doc['options'][k].get('required', False)
if not isinstance(required_value, bool):
raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
if not isinstance(doc['options'][k]['description'], list):
doc['options'][k]['description'] = [doc['options'][k]['description']]
all_keys.append(k)
all_keys = sorted(all_keys)
doc['option_keys'] = all_keys
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples # plain text
doc['metadata'] = metadata
if returndocs:
try:
doc['returndocs'] = yaml.safe_load(returndocs)
except:
print("could not load yaml: %s" % returndocs)
raise
else:
doc['returndocs'] = None
if module_map[module]['aliases']:
doc['aliases'] = module_map[module]['aliases']
# here is where we build the table of contents...
try:
text = template.render(doc)
except Exception as e:
raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e)))
write_data(text, options, outputname, module)
return doc['short_description']
def print_modules(module, category_file, deprecated, options, env, template, outputname, module_map, aliases):
modstring = module
if modstring.startswith('_'):
modstring = module[1:]
modname = modstring
if module in deprecated:
modstring = to_bytes(modstring) + DEPRECATED
category_file.write(b" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
def process_category(category, categories, options, env, template, outputname):
# FIXME:
# We no longer conceptually deal with a mapping of category names to
# modules to file paths. Instead we want several different records:
# (1) Mapping of module names to file paths (what's presently used
# as categories['all']
# (2) Mapping of category names to lists of module names (what you'd
# presently get from categories[category_name][subcategory_name].keys()
# (3) aliases (what's presently in categories['_aliases']
#
# list_modules() now returns those. Need to refactor this function and
# main to work with them.
module_map = categories[category]
module_info = categories['all']
aliases = {}
if '_aliases' in categories:
aliases = categories['_aliases']
category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
category_file = open(category_file_path, "wb")
print("*** recording category %s in %s ***" % (category, category_file_path))
# start a new category file
category = category.replace("_", " ")
category = category.title()
modules = []
deprecated = []
for module in module_map.keys():
if isinstance(module_map[module], dict):
for mod in (m for m in module_map[module].keys() if m in module_info):
if mod.startswith("_"):
deprecated.append(mod)
# don't show version added information if it's too old to be called out
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
if module not in module_info:
continue
if module.startswith("_"):
deprecated.append(module)
modules.append(module)
modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
category_header = b"%s Modules" % (to_bytes(category.title()))
underscores = b"`" * len(category_header)
category_file.write(b"""\
%s
%s
.. toctree:: :maxdepth: 1
""" % (category_header, underscores))
sections = []
for module in modules:
if module in module_map and isinstance(module_map[module], dict):
sections.append(module)
continue
added = doc['version_added']
# Strip old version_added for the module
if too_old(added):
del doc['version_added']
option_names = []
if 'options' in doc and doc['options']:
for (k, v) in iteritems(doc['options']):
# Error out if there's no description
if 'description' not in doc['options'][k]:
raise AnsibleError("Missing required description for option %s in %s " % (k, module))
# Error out if required isn't a boolean (people have been putting
# information on when something is required in here. Those need
# to go in the description instead).
required_value = doc['options'][k].get('required', False)
if not isinstance(required_value, bool):
raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
# Strip old version_added information for options
if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
del doc['options'][k]['version_added']
# Make sure description is a list of lines for later formatting
if not isinstance(doc['options'][k]['description'], list):
doc['options'][k]['description'] = [doc['options'][k]['description']]
option_names.append(k)
option_names.sort()
doc['option_keys'] = option_names
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = ansible_version
doc['plainexamples'] = module_map[module]['examples'] # plain text
doc['metadata'] = module_map[module]['metadata']
if module_map[module]['returndocs']:
try:
doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
except:
print("could not load yaml: %s" % module_map[module]['returndocs'])
raise
else:
print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
sections.sort()
for section in sections:
category_file.write(b"\n%s\n%s\n\n" % (to_bytes(section.replace("_", " ").title()), b'-' * len(section)))
category_file.write(b".. toctree:: :maxdepth: 1\n\n")
section_modules = list(module_map[section].keys())
section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
# for module in module_map[section]:
for module in (m for m in section_modules if m in module_info):
print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
category_file.write(b"""\n\n
.. note::
- %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
The module documentation details page may explain more about this rationale.
""" % DEPRECATED)
category_file.close()
doc['returndocs'] = None
text = templates['plugin'].render(doc)
write_data(text, output_dir, outputname, module)
def process_categories(mod_info, categories, templates, output_dir, output_name):
for category in sorted(categories.keys()):
module_map = categories[category]
category_filename = output_name % category
print("*** recording category %s in %s ***" % (category, category_filename))
# start a new category file
category = category.replace("_", " ")
category = category.title()
subcategories = dict((k, v) for k, v in module_map.items() if k != '_modules')
template_data = {'title': category,
'category': module_map,
'subcategories': subcategories,
'module_info': mod_info,
}
text = templates['list_of_CATEGORY_modules'].render(template_data)
write_data(text, output_dir, category_filename)
def process_support_levels(mod_info, templates, output_dir):
supported_by = {'Ansible Core Team': {'slug': 'core_supported',
'modules': [],
'output': 'core_maintained.rst'},
'Ansible Network Team': {'slug': 'network_supported',
'modules': [],
'output': 'network_maintained.rst'},
'Ansible Partners': {'slug': 'partner_supported',
'modules': [],
'output': 'partner_maintained.rst'},
'Ansible Community': {'slug': 'community_supported',
'modules': [],
'output': 'community_maintained.rst'},
}
# Separate the modules by support_level
for module, info in mod_info.items():
if info['metadata']['supported_by'] == 'core':
supported_by['Ansible Core Team']['modules'].append(module)
elif info['metadata']['supported_by'] == 'network':
supported_by['Ansible Network Team']['modules'].append(module)
elif info['metadata']['supported_by'] == 'certified':
supported_by['Ansible Partners']['modules'].append(module)
elif info['metadata']['supported_by'] == 'community':
supported_by['Ansible Community']['modules'].append(module)
else:
raise AnsibleError('Unknown supported_by value: %s' % info['metadata']['supported_by'])
# TODO: end a new category file
# Render the module lists
for maintainers, data in supported_by.items():
template_data = {'maintainers': maintainers,
'modules': data['modules'],
'slug': data['slug'],
'module_info': mod_info,
}
text = templates['support_list'].render(template_data)
write_data(text, output_dir, data['output'])
def validate_options(options):
@ -435,43 +463,34 @@ def main():
(options, args) = p.parse_args()
validate_options(options)
env, template, outputname = jinja2_environment(options.template_dir, options.type)
templates, outputname = jinja2_environment(options.template_dir, options.type)
# Convert passed-in limit_to_modules to None or list of modules.
if options.limit_to_modules is not None:
options.limit_to_modules = [s.lower() for s in options.limit_to_modules.split(",")]
mod_info, categories, aliases = list_modules(options.module_dir, limit_to_modules=options.limit_to_modules)
categories['all'] = mod_info
categories['_aliases'] = aliases
category_names = [c for c in categories.keys() if not c.startswith('_')]
category_names.sort()
mod_info, categories = get_module_info(options.module_dir, limit_to_modules=options.limit_to_modules,
verbose=options.verbose)
categories['all'] = {'_modules': mod_info.keys()}
# Transform the data
if options.type == 'rst':
for record in mod_info.values():
record['doc']['short_description'] = rst_ify(record['doc']['short_description'])
# Write master category list
category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
with open(category_list_path, "wb") as category_list_file:
category_list_file.write(b"Module Index\n")
category_list_file.write(b"============\n")
category_list_file.write(b"\n\n")
category_list_file.write(b".. toctree::\n")
category_list_file.write(b" :maxdepth: 1\n\n")
for category in category_names:
category_list_file.write(b" list_of_%s_modules\n" % to_bytes(category))
# Import all the docs into memory
module_map = mod_info.copy()
for modname in module_map:
result = process_module(modname, options, env, template, outputname, module_map, aliases)
if result == 'SKIPPED':
del categories['all'][modname]
else:
categories['all'][modname] = (categories['all'][modname], result)
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
write_data(category_list_text, options.output_dir, 'modules_by_category.rst')
# Render all the individual module pages
process_modules(mod_info, templates, outputname, options.output_dir, options.ansible_version)
# Render all the categories for modules
process_categories(mod_info, categories, templates, options.output_dir, "list_of_%s_modules.rst")
# Render all the docs to rst via category pages
for category in category_names:
process_category(category, categories, options, env, template, outputname)
# Render all the categories for modules
process_support_levels(mod_info, templates, options.output_dir)
if __name__ == '__main__':

13
docs/docsite/rst/modules_support.rst
Unescape Escape View File

@ -21,12 +21,18 @@ Core
These are modules that the Ansible Core Team maintains and will always ship with Ansible itself.
They will also receive slightly higher priority for all requests. Non-core modules are still fully usable.
.. seealso::
List of :ref:`Core Supported<core_supported>` modules
Network
```````
These modules are supported by the Ansible Network Team in a relationship
similar to how the Ansible Core Team maintains the Core modules.
.. seealso::
List of :ref:`Network Supported<network_supported>` modules
Certified
`````````
@ -37,6 +43,9 @@ Also, it is strongly recommended (but not presently required) for these types of
These modules are currently shipped with Ansible, but might be shipped separately in the future.
.. seealso::
List of :ref:`Certified<partner_supported>` modules
Community
`````````
These modules **are not** supported by Core Committers or by companies/partners associated to the module. They are maintained by the community.
@ -45,6 +54,10 @@ They are still fully usable, but the response rate to issues is purely up to the
These modules are currently shipped with Ansible, but will most likely be shipped separately in the future.
.. seealso::
List of Core Supported modules
List of :ref:`Community Supported<community_supported>` modules
.. seealso::

27
docs/templates/list_of_CATEGORY_modules.rst.j2 vendored
Unescape Escape View File

@ -0,0 +1,27 @@
@{ title }@ Modules
@{ '`' * title | length }@````````
.. toctree:: :maxdepth: 1
{% if category['_modules'] %}
{% for module in category['_modules'] | sort %}
@{ module }@{% if module_info[module]['deprecated'] %} **(D)**{% endif%} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
{% endfor %}
{% endif %}
{% for name, info in subcategories.items() | sort %}
@{ name.title() }@
@{ '-' * name | length }@
.. toctree:: :maxdepth: 1
{% for module in info['_modules'] | sort %}
@{ module }@{% if module_info[module]['deprecated'] %}**(D)**{% endif%} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
{% endfor %}
{% endfor %}
.. note::
- **(D)**: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
The module documentation details page may explain more about this rationale.

9
docs/templates/modules_by_category.rst.j2 vendored
Unescape Escape View File

@ -0,0 +1,9 @@
Module Index
============
.. toctree:: :maxdepth: 1
{% for name in categories %}
list_of_@{ name }@_modules
{% endfor %}

15
docs/templates/modules_by_support.rst.j2 vendored
Unescape Escape View File

@ -0,0 +1,15 @@
.. _@{ slug }@:
Modules Maintained by the @{ maintainers }@
``````````````````````````@{ '`' * maintainers | length }@
.. toctree:: :maxdepth: 1
{% for module in modules | sort %}
@{ module }@{% if module_info[module]['deprecated'] %}**(D)**{% endif %} - @{ module_info[module]['doc']['short_description'] }@ <@{ module }@_module>
{% endfor %}
.. note::
- **(D)**: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
The module documentation details page may explain more about this rationale.
Write Preview
Loading…
Cancel
Save