ansible/hacking/module_formatter.py

#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import glob
import sys
import yaml
import codecs
import json
import ast
import re
import optparse
import time
import datetime
import subprocess
import cgi
from jinja2 import Environment, FileSystemLoader

import ansible.utils
import ansible.utils.module_docs as module_docs

#####################################################################################
# constants and paths

# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.0

# Get parent directory of the directory this script lives in
MODULEDIR=os.path.abspath(os.path.join(
    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules'
))

# The name of the DOCUMENTATION template
EXAMPLE_YAML=os.path.abspath(os.path.join(
    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
))

_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD   = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL    = re.compile(r"U\(([^)]+)\)")
_CONST  = re.compile(r"C\(([^)]+)\)")

DEPRECATED = " (D)"
NOTCORE    = " (E)"
#####################################################################################

def rst_ify(text):
    ''' convert symbols like I(this is in italics) to valid restructured text '''

    t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
    t = _BOLD.sub(r'**' + r"\1" + r"**", t)
    t = _MODULE.sub(r'``' + r"\1" + r"``", t)
    t = _URL.sub(r"\1", t)
    t = _CONST.sub(r'``' + r"\1" + r"``", t)

    return t

#####################################################################################

def html_ify(text):
    ''' convert symbols like I(this is in italics) to valid HTML '''

    t = cgi.escape(text)
    t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
    t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
    t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
    t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
    t = _CONST.sub("<code>" + r"\1" + "</code>", t)

    return t


#####################################################################################

def rst_fmt(text, fmt):
    ''' helper for Jinja2 to do format strings '''

    return fmt % (text)

#####################################################################################

def rst_xline(width, char="="):
    ''' return a restructured text line of a given length '''

    return char * width

#####################################################################################

def write_data(text, options, outputname, module):
    ''' dumps module output to a file or the screen, as requested '''

    if options.output_dir is not None:
        fname = os.path.join(options.output_dir, outputname % module)
        fname = fname.replace(".py","")
        f = open(fname, 'w')
        f.write(text.encode('utf-8'))
        f.close()
    else:
        print text

#####################################################################################


def list_modules(module_dir):
    ''' returns a hash of categories, each category being a hash of module names to file paths '''

    categories = dict(all=dict())
    files = glob.glob("%s/*/*" % module_dir)
    for d in files:
        if os.path.isdir(d):
            files2 = glob.glob("%s/*" % d)
            for f in files2:

                module = os.path.splitext(os.path.basename(f))[0]
                category = os.path.dirname(f).split("/")[-1]

                if not f.endswith(".py") or f.endswith('__init__.py'):
                    # windows powershell modules have documentation stubs in python docstring
                    # format (they are not executed) so skip the ps1 format files
                    continue
                elif module.startswith("_") and os.path.islink(f):   # ignores aliases
                    continue

                if not category in categories:
                    categories[category] = {}
                categories[category][module] = f
                categories['all'][module] = f

    return categories

#####################################################################################

def generate_parser():
    ''' generate an optparse parser '''

    p = optparse.OptionParser(
        version='%prog 1.0',
        usage='usage: %prog [options] arg1 arg2',
        description='Generate module documentation from metadata',
    )

    p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
    p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
    p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
    p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
    p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
    p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
    p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
    p.add_option('-V', action='version', help='Show version number and exit')
    return p

#####################################################################################

def jinja2_environment(template_dir, typ):

    env = Environment(loader=FileSystemLoader(template_dir),
        variable_start_string="@{",
        variable_end_string="}@",
        trim_blocks=True,
    )
    env.globals['xline'] = rst_xline

    if typ == 'rst':
        env.filters['convert_symbols_to_format'] = rst_ify
        env.filters['html_ify'] = html_ify
        env.filters['fmt'] = rst_fmt
        env.filters['xline'] = rst_xline
        template = env.get_template('rst.j2')
        outputname = "%s_module.rst"
    else:
        raise Exception("unknown module format type: %s" % typ)

    return env, template, outputname

#####################################################################################

def process_module(module, options, env, template, outputname, module_map):

    fname = module_map[module]
    basename = os.path.basename(fname)
    deprecated = False

    # ignore files with extensions
    if not basename.endswith(".py"):
        return
    elif module.startswith("_"):
        if os.path.islink(fname):
            return  # ignore, its an alias
        deprecated = True
        module = module.replace("_","",1)

    print "rendering: %s" % module

    # use ansible core library to parse out doc metadata YAML and plaintext examples
    doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)

    # crash if module is missing documentation and not explicitly hidden from docs index
    if doc is None:
        if module in ansible.utils.module_docs.BLACKLIST_MODULES:
            return "SKIPPED"
        else:
            sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
            sys.exit(1)

    if deprecated and 'deprecated' not in doc:
        sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
        sys.exit(1)

    if "/core/" in fname:
        doc['core'] = True
    else:
        doc['core'] = False


    all_keys = []

    if not 'version_added' in doc:
        sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module)
        sys.exit(1)

    added = 0
    if doc['version_added'] == 'historical':
        del doc['version_added']
    else:
        added = doc['version_added']

    # don't show version added information if it's too old to be called out
    if added:
        added_tokens = str(added).split(".")
        added = added_tokens[0] + "." + added_tokens[1]
        added_float = float(added)
        if added and added_float < TO_OLD_TO_BE_NOTABLE:
            del doc['version_added']

    for (k,v) in doc['options'].iteritems():
        all_keys.append(k)

    all_keys = sorted(all_keys)

    doc['option_keys']      = all_keys
    doc['filename']         = fname
    doc['docuri']           = doc['module'].replace('_', '-')
    doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
    doc['ansible_version']  = options.ansible_version
    doc['plainexamples']    = examples  #plain text

    # here is where we build the table of contents...

    text = template.render(doc)
    write_data(text, options, outputname, module)
    return doc['short_description']

#####################################################################################

def process_category(category, categories, options, env, template, outputname):

    module_map = categories[category]

    category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
    category_file = open(category_file_path, "w")
    print "*** recording category %s in %s ***" % (category, category_file_path)

    # TODO: start a new category file

    category = category.replace("_"," ")
    category = category.title()

    modules = []
    deprecated = []
    core = []
    for module in module_map.keys():

        if module.startswith("_"):
            module = module.replace("_","",1)
            deprecated.append(module)
        elif '/core/' in module_map[module]:
            core.append(module)

        modules.append(module)

    modules.sort()

    category_header = "%s Modules" % (category.title())
    underscores = "`" * len(category_header)

    category_file.write("""\
%s
%s

.. toctree:: :maxdepth: 1

""" % (category_header, underscores))

    for module in modules:

        modstring = module
        modname = module
        if module in deprecated:
            modstring = modstring + DEPRECATED
            modname = "_" + module
        elif module not in core:
            modstring = modstring + NOTCORE

        result = process_module(modname, options, env, template, outputname, module_map)

        if result != "SKIPPED":
            category_file.write("  %s - %s <%s_module>\n" % (modstring, result, module))

    category_file.write("""\n\n
.. note::
    - %s: This marks a module as deprecated, kept for backwards compatibility but use is discouraged
    - %s: Denotes that this module is not part of core, it can be found in the extras or some other external repo
""" % (DEPRECATED, NOTCORE))
    category_file.close()

    # TODO: end a new category file

#####################################################################################

def validate_options(options):
    ''' validate option parser options '''

    if not options.module_dir:
        print >>sys.stderr, "--module-dir is required"
        sys.exit(1)
    if not os.path.exists(options.module_dir):
        print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir
        sys.exit(1)
    if not options.template_dir:
        print "--template-dir must be specified"
        sys.exit(1)

#####################################################################################

def main():

    p = generate_parser()

    (options, args) = p.parse_args()
    validate_options(options)

    env, template, outputname = jinja2_environment(options.template_dir, options.type)

    categories = list_modules(options.module_dir)
    last_category = None
    category_names = categories.keys()
    category_names.sort()

    category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
    category_list_file = open(category_list_path, "w")
    category_list_file.write("Module Index\n")
    category_list_file.write("============\n")
    category_list_file.write("\n\n")
    category_list_file.write(".. toctree::\n")
    category_list_file.write("   :maxdepth: 1\n\n")

    for category in category_names:
        category_list_file.write("   list_of_%s_modules\n" % category)
        process_category(category, categories, options, env, template, outputname)

    category_list_file.close()

if __name__ == '__main__':
    main()