archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP on refactoring the module formatter code that we use to build the doc site with.

Browse Source
pull/5421/merge
Michael DeHaan 12 years ago
parent 31d0060de8
commit fe2d00d9d3
6 changed files with 162 additions and 450 deletions
Show Stats Download Patch File Download Diff File

10
Makefile
Unescape Escape View File

@ -64,9 +64,6 @@ all: clean python
tests:
PYTHONPATH=./lib ANSIBLE_LIBRARY=./library $(NOSETESTS) -d -v
# To force a rebuild of the docs run 'touch VERSION && make docs'
docs: $(MANPAGES) modulepages
authors:
sh hacking/authors.sh
@ -172,11 +169,6 @@ deb: debian
# for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory
modulepages:
PYTHONPATH=./lib $(PYTHON) hacking/module_formatter.py -A $(VERSION) -t man -o docs/man/man3/ --module-dir=library --template-dir=hacking/templates # --verbose
# because this requires Sphinx it is not run as part of every build, those building the RPM and so on can ignore this
webdocs:
webdocs: $(MANPAGES)
(cd docsite/; make docs)

450
hacking/module_formatter.py
Unescape Escape View File

@ -24,16 +24,20 @@ import yaml
import codecs
import json
import ast
from jinja2 import Environment, FileSystemLoader
import re
import optparse
import time
import datetime
import subprocess
import cgi
from jinja2 import Environment, FileSystemLoader
import ansible.utils
import ansible.utils.module_docs as module_docs
#####################################################################################
# constants and paths
# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.0
@ -48,65 +52,16 @@ EXAMPLE_YAML=os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
))
# There is a better way of doing this!
# TODO: somebody add U(text, http://foo.bar/) as described by Tim in #991
_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL = re.compile(r"U\(([^)]+)\)")
_CONST = re.compile(r"C\(([^)]+)\)")
def latex_ify(text):
t = _ITALIC.sub("\\I{" + r"\1" + "}", text)
t = _BOLD.sub("\\B{" + r"\1" + "}", t)
t = _MODULE.sub("\\M{" + r"\1" + "}", t)
t = _URL.sub("\\url{" + r"\1" + "}", t)
t = _CONST.sub("\\C{" + r"\1" + "}", t)
return t
def html_ify(text):
#print "DEBUG: text=%s" % text
t = cgi.escape(text)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def json_ify(text):
t = _ITALIC.sub("<em>" + r"\1" + "</em>", text)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def js_ify(text):
return text
def man_ify(text):
t = _ITALIC.sub(r'\\fI' + r"\1" + r"\\fR", text)
t = _BOLD.sub(r'\\fB' + r"\1" + r"\\fR", t)
t = _MODULE.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _URL.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _CONST.sub(r'\\fC' + r"\1" + r"\\fR", t)
return t
#####################################################################################
def rst_ify(text):
''' convert symbols like I(this is in italics) to valid restructured text '''
t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
t = _BOLD.sub(r'**' + r"\1" + r"**", t)
@ -116,31 +71,40 @@ def rst_ify(text):
return t
_MARKDOWN = re.compile(r"[*_`]")
#####################################################################################
def markdown_ify(text):
def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML '''
t = cgi.escape(text)
t = _MARKDOWN.sub(r"\\\g<0>", t)
t = _ITALIC.sub("_" + r"\1" + "_", t)
t = _BOLD.sub("**" + r"\1" + "**", t)
t = _MODULE.sub("*" + r"\1" + "*", t)
t = _URL.sub("[" + r"\1" + "](" + r"\1" + ")", t)
t = _CONST.sub("`" + r"\1" + "`", t)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
# Helper for Jinja2 (format() doesn't work here...)
#####################################################################################
def rst_fmt(text, fmt):
''' helper for Jinja2 to do format strings '''
return fmt % (text)
#####################################################################################
def rst_xline(width, char="="):
''' return a restructured text line of a given length '''
return char * width
def load_examples_section(text):
return text.split('***BREAK***')
#####################################################################################
def return_data(text, options, outputname, module):
''' dumps module output to a file or the screen, as requested '''
if options.output_dir is not None:
f = open(os.path.join(options.output_dir, outputname % module), 'w')
f.write(text.encode('utf-8'))
@ -148,15 +112,29 @@ def return_data(text, options, outputname, module):
else:
print text
#####################################################################################
def boilerplate():
''' prints the boilerplate for module docs '''
if not os.path.exists(EXAMPLE_YAML):
print >>sys.stderr, "Missing example boiler plate: %s" % EXAMPLE_YAML
print "DOCUMENTATION = '''"
print file(EXAMPLE_YAML).read()
print "'''"
print ""
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
#####################################################################################
def list_modules(module_dir):
''' returns a hash of categories, each category being a hash of module names to file paths '''
categories = {}
files = glob.glob("%s/*" % module_dir)
for d in files:
@ -171,271 +149,165 @@ def list_modules(module_dir):
categories[category][module] = f
return categories
def main():
#####################################################################################
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2',
description='Convert Ansible module DOCUMENTATION strings to other formats',
description='Generate module documentation from metadata',
)
p.add_option("-A", "--ansible-version",
action="store",
dest="ansible_version",
default="unknown",
help="Ansible version number")
p.add_option("-M", "--module-dir",
action="store",
dest="module_dir",
default=MODULEDIR,
help="Ansible modules/ directory")
p.add_option("-T", "--template-dir",
action="store",
dest="template_dir",
default="hacking/templates",
help="directory containing Jinja2 templates")
p.add_option("-t", "--type",
action='store',
dest='type',
choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'],
default='latex',
help="Output type")
p.add_option("-m", "--module",
action='append',
default=[],
dest='module_list',
help="Add modules to process in module_dir")
p.add_option("-v", "--verbose",
action='store_true',
default=False,
help="Verbose")
p.add_option("-o", "--output-dir",
action="store",
dest="output_dir",
default=None,
help="Output directory for module files")
p.add_option("-I", "--includes-file",
action="store",
dest="includes_file",
default=None,
help="Create a file containing list of processed modules")
p.add_option("-G", "--generate",
action="store_true",
dest="do_boilerplate",
default=False,
help="generate boilerplate DOCUMENTATION to stdout")
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
p.add_option("-t", "--type", action='store', dest='type', choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'], default='latex', help="Document type")
p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
p.add_option("-G", "--generate", action="store_true", dest="do_boilerplate", default=False, help="generate boilerplate docs to stdout")
p.add_option('-V', action='version', help='Show version number and exit')
return p
(options, args) = p.parse_args()
# print "M: %s" % options.module_dir
# print "t: %s" % options.type
# print "m: %s" % options.module_list
# print "v: %s" % options.verbose
if options.do_boilerplate:
boilerplate()
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
sys.exit(0)
#####################################################################################
if not options.module_dir:
print "Need module_dir"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "Module directory does not exist: %s" % options.module_dir
sys.exit(1)
def jinja2_environment(template_dir, typ):
if not options.template_dir:
print "Need template_dir"
sys.exit(1)
env = Environment(loader=FileSystemLoader(options.template_dir),
env = Environment(loader=FileSystemLoader(template_dir),
variable_start_string="@{",
variable_end_string="}@",
trim_blocks=True,
)
env.globals['xline'] = rst_xline
if options.type == 'latex':
env.filters['convert_symbols_to_format'] = latex_ify
template = env.get_template('latex.j2')
outputname = "%s.tex"
includecmt = ""
includefmt = "%s\n"
if options.type == 'html':
env.filters['convert_symbols_to_format'] = html_ify
template = env.get_template('html.j2')
outputname = "%s.html"
includecmt = ""
includefmt = ""
if options.type == 'man':
env.filters['convert_symbols_to_format'] = man_ify
template = env.get_template('man.j2')
outputname = "ansible.%s.3"
includecmt = ""
includefmt = ""
if options.type == 'rst':
if typ == 'rst':
env.filters['convert_symbols_to_format'] = rst_ify
env.filters['html_ify'] = html_ify
env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline
template = env.get_template('rst.j2')
outputname = "%s.rst"
includecmt = ".. Generated by module_formatter\n"
includefmt = ".. include:: modules/%s.rst\n"
if options.type == 'json':
env.filters['convert_symbols_to_format'] = json_ify
outputname = "%s.json"
includecmt = ""
includefmt = ""
if options.type == 'js':
env.filters['convert_symbols_to_format'] = js_ify
template = env.get_template('js.j2')
outputname = "%s.js"
if options.type == 'markdown':
env.filters['convert_symbols_to_format'] = markdown_ify
env.filters['html_ify'] = html_ify
template = env.get_template('markdown.j2')
outputname = "%s.md"
includecmt = ""
includefmt = ""
else:
raise Exception("unknown module format type: %s" % typ)
if options.includes_file is not None and includefmt != "":
incfile = open(options.includes_file, "w")
incfile.write(includecmt)
return env, template, outputname
# Temporary variable required to genrate aggregated content in 'js' format.
js_data = []
#####################################################################################
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
for category in category_names:
module_map = categories[category]
category = category.replace("_"," ")
category = category.title()
def process_module(module, options, env, template, outputname, module_map):
print "rendering: %s" % module
fname = module_map[module]
# ignore files with extensions
if os.path.basename(fname).find(".") != -1:
return
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)
# crash if module is missing documentation and not explicitly hidden from docs index
if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES:
sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
sys.exit(1)
if doc is None:
return
all_keys = []
modules = module_map.keys()
modules.sort()
if not 'version_added' in doc:
sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module)
sys.exit(1)
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
added = doc['version_added']
# don't show version added information if it's too old to be called out
if added:
added_tokens = str(added).split(".")
added = added_tokens[0] + "." + added_tokens[1]
added_float = float(added)
if added and added_float < TO_OLD_TO_BE_NOTABLE:
del doc['version_added']
for (k,v) in doc['options'].iteritems():
all_keys.append(k)
all_keys = sorted(all_keys)
doc['option_keys'] = all_keys
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples #plain text
# here is where we build the table of contents...
text = template.render(doc)
return_data(text, options, outputname, module)
for module in modules:
#####################################################################################
print "rendering: %s" % module
def process_category(category, categories, options, env, template, outputname):
fname = module_map[module]
module_map = categories[category]
if len(options.module_list):
if not module in options.module_list:
continue
# TODO: start a new category file
# fname = os.path.join(options.module_dir, module)
category = category.replace("_"," ")
category = category.title()
modules = module_map.keys()
modules.sort()
for module in modules:
process_module(module, options, env, template, outputname, module_map)
# TODO: end a new category file
#####################################################################################
def validate_options(options):
''' validate option parser options '''
if options.do_boilerplate:
boilerplate()
sys.exit(0)
extra = os.path.join("inc", "%s.tex" % module)
if not options.module_dir:
print >>sys.stderr, "--module-dir is required"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir
sys.exit(1)
if not options.template_dir:
print "--template-dir must be specified"
sys.exit(1)
# probably could just throw out everything with extensions
if fname.endswith(".swp") or fname.endswith(".orig") or fname.endswith(".rej"):
continue
#####################################################################################
# print " processing module source ---> %s" % fname
def main():
if options.type == 'js':
if fname.endswith(".json"):
f = open(fname)
j = json.load(f)
f.close()
js_data.append(j)
continue
p = generate_parser()
doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)
(options, args) = p.parse_args()
validate_options(options)
if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES:
print " while processing module source ---> %s" % fname
sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s ***\n" % module)
#sys.exit(1)
env, template, outputname = jinja2_environment(options.template_dir, options.type)
if not doc is None:
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
all_keys = []
if not 'version_added' in doc:
sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module)
sys.exit(1)
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
added = doc['version_added']
# don't show version added information if it's too old to be called out
if added:
added_tokens = str(added).split(".")
added = added_tokens[0] + "." + added_tokens[1]
added_float = float(added)
if added and added_float < TO_OLD_TO_BE_NOTABLE:
del doc['version_added']
for (k,v) in doc['options'].iteritems():
all_keys.append(k)
all_keys = sorted(all_keys)
doc['option_keys'] = all_keys
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples #plain text
# BOOKMARK: here is where we build the table of contents...
if options.includes_file is not None and includefmt != "":
if last_category != category:
incfile.write("\n\n")
incfile.write(category)
incfile.write("\n")
incfile.write('`' * len(category))
incfile.write("\n\n")
last_category = category
incfile.write(includefmt % module)
if options.verbose:
print json.dumps(doc, indent=4)
if options.type == 'latex':
if os.path.exists(extra):
f = open(extra)
extradata = f.read()
f.close()
doc['extradata'] = extradata
if options.type == 'json':
text = json.dumps(doc, indent=2)
else:
text = template.render(doc)
return_data(text, options, outputname, module)
if options.type == 'js':
docs = {}
docs['json'] = json.dumps(js_data, indent=2)
text = template.render(docs)
return_data(text, options, outputname, 'modules')
for category in category_names:
process_category(category, categories, options, env, template, outputname)
if __name__ == '__main__':
main()

7
hacking/templates/html.j2
Unescape Escape View File

@ -1,7 +0,0 @@
<!-- @{ module | upper }@ -->
<h2>@{module}@</h2>
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}

5
hacking/templates/js.j2
Unescape Escape View File

@ -1,5 +0,0 @@
function AnsibleModules($scope) {
$scope.modules = @{ json }@;
$scope.orderProp = "module";
}

76
hacking/templates/latex.j2
Unescape Escape View File

@ -1,76 +0,0 @@
{# -------------------------------------------------------------------
template for module_formatter.py for LaTeX output (Ansible Booklet)
by Jan-Piet Mens.
Note: nodes & code examples are omitted on purpose.
-------------------------------------------------------------------- #}
%--- @{ module | upper }@ ---- from @{ filename }@ ---
%: -- module header
\mods{@{module}@}{@{docuri}@}{
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor -%}
{% if version_added is defined -%}
(\I{* new in version @{ version_added }@})
{% endif -%}
}
%: -- module options
{% if options %}
\begin{xlist}{abcdefghijklmno}
{% for (opt,v) in options.iteritems() %}
{% if v['required'] %}
\item[\man\,\C{@{ opt }@}]
{% else %}
\item[\opt\,\C{@{ opt }@}]
{% endif %}
{# -------- option description ----------#}
{% for desc in v.description %}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if v['choices'] %}
\B{Choices}:\,
{% for choice in v['choices'] %}\C{@{ choice }@}{% if not loop.last %},{% else %}.{% endif %}
{% endfor %}
{% endif %}
{% if v['default'] %}
(default \C{@{ v['default'] }@})
{% endif %}
{% if v['version_added'] is defined %}
(\I{* version @{ v['version_added'] }@})
{% endif %}
{% endfor %}
\end{xlist}
{% endif %}
{# ---------------------------------------
{% if notes %}
{% for note in notes %}
\I{@{ note | convert_symbols_to_format }@}
{% endfor %}
{% endif %}
----------------------------- #}
{#-------------------------------------------
{% if examples is defined -%}
{% for e in examples %}
\begin{extymeta}
@{ e['code'] }@
\end{extymeta}
{% endfor %}
{% endif %}
----------------------------------- #}
{% if extradata is defined %}
%--- BEGIN-EXTRADATA
\begin{extymeta}
@{ extradata }@
\end{extymeta}
%----- END-EXTRADATA
{% endif %}

64
hacking/templates/markdown.j2
Unescape Escape View File

@ -1,64 +0,0 @@
## @{ module | convert_symbols_to_format }@
{# ------------------------------------------
#
# This is Github-flavored Markdown
#
--------------------------------------------#}
{% if version_added is defined -%}
New in version @{ version_added }@.
{% endif %}
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if options -%}
<table>
<tr>
<th class="head">parameter</th>
<th class="head">required</th>
<th class="head">default</th>
<th class="head">choices</th>
<th class="head">comments</th>
</tr>
{% for (k,v) in options.iteritems() %}
<tr>
<td>@{ k }@</td>
<td>{% if v.get('required', False) %}yes{% else %}no{% endif %}</td>
<td>{% if v['default'] %}@{ v['default'] }@{% endif %}</td>
<td><ul>{% for choice in v.get('choices',[]) -%}<li>@{ choice }@</li>{% endfor -%}</ul></td>
<td>{% for desc in v.description -%}@{ desc | html_ify }@{% endfor -%}{% if v['version_added'] %} (added in Ansible @{v['version_added']}@){% endif %}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if examples or plainexamples %}
#### Examples
{% endif %}
{% for example in examples %}
{% if example['description'] %}
* @{ example['description'] | convert_symbols_to_format }@
{% endif %}
```
@{ example['code'] }@
```
{% endfor %}
{% if plainexamples -%}
```
@{ plainexamples }@
```
{% endif %}
{% if notes %}
#### Notes
{% for note in notes %}
@{ note | convert_symbols_to_format }@
{% endfor %}
{% endif %}
Write Preview
Loading…
Cancel
Save