mirror of https://github.com/ansible/ansible.git
Move common build code from _build_helpers (#55986)
We have some common code used by several docs scripts. Migrate that into the build-only shared code repository. * Move lib/ansible/utils/_build_helpers.py to the directory for common build code * Migrate docs/bin/dump_config.py to a build-ansible subcommand * Migrate dump_keywords to the build-ansible framework * Make the script more maintainable by using functions and good variable names * Port to Python3 idioms * Fix bug so that private attributes will be undocumented * Move generate_man to a build-ansible subcommand * Port plugin_formatter to a build-ansible subcommand * Rework command_plugins so that docs scripts can target Python-3.4+ and releng-only subcommands can use more recent versions of Python. The architecture is now that command_plugins/* need to be importable on Python-3.4. The init_parsers() method needs to run on Python-3.4. But the main() method can utilize features of more recent Python as long as it fits within those parameters. * Update docs build requirements Port the plugin_formatter to build-ansible frameworkpull/59154/head
parent
65e0f37fc0
commit
019d078a5a
@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from ansible.module_utils._text import to_bytes
|
||||
from ansible.utils._build_helpers import update_file_if_different
|
||||
|
||||
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
|
||||
|
||||
|
||||
def generate_parser():
|
||||
p = optparse.OptionParser(
|
||||
version='%prog 1.0',
|
||||
usage='usage: %prog [options]',
|
||||
description='Generate module documentation from metadata',
|
||||
)
|
||||
p.add_option("-t", "--template-file", action="store", dest="template_file", default=DEFAULT_TEMPLATE_FILE, help="directory containing Jinja2 templates")
|
||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
|
||||
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
|
||||
|
||||
(options, args) = p.parse_args()
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def fix_description(config_options):
|
||||
'''some descriptions are strings, some are lists. workaround it...'''
|
||||
|
||||
for config_key in config_options:
|
||||
description = config_options[config_key].get('description', [])
|
||||
if isinstance(description, list):
|
||||
desc_list = description
|
||||
else:
|
||||
desc_list = [description]
|
||||
config_options[config_key]['description'] = desc_list
|
||||
return config_options
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
parser = generate_parser()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
output_dir = os.path.abspath(options.output_dir)
|
||||
template_file_full_path = os.path.abspath(options.template_file)
|
||||
template_file = os.path.basename(template_file_full_path)
|
||||
template_dir = os.path.dirname(os.path.abspath(template_file_full_path))
|
||||
|
||||
if options.docs:
|
||||
with open(options.docs) as f:
|
||||
docs = yaml.safe_load(f)
|
||||
else:
|
||||
docs = {}
|
||||
|
||||
config_options = docs
|
||||
config_options = fix_description(config_options)
|
||||
|
||||
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
||||
template = env.get_template(template_file)
|
||||
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
|
||||
temp_vars = {'config_options': config_options}
|
||||
|
||||
data = to_bytes(template.render(temp_vars))
|
||||
update_file_if_different(output_name, data)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv[:]))
|
@ -1,84 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import optparse
|
||||
import re
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
import jinja2
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from ansible.module_utils._text import to_bytes
|
||||
from ansible.playbook import Play
|
||||
from ansible.playbook.block import Block
|
||||
from ansible.playbook.role import Role
|
||||
from ansible.playbook.task import Task
|
||||
from ansible.utils._build_helpers import update_file_if_different
|
||||
|
||||
template_file = 'playbooks_keywords.rst.j2'
|
||||
oblist = {}
|
||||
clist = []
|
||||
class_list = [Play, Role, Block, Task]
|
||||
|
||||
p = optparse.OptionParser(
|
||||
version='%prog 1.0',
|
||||
usage='usage: %prog [options]',
|
||||
description='Generate playbook keyword documentation from code and descriptions',
|
||||
)
|
||||
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="../templates", help="directory containing Jinja2 templates")
|
||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
|
||||
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
|
||||
|
||||
(options, args) = p.parse_args()
|
||||
|
||||
for aclass in class_list:
|
||||
aobj = aclass()
|
||||
name = type(aobj).__name__
|
||||
|
||||
if options.docs:
|
||||
with open(options.docs) as f:
|
||||
docs = yaml.safe_load(f)
|
||||
else:
|
||||
docs = {}
|
||||
|
||||
# build ordered list to loop over and dict with attributes
|
||||
clist.append(name)
|
||||
oblist[name] = dict((x, aobj.__dict__['_attributes'][x]) for x in aobj.__dict__['_attributes'] if 'private' not in x or not x.private)
|
||||
|
||||
# pick up docs if they exist
|
||||
for a in oblist[name]:
|
||||
if a in docs:
|
||||
oblist[name][a] = docs[a]
|
||||
else:
|
||||
# check if there is an alias, otherwise undocumented
|
||||
alias = getattr(getattr(aobj, '_%s' % a), 'alias', None)
|
||||
if alias and alias in docs:
|
||||
oblist[name][alias] = docs[alias]
|
||||
del oblist[name][a]
|
||||
else:
|
||||
oblist[name][a] = ' UNDOCUMENTED!! '
|
||||
|
||||
# loop is really with_ for users
|
||||
if name == 'Task':
|
||||
oblist[name]['with_<lookup_plugin>'] = 'The same as ``loop`` but magically adds the output of any lookup plugin to generate the item list.'
|
||||
|
||||
# local_action is implicit with action
|
||||
if 'action' in oblist[name]:
|
||||
oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``'
|
||||
|
||||
# remove unusable (used to be private?)
|
||||
for nouse in ('loop_args', 'loop_with'):
|
||||
if nouse in oblist[name]:
|
||||
del oblist[name][nouse]
|
||||
|
||||
env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
|
||||
template = env.get_template(template_file)
|
||||
outputname = options.output_dir + template_file.replace('.j2', '')
|
||||
tempvars = {'oblist': oblist, 'clist': clist}
|
||||
|
||||
keyword_page = template.render(tempvars)
|
||||
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
|
||||
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
|
||||
keyword_page = re.sub(' +\n', '\n', keyword_page)
|
||||
|
||||
update_file_if_different(outputname, to_bytes(keyword_page))
|
@ -0,0 +1,80 @@
|
||||
# coding: utf-8
|
||||
# Copyright: (c) 2019, Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import pathlib
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from ansible.module_utils._text import to_bytes
|
||||
|
||||
# Pylint doesn't understand Python3 namespace modules.
|
||||
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||
|
||||
|
||||
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
|
||||
DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
|
||||
|
||||
|
||||
def fix_description(config_options):
|
||||
'''some descriptions are strings, some are lists. workaround it...'''
|
||||
|
||||
for config_key in config_options:
|
||||
description = config_options[config_key].get('description', [])
|
||||
if isinstance(description, list):
|
||||
desc_list = description
|
||||
else:
|
||||
desc_list = [description]
|
||||
config_options[config_key]['description'] = desc_list
|
||||
return config_options
|
||||
|
||||
|
||||
class DocumentConfig(Command):
|
||||
name = 'document-config'
|
||||
|
||||
@classmethod
|
||||
def init_parser(cls, add_parser):
|
||||
parser = add_parser(cls.name, description='Generate module documentation from metadata')
|
||||
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
|
||||
default=DEFAULT_TEMPLATE_FILE,
|
||||
help="Jinja2 template to use for the config")
|
||||
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
|
||||
default=DEFAULT_TEMPLATE_DIR,
|
||||
help="directory containing Jinja2 templates")
|
||||
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
|
||||
help="Output directory for rst files")
|
||||
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
|
||||
help="Source for attribute docs")
|
||||
|
||||
@staticmethod
|
||||
def main(args):
|
||||
output_dir = os.path.abspath(args.output_dir)
|
||||
template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
|
||||
template_file = os.path.basename(template_file_full_path)
|
||||
template_dir = os.path.dirname(template_file_full_path)
|
||||
|
||||
if args.docs:
|
||||
with open(args.docs) as f:
|
||||
docs = yaml.safe_load(f)
|
||||
else:
|
||||
docs = {}
|
||||
|
||||
config_options = docs
|
||||
config_options = fix_description(config_options)
|
||||
|
||||
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
||||
template = env.get_template(template_file)
|
||||
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
|
||||
temp_vars = {'config_options': config_options}
|
||||
|
||||
data = to_bytes(template.render(temp_vars))
|
||||
update_file_if_different(output_name, data)
|
||||
|
||||
return 0
|
@ -0,0 +1,125 @@
|
||||
# coding: utf-8
|
||||
# Copyright: (c) 2019, Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import importlib
|
||||
import os.path
|
||||
import pathlib
|
||||
import re
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
import jinja2
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from ansible.module_utils._text import to_bytes
|
||||
|
||||
# Pylint doesn't understand Python3 namespace modules.
|
||||
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||
|
||||
|
||||
DEFAULT_TEMPLATE_DIR = str(pathlib.Path(__file__).resolve().parents[4] / 'docs/templates')
|
||||
TEMPLATE_FILE = 'playbooks_keywords.rst.j2'
|
||||
PLAYBOOK_CLASS_NAMES = ['Play', 'Role', 'Block', 'Task']
|
||||
|
||||
|
||||
def load_definitions(keyword_definitions_file):
|
||||
docs = {}
|
||||
with open(keyword_definitions_file) as f:
|
||||
docs = yaml.safe_load(f)
|
||||
|
||||
return docs
|
||||
|
||||
|
||||
def extract_keywords(keyword_definitions):
|
||||
pb_keywords = {}
|
||||
for pb_class_name in PLAYBOOK_CLASS_NAMES:
|
||||
if pb_class_name == 'Play':
|
||||
module_name = 'ansible.playbook'
|
||||
else:
|
||||
module_name = 'ansible.playbook.{0}'.format(pb_class_name.lower())
|
||||
module = importlib.import_module(module_name)
|
||||
playbook_class = getattr(module, pb_class_name, None)
|
||||
if playbook_class is None:
|
||||
raise ImportError("We weren't able to import the module {0}".format(module_name))
|
||||
|
||||
# Maintain order of the actual class names for our output
|
||||
# Build up a mapping of playbook classes to the attributes that they hold
|
||||
pb_keywords[pb_class_name] = {k: v for (k, v) in playbook_class._valid_attrs.items()
|
||||
# Filter private attributes as they're not usable in playbooks
|
||||
if not v.private}
|
||||
|
||||
# pick up definitions if they exist
|
||||
for keyword in tuple(pb_keywords[pb_class_name]):
|
||||
if keyword in keyword_definitions:
|
||||
pb_keywords[pb_class_name][keyword] = keyword_definitions[keyword]
|
||||
else:
|
||||
# check if there is an alias, otherwise undocumented
|
||||
alias = getattr(getattr(playbook_class, '_%s' % keyword), 'alias', None)
|
||||
if alias and alias in keyword_definitions:
|
||||
pb_keywords[pb_class_name][alias] = keyword_definitions[alias]
|
||||
del pb_keywords[pb_class_name][keyword]
|
||||
else:
|
||||
pb_keywords[pb_class_name][keyword] = ' UNDOCUMENTED!! '
|
||||
|
||||
# loop is really with_ for users
|
||||
if pb_class_name == 'Task':
|
||||
pb_keywords[pb_class_name]['with_<lookup_plugin>'] = (
|
||||
'The same as ``loop`` but magically adds the output of any lookup plugin to'
|
||||
' generate the item list.')
|
||||
|
||||
# local_action is implicit with action
|
||||
if 'action' in pb_keywords[pb_class_name]:
|
||||
pb_keywords[pb_class_name]['local_action'] = ('Same as action but also implies'
|
||||
' ``delegate_to: localhost``')
|
||||
|
||||
return pb_keywords
|
||||
|
||||
|
||||
def generate_page(pb_keywords, template_dir):
|
||||
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
||||
template = env.get_template(TEMPLATE_FILE)
|
||||
tempvars = {'pb_keywords': pb_keywords, 'playbook_class_names': PLAYBOOK_CLASS_NAMES}
|
||||
|
||||
keyword_page = template.render(tempvars)
|
||||
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
|
||||
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
|
||||
keyword_page = re.sub(' +\n', '\n', keyword_page)
|
||||
|
||||
return keyword_page
|
||||
|
||||
|
||||
class DocumentKeywords(Command):
|
||||
name = 'document-keywords'
|
||||
|
||||
@classmethod
|
||||
def init_parser(cls, add_parser):
|
||||
parser = add_parser(cls.name, description='Generate playbook keyword documentation from'
|
||||
' code and descriptions')
|
||||
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
|
||||
default=DEFAULT_TEMPLATE_DIR,
|
||||
help="directory containing Jinja2 templates")
|
||||
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
|
||||
default='/tmp/', help="Output directory for rst files")
|
||||
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
|
||||
help="Source for attribute docs")
|
||||
|
||||
@staticmethod
|
||||
def main(args):
|
||||
if not args.docs:
|
||||
print('Definitions for keywords must be specified via `--docs-source FILENAME`')
|
||||
return 1
|
||||
|
||||
keyword_definitions = load_definitions(args.docs)
|
||||
pb_keywords = extract_keywords(keyword_definitions)
|
||||
|
||||
keyword_page = generate_page(pb_keywords, args.template_dir)
|
||||
outputname = os.path.join(args.output_dir, TEMPLATE_FILE.replace('.j2', ''))
|
||||
update_file_if_different(outputname, to_bytes(keyword_page))
|
||||
|
||||
return 0
|
@ -0,0 +1,12 @@
|
||||
# coding: utf-8
|
||||
# Copyright: (c) 2019, Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
|
||||
class DependencyError(Exception):
|
||||
"""Used when a dependency is unmet"""
|
||||
pass
|
@ -1,3 +1,11 @@
|
||||
# The following are only run by release engineers who can be asked to have newer Python3 on their systems
|
||||
hacking/build_library/build_ansible/command_plugins/porting_guide.py
|
||||
hacking/build_library/build_ansible/command_plugins/release_announcement.py
|
||||
|
||||
# The following are used to build docs. Since we explicitly say that the controller won't run on
|
||||
# Python-2.6 (docs are built controller-side) and EPEL-6, the only LTS platform with Python-2.6,
|
||||
# doesn't have a new enough sphinx to build docs, do not test these under Python-2.6
|
||||
hacking/build_library/build_ansible/command_plugins/dump_config.py
|
||||
hacking/build_library/build_ansible/command_plugins/dump_keywords.py
|
||||
hacking/build_library/build_ansible/command_plugins/generate_man.py
|
||||
hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
|
||||
|
Loading…
Reference in New Issue