matrix-spec/scripts/templating/build.py

#!/usr/bin/env python
#
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Batesian: A simple templating system using Jinja.

Architecture
============

                                                         INPUT FILE --------+
+-------+                           +----------+                            |
| units |-+                         | sections |-+                          V
+-------+ |-+ == used to create ==> +----------- | == provides vars to ==> Jinja
  +-------+ |                         +----------+                          |
   +--------+                                                               V
RAW DATA (e.g. json)                 Blobs of text                   OUTPUT FILE

Units
=====
Units are random bits of unprocessed data, e.g. schema JSON files. Anything can
be done to them, from processing it with Jinja to arbitrary python processing.
They are typically dicts.

Sections
========
Sections are strings, typically short segments of RST. They will be dropped in
to the provided input file based on their section key name (template var)
They typically use a combination of templates + units to construct bits of RST.

Input File
==========
The input file is a text file which is passed through Jinja along with the
section keys as template variables.

Processing
==========
- Execute all unit functions to load units into memory and process them.
- Execute all section functions (which can now be done because the units exist)
- Process the input file through Jinja, giving it the sections as template vars.
"""
from batesian import AccessKeyStore

from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta
from argparse import ArgumentParser, FileType
import importlib
import json
import logging
import os
import sys
from textwrap import TextWrapper

from matrix_templates.units import TypeTableRow


def create_from_template(template, sections):
    return template.render(sections)

def check_unaccessed(name, store):
    unaccessed_keys = store.get_unaccessed_set()
    if len(unaccessed_keys) > 0:
        log("Found %s unused %s keys." % (len(unaccessed_keys), name))
        log(unaccessed_keys)

def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
    if out_dir and not os.path.exists(out_dir):
        os.makedirs(out_dir)

    in_mod = importlib.import_module(input_module)

    # add a template filter to produce pretty pretty JSON
    def jsonify(input, indent=None, pre_whitespace=0):
        code = json.dumps(input, indent=indent, sort_keys=True)
        if pre_whitespace:
            code = code.replace("\n", ("\n" +" "*pre_whitespace))

        return code

    def indent_block(input, indent):
        return input.replace("\n", ("\n" + " "*indent))

    def indent(input, indent):
        return " "*indent + input

    def wrap(input, wrap=80, initial_indent=""):
        if len(input) == 0:
            return initial_indent
        # TextWrapper collapses newlines into single spaces; we do our own
        # splitting on newlines to prevent this, so that newlines can actually
        # be intentionally inserted in text.
        input_lines = input.split('\n\n')
        wrapper = TextWrapper(initial_indent=initial_indent, width=wrap)
        output_lines = [wrapper.fill(line) for line in input_lines]

        for i in range(len(output_lines)):
            line = output_lines[i]
            in_bullet = line.startswith("- ")
            if in_bullet:
                output_lines[i] = line.replace("\n", "\n  " + initial_indent)

        return '\n\n'.join(output_lines)

    def fieldwidths(input, keys, defaults=[], default_width=15):
        """
        A template filter to help in the generation of tables.

        Given a list of rows, returns a list giving the maximum length of the
        values in each column.

        :param list[TypeTableRow|dict[str,str]] input:
            a list of rows
        :param list[str] keys: the keys corresponding to the table columns
        :param list[int] defaults: for each column, the default column width.
        :param int default_width: if ``defaults`` is shorter than ``keys``, this
           will be used as a fallback
        """
        def getrowattribute(row, k):
            # the row may be a dict (particularly the title row, which is
            # generated by the template
            if not isinstance(row, TypeTableRow):
                return row[k]
            return getattr(row, k)

        def colwidth(key, default):
            rowwidths = (len(getrowattribute(row, key)) for row in input)
            return reduce(max, rowwidths,
                          default if default is not None else default_width)

        results = map(colwidth, keys, defaults)
        return results

    # make Jinja aware of the templates and filters
    env = Environment(
        loader=FileSystemLoader(in_mod.exports["templates"]),
        undefined=StrictUndefined
    )
    env.filters["jsonify"] = jsonify
    env.filters["indent"] = indent
    env.filters["indent_block"] = indent_block
    env.filters["wrap"] = wrap
    env.filters["fieldwidths"] = fieldwidths

    # load up and parse the lowest single units possible: we don't know or care
    # which spec section will use it, we just need it there in memory for when
    # they want it.
    units = AccessKeyStore(
        existing_data=in_mod.exports["units"](
            debug=verbose,
            substitutions=substitutions,
        ).get_units()
    )

    # use the units to create RST sections
    sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()

    # print out valid section keys if no file supplied
    if not files:
        print "\nValid template variables:"
        for key in sections.keys():
            sec_text = "" if (len(sections[key]) > 75) else (
                "(Value: '%s')" % sections[key]
            )
            sec_info = "%s characters" % len(sections[key])
            if sections[key].count("\n") > 0:
                sec_info += ", %s lines" % sections[key].count("\n")
            print "  %s" % key
            print "      %s %s" % (sec_info, sec_text)
        return

    # check the input files and substitute in sections where required
    for input_filename in files:
        output_filename = os.path.join(out_dir,
                                       os.path.basename(input_filename))
        process_file(env, sections, input_filename, output_filename)

    check_unaccessed("units", units)

def process_file(env, sections, filename, output_filename):
    log("Parsing input template: %s" % filename)

    with open(filename, "r") as file_stream:
        temp_str = file_stream.read().decode("utf-8")

    # do sanity checking on the template to make sure they aren't reffing things
    # which will never be replaced with a section.
    ast = env.parse(temp_str)
    template_vars = meta.find_undeclared_variables(ast)
    unused_vars = [var for var in template_vars if var not in sections]
    if len(unused_vars) > 0:
        raise Exception(
            "You have {{ variables }} which are not found in sections: %s" %
            (unused_vars,)
        )
    # process the template
    temp = Template(temp_str)
    output = create_from_template(temp, sections)

    # Do these substitutions outside of the ordinary templating system because
    # we want them to apply to things like the underlying swagger used to
    # generate the templates, not just the top-level sections.
    for old, new in substitutions.items():
        output = output.replace(old, new)

    with open(output_filename, "w") as f:
        f.write(output.encode("utf-8"))
    log("Output file for: %s" % output_filename)


def log(line):
    print "batesian: %s" % line

if __name__ == '__main__':
    parser = ArgumentParser(
        "Processes a file (typically .rst) through Jinja to replace templated "+
        "areas with section information from the provided input module. For a "+
        "list of possible template variables, add --show-template-vars."
    )
    parser.add_argument(
        "files", nargs="+",
        help="The input files to process. These will be passed through Jinja "+
        "then output under the same name to the output directory."
    )
    parser.add_argument(
        "--input", "-i",
        help="The python module (not file) which contains the sections/units "+
        "classes. This module must have an 'exports' dict which has "+
        "{ 'units': UnitClass, 'sections': SectionClass, "+
        "'templates': 'template/dir' }"
    )
    parser.add_argument(
        "--out-directory", "-o", help="The directory to output the file to."+
        " Default: /out",
        default="out"
    )
    parser.add_argument(
        "--show-template-vars", "-s", action="store_true",
        help="Show a list of all possible variables (sections) you can use in"+
        " the input file."
    )
    parser.add_argument(
        "--verbose", "-v", action="store_true",
        help="Turn on verbose mode."
    )
    parser.add_argument(
        "--substitution", action="append",
        help="Substitutions to apply to the generated output, of form NEEDLE=REPLACEMENT.",
        default=[],
    )
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    if not args.input:
        raise Exception("Missing [i]nput python module.")

    if (args.show_template_vars):
        main(args.input, verbose=args.verbose)
        sys.exit(0)

    substitutions = {}
    for substitution in args.substitution:
        parts = substitution.split("=", 1)
        if len(parts) != 2:
            raise Exception("Invalid substitution")
        substitutions[parts[0]] = parts[1]

    main(
        args.input, files=args.files, out_dir=args.out_directory,
        substitutions=substitutions, verbose=args.verbose
    )