diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f9120551..33bd1fe6 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -34,7 +34,12 @@ def rst2html(i, o): def run_through_template(input): null = open(os.devnull, 'w') subprocess.check_output( - ['python', 'build.py', "-o", "../scripts/tmp", "../scripts/"+input], + [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ], stderr=null, cwd="../templating", ) diff --git a/templating/README.md b/templating/README.md index c1992334..f76051a5 100644 --- a/templating/README.md +++ b/templating/README.md @@ -1,8 +1,6 @@ -This folder contains the templates and templating system for creating the spec. -We use the templating system Jinja2 in Python. This was chosen over other -systems such as Handlebars.js and Templetor because we already have a Python -dependency on the spec build system, and Jinja provides a rich set of template -operations beyond basic control flow. +This folder contains the templates and a home-brewed templating system called +Batesian for creating the spec. Batesian uses the templating system Jinja2 in +Python. Installation ------------ @@ -12,10 +10,65 @@ Installation Running ------- -To build the spec: +To pass arbitrary files (not limited to RST) through the templating system: ``` - $ python build.py + $ python build.py -i matrix_templates /random/file/path/here.rst ``` -This will output ``spec.rst`` which can then be fed into the RST->HTML -converter located in ``matrix-doc/scripts``. +The template output can be found at ``out/here.rst``. For a full list of +options, type ``python build.py --help``. + +Developing +---------- + +## Sections and Units +Batesian is built around the concept of Sections and Units. Sections are strings +which will be inserted into the provided document. Every section has a unique +key name which is the template variable that it represents. Units are arbitrary +python data. They are also represented by unique key names. + +## Adding template variables +If you want to add a new template variable e.g. `{{foo_bar}}` which is replaced +with the text `foobar`, you need to add a new Section: + + - Open `matrix_templates/sections.py`. + - Add a new function to `MatrixSections` called `render_foo_bar`. The function + name after `render_` determines the template variable name, and the return + value of this function determines what will be inserted. + ```python + def render_foo_bar(self): + return "foobar" + ``` + - Run `build.py`! + +## Adding data for template variables +If you want to expose arbitrary data which can be used by `MatrixSections`, you +need to add a new Unit: + + - Open `matrix_templates/units.py`. + - Add a new function to `MatrixUnits` called `load_some_data`. Similar to + sections, the function name after `load_` determines the unit name, and the + return value of this function determines the value of the unit. + ```python + def load_some_data(self): + return { + "data": "this could be JSON from file from json.loads()", + "processed_data": "this data may have helper keys added", + "types": "it doesn't even need to be a dict. Whatever you want!" + } + ``` + - You can now call `self.units.get("some_data")` to retrieve the value you + returned. + +## Using Jinja templates +Sections can use Jinja templates to return text. Batesian will attempt to load +all templates from `matrix_templates/templates/`. These can be accessed in +Section code via `template = self.env.get_template("name_of_template.tmpl")`. At +this point, the `template` is just a standard `jinja2.Template`. In fact, +`self.env` is just a `jinja2.Environment`. + +## Debugging +If you don't know why your template isn't behaving as you'd expect, or you just +want to add some informative logging, use `self.log` in either the Sections +class or Units class. You'll need to add `-v` to `build.py` for these lines to +show. \ No newline at end of file diff --git a/templating/internal/__init__.py b/templating/batesian/__init__.py similarity index 72% rename from templating/internal/__init__.py rename to templating/batesian/__init__.py index 7a9c7a22..1c394e7e 100644 --- a/templating/internal/__init__.py +++ b/templating/batesian/__init__.py @@ -5,8 +5,10 @@ class AccessKeyStore(object): """Storage for arbitrary data. Monitors get calls so we know if they were used or not.""" - def __init__(self): - self.data = {} + def __init__(self, existing_data=None): + if not existing_data: + existing_data = {} + self.data = existing_data self.accessed_set = Set() def keys(self): diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py new file mode 100644 index 00000000..9b1da5b5 --- /dev/null +++ b/templating/batesian/sections.py @@ -0,0 +1,33 @@ +"""Parent class for writing sections.""" +import inspect +import os + + +class Sections(object): + """A class which creates sections for each method starting with "render_". + The key for the section is the text after "render_" + e.g. "render_room_events" has the section key "room_events" + """ + + def __init__(self, env, units, debug=False): + self.env = env + self.units = units + self.debug = debug + + def log(self, text): + if self.debug: + print text + + def get_sections(self): + render_list = inspect.getmembers(self, predicate=inspect.ismethod) + section_dict = {} + for (func_name, func) in render_list: + if not func_name.startswith("render_"): + continue + section_key = func_name[len("render_"):] + section = func() + section_dict[section_key] = section + self.log("Generated section '%s' : %s" % ( + section_key, section[:60].replace("\n","") + )) + return section_dict \ No newline at end of file diff --git a/templating/batesian/units.py b/templating/batesian/units.py new file mode 100644 index 00000000..c20a2d1f --- /dev/null +++ b/templating/batesian/units.py @@ -0,0 +1,40 @@ +"""Parent class for writing units.""" +from . import AccessKeyStore +import inspect +import json +import os +import subprocess + +class Units(object): + + @staticmethod + def prop(obj, path): + # Helper method to extract nested property values + nested_keys = path.split("/") + val = obj + for key in nested_keys: + val = val.get(key, {}) + return val + + + def __init__(self, debug=False): + self.debug = debug + + def log(self, text): + if self.debug: + print text + + def get_units(self, debug=False): + unit_list = inspect.getmembers(self, predicate=inspect.ismethod) + unit_dict = {} + for (func_name, func) in unit_list: + if not func_name.startswith("load_"): + continue + unit_key = func_name[len("load_"):] + unit_dict[unit_key] = func() + self.log("Generated unit '%s' : %s" % ( + unit_key, json.dumps(unit_dict[unit_key])[:50].replace( + "\n","" + ) + )) + return unit_dict diff --git a/templating/build.py b/templating/build.py index 6594c4f6..d6b9a1ba 100755 --- a/templating/build.py +++ b/templating/build.py @@ -39,27 +39,18 @@ Checks - Any units made which were not used at least once will produce a warning. - Any sections made but not used in the skeleton will produce a warning. """ +from batesian import AccessKeyStore from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template from argparse import ArgumentParser, FileType +import importlib import json import os import sys from textwrap import TextWrapper -import internal.units -import internal.sections - -def load_units(): - print "Loading units..." - return internal.units.load() - -def load_sections(env, units): - print "\nLoading sections..." - return internal.sections.load(env, units) - def create_from_template(template, sections): - return template.render(sections.data) + return template.render(sections) def check_unaccessed(name, store): unaccessed_keys = store.get_unaccessed_set() @@ -67,10 +58,12 @@ def check_unaccessed(name, store): print "Found %s unused %s keys." % (len(unaccessed_keys), name) print unaccessed_keys -def main(file_stream=None, out_dir=None): +def main(input_module, file_stream=None, out_dir=None, verbose=False): if out_dir and not os.path.exists(out_dir): os.makedirs(out_dir) + in_mod = importlib.import_module(input_module) + # add a template filter to produce pretty pretty JSON def jsonify(input, indent=None, pre_whitespace=0): code = json.dumps(input, indent=indent, sort_keys=True) @@ -93,7 +86,7 @@ def main(file_stream=None, out_dir=None): # make Jinja aware of the templates and filters env = Environment( - loader=FileSystemLoader("templates"), + loader=FileSystemLoader(in_mod.exports["templates"]), undefined=StrictUndefined ) env.filters["jsonify"] = jsonify @@ -104,10 +97,12 @@ def main(file_stream=None, out_dir=None): # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when # they want it. - units = load_units() + units = AccessKeyStore( + existing_data=in_mod.exports["units"](debug=verbose).get_units() + ) # use the units to create RST sections - sections = load_sections(env, units) + sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections() # print out valid section keys if no file supplied if not file_stream: @@ -132,14 +127,18 @@ def main(file_stream=None, out_dir=None): if __name__ == '__main__': parser = ArgumentParser( - "Process a file (typically .rst) and replace templated areas with spec"+ - " info. For a list of possible template variables, add"+ - " --show-template-vars." + "Process a file (typically .rst) and replace templated areas with "+ + "section information from the provided input module. For a list of "+ + "possible template variables, add --show-template-vars." ) parser.add_argument( "file", nargs="?", type=FileType('r'), help="The input file to process." ) + parser.add_argument( + "--input", "-i", + help="The python module which contains the sections/units classes." + ) parser.add_argument( "--out-directory", "-o", help="The directory to output the file to."+ " Default: /out", @@ -150,10 +149,17 @@ if __name__ == '__main__': help="Show a list of all possible variables you can use in the"+ " input file." ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Turn on verbose mode." + ) args = parser.parse_args() + if not args.input: + raise Exception("Missing input module") + if (args.show_template_vars): - main() + main(args.input, verbose=args.verbose) sys.exit(0) if not args.file: @@ -161,4 +167,7 @@ if __name__ == '__main__': parser.print_help() sys.exit(1) - main(file_stream=args.file, out_dir=args.out_directory) + main( + args.input, file_stream=args.file, out_dir=args.out_directory, + verbose=args.verbose + ) diff --git a/templating/matrix_templates/__init__.py b/templating/matrix_templates/__init__.py new file mode 100644 index 00000000..1e161700 --- /dev/null +++ b/templating/matrix_templates/__init__.py @@ -0,0 +1,9 @@ +from sections import MatrixSections +from units import MatrixUnits +import os + +exports = { + "units": MatrixUnits, + "sections": MatrixSections, + "templates": os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") +} \ No newline at end of file diff --git a/templating/internal/sections.py b/templating/matrix_templates/sections.py similarity index 50% rename from templating/internal/sections.py rename to templating/matrix_templates/sections.py index 0693f336..efeb00cc 100644 --- a/templating/internal/sections.py +++ b/templating/matrix_templates/sections.py @@ -1,37 +1,11 @@ """Contains all the sections for the spec.""" -from . import AccessKeyStore +from batesian import AccessKeyStore +from batesian.sections import Sections import inspect import os -class Sections(object): - """A class which creates sections for each method starting with "render_". - The key for the section is the text after "render_" - e.g. "render_room_events" has the section key "room_events" - """ - - def __init__(self, env, units, debug=False): - self.env = env - self.units = units - self.debug = debug - - def log(self, text): - if self.debug: - print text - - def get_sections(self): - render_list = inspect.getmembers(self, predicate=inspect.ismethod) - section_dict = {} - for (func_name, func) in render_list: - if not func_name.startswith("render_"): - continue - section_key = func_name[len("render_"):] - section = func() - section_dict[section_key] = section - self.log("Generated section '%s' : %s" % ( - section_key, section[:60].replace("\n","") - )) - return section_dict +class MatrixSections(Sections): def render_room_events(self): template = self.env.get_template("events.tmpl") @@ -64,12 +38,3 @@ class Sections(object): def render_common_state_event_fields(self): return self._render_ce_type("state_event") - - -def load(env, units): - store = AccessKeyStore() - sections = Sections(env, units) - section_dict = sections.get_sections() - for section_key in section_dict: - store.add(section_key, section_dict[section_key]) - return store \ No newline at end of file diff --git a/templating/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl similarity index 100% rename from templating/templates/common-event-fields.tmpl rename to templating/matrix_templates/templates/common-event-fields.tmpl diff --git a/templating/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl similarity index 100% rename from templating/templates/events.tmpl rename to templating/matrix_templates/templates/events.tmpl diff --git a/templating/internal/units.py b/templating/matrix_templates/units.py similarity index 85% rename from templating/internal/units.py rename to templating/matrix_templates/units.py index 3ff5714f..ea62725b 100644 --- a/templating/internal/units.py +++ b/templating/matrix_templates/units.py @@ -1,41 +1,12 @@ """Contains all the units for the spec.""" -from . import AccessKeyStore +from batesian.units import Units import inspect import json import os import subprocess -def prop(obj, path): - # Helper method to extract nested property values - nested_keys = path.split("/") - val = obj - for key in nested_keys: - val = val.get(key, {}) - return val -class Units(object): - - def __init__(self, debug=False): - self.debug = debug - - def log(self, text): - if self.debug: - print text - - def get_units(self, debug=False): - unit_list = inspect.getmembers(self, predicate=inspect.ismethod) - unit_dict = {} - for (func_name, func) in unit_list: - if not func_name.startswith("load_"): - continue - unit_key = func_name[len("load_"):] - unit_dict[unit_key] = func() - self.log("Generated unit '%s' : %s" % ( - unit_key, json.dumps(unit_dict[unit_key])[:50].replace( - "\n","" - ) - )) - return unit_dict +class MatrixUnits(Units): def load_common_event_fields(self): path = "../event-schemas/schema/v1/core" @@ -177,7 +148,9 @@ class Units(object): ) # add type - schema["type"] = prop(json_schema, "properties/type/enum")[0] + schema["type"] = Units.prop( + json_schema, "properties/type/enum" + )[0] # add summary and desc schema["title"] = json_schema.get("title") @@ -185,12 +158,14 @@ class Units(object): # walk the object for field info schema["content_fields"] = get_content_fields( - prop(json_schema, "properties/content") + Units.prop(json_schema, "properties/content") ) # Assign state key info if schema["typeof"] == "State Event": - skey_desc = prop(json_schema, "properties/state_key/description") + skey_desc = Units.prop( + json_schema, "properties/state_key/description" + ) if not skey_desc: raise Exception("Missing description for state_key") schema["typeof_info"] = "``state_key``: %s" % skey_desc @@ -245,12 +220,3 @@ class Units(object): ) return git_version.encode("ascii") return "Unknown rev" - - -def load(): - store = AccessKeyStore() - units = Units() - unit_dict = units.get_units() - for unit_key in unit_dict: - store.add(unit_key, unit_dict[unit_key]) - return store