From 5b31c442f5cdb081561f079859b0c27c7ed8d618 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 26 May 2015 16:20:15 +0100
Subject: [PATCH] Completely split up the templating system from the Matrix
 Spec template code.

The two are now linked together in build.py by specifying the input module.
Updated gendoc.py to specify the right module.
---
 scripts/gendoc.py                             |  7 +-
 templating/README.md                          | 71 ++++++++++++++++---
 templating/{internal => batesian}/__init__.py |  6 +-
 templating/batesian/sections.py               | 33 +++++++++
 templating/batesian/units.py                  | 40 +++++++++++
 templating/build.py                           | 51 +++++++------
 templating/matrix_templates/__init__.py       |  9 +++
 .../sections.py                               | 41 +----------
 .../templates/common-event-fields.tmpl        |  0
 .../templates/events.tmpl                     |  0
 .../{internal => matrix_templates}/units.py   | 52 +++-----------
 11 files changed, 196 insertions(+), 114 deletions(-)
 rename templating/{internal => batesian}/__init__.py (72%)
 create mode 100644 templating/batesian/sections.py
 create mode 100644 templating/batesian/units.py
 create mode 100644 templating/matrix_templates/__init__.py
 rename templating/{internal => matrix_templates}/sections.py (50%)
 rename templating/{ => matrix_templates}/templates/common-event-fields.tmpl (100%)
 rename templating/{ => matrix_templates}/templates/events.tmpl (100%)
 rename templating/{internal => matrix_templates}/units.py (85%)

diff --git a/scripts/gendoc.py b/scripts/gendoc.py
index f9120551..33bd1fe6 100755
--- a/scripts/gendoc.py
+++ b/scripts/gendoc.py
@@ -34,7 +34,12 @@ def rst2html(i, o):
 def run_through_template(input):
     null = open(os.devnull, 'w')
     subprocess.check_output(
-        ['python', 'build.py', "-o", "../scripts/tmp", "../scripts/"+input],
+        [
+            'python', 'build.py', 
+            "-i", "matrix_templates", 
+            "-o", "../scripts/tmp", 
+            "../scripts/"+input
+        ],
         stderr=null,
         cwd="../templating",
     )
diff --git a/templating/README.md b/templating/README.md
index c1992334..f76051a5 100644
--- a/templating/README.md
+++ b/templating/README.md
@@ -1,8 +1,6 @@
-This folder contains the templates and templating system for creating the spec.
-We use the templating system Jinja2 in Python. This was chosen over other
-systems such as Handlebars.js and Templetor because we already have a Python
-dependency on the spec build system, and Jinja provides a rich set of template
-operations beyond basic control flow.
+This folder contains the templates and a home-brewed templating system called
+Batesian for creating the spec. Batesian uses the templating system Jinja2 in
+Python.
 
 Installation
 ------------
@@ -12,10 +10,65 @@ Installation
 
 Running
 -------
-To build the spec:
+To pass arbitrary files (not limited to RST) through the templating system:
 ```
- $ python build.py
+ $ python build.py -i matrix_templates /random/file/path/here.rst
 ```
 
-This will output ``spec.rst`` which can then be fed into the RST->HTML
-converter located in ``matrix-doc/scripts``.
+The template output can be found at ``out/here.rst``. For a full list of
+options, type ``python build.py --help``.
+
+Developing
+----------
+
+## Sections and Units
+Batesian is built around the concept of Sections and Units. Sections are strings
+which will be inserted into the provided document. Every section has a unique
+key name which is the template variable that it represents. Units are arbitrary
+python data. They are also represented by unique key names.
+
+## Adding template variables
+If you want to add a new template variable e.g. `{{foo_bar}}` which is replaced
+with the text `foobar`, you need to add a new Section:
+
+ - Open `matrix_templates/sections.py`.
+ - Add a new function to `MatrixSections` called `render_foo_bar`. The function
+   name after `render_` determines the template variable name, and the return
+   value of this function determines what will be inserted.
+   ```python
+   def render_foo_bar(self):
+       return "foobar"
+   ```
+ - Run `build.py`!
+
+## Adding data for template variables
+If you want to expose arbitrary data which can be used by `MatrixSections`, you
+need to add a new Unit:
+
+ - Open `matrix_templates/units.py`.
+ - Add a new function to `MatrixUnits` called `load_some_data`. Similar to
+   sections, the function name after `load_` determines the unit name, and the
+   return value of this function determines the value of the unit.
+   ```python
+   def load_some_data(self):
+       return {
+          "data": "this could be JSON from file from json.loads()",
+          "processed_data": "this data may have helper keys added",
+          "types": "it doesn't even need to be a dict. Whatever you want!"
+       }
+   ```
+ - You can now call `self.units.get("some_data")` to retrieve the value you
+   returned.
+   
+## Using Jinja templates
+Sections can use Jinja templates to return text. Batesian will attempt to load
+all templates from `matrix_templates/templates/`. These can be accessed in
+Section code via `template = self.env.get_template("name_of_template.tmpl")`. At
+this point, the `template` is just a standard `jinja2.Template`. In fact,
+`self.env` is just a `jinja2.Environment`.
+
+## Debugging
+If you don't know why your template isn't behaving as you'd expect, or you just
+want to add some informative logging, use `self.log` in either the Sections
+class or Units class. You'll need to add `-v` to `build.py` for these lines to
+show.
\ No newline at end of file
diff --git a/templating/internal/__init__.py b/templating/batesian/__init__.py
similarity index 72%
rename from templating/internal/__init__.py
rename to templating/batesian/__init__.py
index 7a9c7a22..1c394e7e 100644
--- a/templating/internal/__init__.py
+++ b/templating/batesian/__init__.py
@@ -5,8 +5,10 @@ class AccessKeyStore(object):
     """Storage for arbitrary data. Monitors get calls so we know if they
     were used or not."""
 
-    def __init__(self):
-        self.data = {}
+    def __init__(self, existing_data=None):
+        if not existing_data:
+            existing_data = {}
+        self.data = existing_data
         self.accessed_set = Set()
 
     def keys(self):
diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py
new file mode 100644
index 00000000..9b1da5b5
--- /dev/null
+++ b/templating/batesian/sections.py
@@ -0,0 +1,33 @@
+"""Parent class for writing sections."""
+import inspect
+import os
+
+
+class Sections(object):
+    """A class which creates sections for each method starting with "render_".
+    The key for the section is the text after "render_"
+    e.g. "render_room_events" has the section key "room_events"
+    """
+
+    def __init__(self, env, units, debug=False):
+        self.env = env
+        self.units = units
+        self.debug = debug
+
+    def log(self, text):
+        if self.debug:
+            print text
+
+    def get_sections(self):
+        render_list = inspect.getmembers(self, predicate=inspect.ismethod)
+        section_dict = {}
+        for (func_name, func) in render_list:
+            if not func_name.startswith("render_"):
+                continue
+            section_key = func_name[len("render_"):]
+            section = func()
+            section_dict[section_key] = section
+            self.log("Generated section '%s' : %s" % (
+                section_key, section[:60].replace("\n","")
+            ))
+        return section_dict
\ No newline at end of file
diff --git a/templating/batesian/units.py b/templating/batesian/units.py
new file mode 100644
index 00000000..c20a2d1f
--- /dev/null
+++ b/templating/batesian/units.py
@@ -0,0 +1,40 @@
+"""Parent class for writing units."""
+from . import AccessKeyStore
+import inspect
+import json
+import os
+import subprocess
+
+class Units(object):
+
+    @staticmethod
+    def prop(obj, path):
+        # Helper method to extract nested property values
+        nested_keys = path.split("/")
+        val = obj
+        for key in nested_keys:
+            val = val.get(key, {})
+        return val
+
+
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    def log(self, text):
+        if self.debug:
+            print text
+
+    def get_units(self, debug=False):
+        unit_list = inspect.getmembers(self, predicate=inspect.ismethod)
+        unit_dict = {}
+        for (func_name, func) in unit_list:
+            if not func_name.startswith("load_"):
+                continue
+            unit_key = func_name[len("load_"):]
+            unit_dict[unit_key] = func()
+            self.log("Generated unit '%s' : %s" % (
+                unit_key, json.dumps(unit_dict[unit_key])[:50].replace(
+                    "\n",""
+                )
+            ))
+        return unit_dict
diff --git a/templating/build.py b/templating/build.py
index 6594c4f6..d6b9a1ba 100755
--- a/templating/build.py
+++ b/templating/build.py
@@ -39,27 +39,18 @@ Checks
 - Any units made which were not used at least once will produce a warning.
 - Any sections made but not used in the skeleton will produce a warning.
 """
+from batesian import AccessKeyStore
 
 from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
 from argparse import ArgumentParser, FileType
+import importlib
 import json
 import os
 import sys
 from textwrap import TextWrapper
 
-import internal.units
-import internal.sections
-
-def load_units():
-    print "Loading units..."
-    return internal.units.load()
-
-def load_sections(env, units):
-    print "\nLoading sections..."
-    return internal.sections.load(env, units)
-
 def create_from_template(template, sections):
-    return template.render(sections.data)
+    return template.render(sections)
 
 def check_unaccessed(name, store):
     unaccessed_keys = store.get_unaccessed_set()
@@ -67,10 +58,12 @@ def check_unaccessed(name, store):
         print "Found %s unused %s keys." % (len(unaccessed_keys), name)
         print unaccessed_keys
 
-def main(file_stream=None, out_dir=None):
+def main(input_module, file_stream=None, out_dir=None, verbose=False):
     if out_dir and not os.path.exists(out_dir):
         os.makedirs(out_dir)
 
+    in_mod = importlib.import_module(input_module)
+
     # add a template filter to produce pretty pretty JSON
     def jsonify(input, indent=None, pre_whitespace=0):
         code = json.dumps(input, indent=indent, sort_keys=True)
@@ -93,7 +86,7 @@ def main(file_stream=None, out_dir=None):
 
     # make Jinja aware of the templates and filters
     env = Environment(
-        loader=FileSystemLoader("templates"),
+        loader=FileSystemLoader(in_mod.exports["templates"]),
         undefined=StrictUndefined
     )
     env.filters["jsonify"] = jsonify
@@ -104,10 +97,12 @@ def main(file_stream=None, out_dir=None):
     # load up and parse the lowest single units possible: we don't know or care
     # which spec section will use it, we just need it there in memory for when
     # they want it.
-    units = load_units()
+    units = AccessKeyStore(
+        existing_data=in_mod.exports["units"](debug=verbose).get_units()
+    )
 
     # use the units to create RST sections
-    sections = load_sections(env, units)
+    sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
 
     # print out valid section keys if no file supplied
     if not file_stream:
@@ -132,14 +127,18 @@ def main(file_stream=None, out_dir=None):
 
 if __name__ == '__main__':
     parser = ArgumentParser(
-        "Process a file (typically .rst) and replace templated areas with spec"+
-        " info. For a list of possible template variables, add"+
-        " --show-template-vars."
+        "Process a file (typically .rst) and replace templated areas with "+
+        "section information from the provided input module. For a list of "+
+        "possible template variables, add --show-template-vars."
     )
     parser.add_argument(
         "file", nargs="?", type=FileType('r'),
         help="The input file to process."
     )
+    parser.add_argument(
+        "--input", "-i", 
+        help="The python module which contains the sections/units classes."
+    )
     parser.add_argument(
         "--out-directory", "-o", help="The directory to output the file to."+
         " Default: /out",
@@ -150,10 +149,17 @@ if __name__ == '__main__':
         help="Show a list of all possible variables you can use in the"+
         " input file."
     )
+    parser.add_argument(
+        "--verbose", "-v", action="store_true",
+        help="Turn on verbose mode."
+    )
     args = parser.parse_args()
 
+    if not args.input:
+        raise Exception("Missing input module")
+
     if (args.show_template_vars):
-        main()
+        main(args.input, verbose=args.verbose)
         sys.exit(0)
 
     if not args.file:
@@ -161,4 +167,7 @@ if __name__ == '__main__':
         parser.print_help()
         sys.exit(1)
 
-    main(file_stream=args.file, out_dir=args.out_directory)
+    main(
+        args.input, file_stream=args.file, out_dir=args.out_directory,
+        verbose=args.verbose
+    )
diff --git a/templating/matrix_templates/__init__.py b/templating/matrix_templates/__init__.py
new file mode 100644
index 00000000..1e161700
--- /dev/null
+++ b/templating/matrix_templates/__init__.py
@@ -0,0 +1,9 @@
+from sections import MatrixSections
+from units import MatrixUnits
+import os
+
+exports = {
+    "units": MatrixUnits,
+    "sections": MatrixSections,
+    "templates": os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")
+}
\ No newline at end of file
diff --git a/templating/internal/sections.py b/templating/matrix_templates/sections.py
similarity index 50%
rename from templating/internal/sections.py
rename to templating/matrix_templates/sections.py
index 0693f336..efeb00cc 100644
--- a/templating/internal/sections.py
+++ b/templating/matrix_templates/sections.py
@@ -1,37 +1,11 @@
 """Contains all the sections for the spec."""
-from . import AccessKeyStore
+from batesian import AccessKeyStore
+from batesian.sections import Sections
 import inspect
 import os
 
 
-class Sections(object):
-    """A class which creates sections for each method starting with "render_".
-    The key for the section is the text after "render_"
-    e.g. "render_room_events" has the section key "room_events"
-    """
-
-    def __init__(self, env, units, debug=False):
-        self.env = env
-        self.units = units
-        self.debug = debug
-
-    def log(self, text):
-        if self.debug:
-            print text
-
-    def get_sections(self):
-        render_list = inspect.getmembers(self, predicate=inspect.ismethod)
-        section_dict = {}
-        for (func_name, func) in render_list:
-            if not func_name.startswith("render_"):
-                continue
-            section_key = func_name[len("render_"):]
-            section = func()
-            section_dict[section_key] = section
-            self.log("Generated section '%s' : %s" % (
-                section_key, section[:60].replace("\n","")
-            ))
-        return section_dict
+class MatrixSections(Sections):
 
     def render_room_events(self):
         template = self.env.get_template("events.tmpl")
@@ -64,12 +38,3 @@ class Sections(object):
 
     def render_common_state_event_fields(self):
         return self._render_ce_type("state_event")
-
-
-def load(env, units):
-    store = AccessKeyStore()
-    sections = Sections(env, units)
-    section_dict = sections.get_sections()
-    for section_key in section_dict:
-        store.add(section_key, section_dict[section_key])
-    return store
\ No newline at end of file
diff --git a/templating/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl
similarity index 100%
rename from templating/templates/common-event-fields.tmpl
rename to templating/matrix_templates/templates/common-event-fields.tmpl
diff --git a/templating/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl
similarity index 100%
rename from templating/templates/events.tmpl
rename to templating/matrix_templates/templates/events.tmpl
diff --git a/templating/internal/units.py b/templating/matrix_templates/units.py
similarity index 85%
rename from templating/internal/units.py
rename to templating/matrix_templates/units.py
index 3ff5714f..ea62725b 100644
--- a/templating/internal/units.py
+++ b/templating/matrix_templates/units.py
@@ -1,41 +1,12 @@
 """Contains all the units for the spec."""
-from . import AccessKeyStore
+from batesian.units import Units
 import inspect
 import json
 import os
 import subprocess
 
-def prop(obj, path):
-    # Helper method to extract nested property values
-    nested_keys = path.split("/")
-    val = obj
-    for key in nested_keys:
-        val = val.get(key, {})
-    return val
 
-class Units(object):
-
-    def __init__(self, debug=False):
-        self.debug = debug
-
-    def log(self, text):
-        if self.debug:
-            print text
-
-    def get_units(self, debug=False):
-        unit_list = inspect.getmembers(self, predicate=inspect.ismethod)
-        unit_dict = {}
-        for (func_name, func) in unit_list:
-            if not func_name.startswith("load_"):
-                continue
-            unit_key = func_name[len("load_"):]
-            unit_dict[unit_key] = func()
-            self.log("Generated unit '%s' : %s" % (
-                unit_key, json.dumps(unit_dict[unit_key])[:50].replace(
-                    "\n",""
-                )
-            ))
-        return unit_dict
+class MatrixUnits(Units):
 
     def load_common_event_fields(self):
         path = "../event-schemas/schema/v1/core"
@@ -177,7 +148,9 @@ class Units(object):
                     )
 
                 # add type
-                schema["type"] = prop(json_schema, "properties/type/enum")[0]
+                schema["type"] = Units.prop(
+                    json_schema, "properties/type/enum"
+                )[0]
 
                 # add summary and desc
                 schema["title"] = json_schema.get("title")
@@ -185,12 +158,14 @@ class Units(object):
 
                 # walk the object for field info
                 schema["content_fields"] = get_content_fields(
-                    prop(json_schema, "properties/content")
+                    Units.prop(json_schema, "properties/content")
                 )
 
                 # Assign state key info
                 if schema["typeof"] == "State Event":
-                    skey_desc = prop(json_schema, "properties/state_key/description")
+                    skey_desc = Units.prop(
+                        json_schema, "properties/state_key/description"
+                    )
                     if not skey_desc:
                         raise Exception("Missing description for state_key")
                     schema["typeof_info"] = "``state_key``: %s" % skey_desc
@@ -245,12 +220,3 @@ class Units(object):
             )
             return git_version.encode("ascii")
         return "Unknown rev"
-
-
-def load():
-    store = AccessKeyStore()
-    units = Units()
-    unit_dict = units.get_units()
-    for unit_key in unit_dict:
-        store.add(unit_key, unit_dict[unit_key])
-    return store