Merge pull request #19 from matrix-org/templating

Implement templating system

.gitignore

@@ -1 +1,2 @@
 scripts/gen
+*.pyc

event-schemas/check.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+# Runs z-schema over all of the schema files (looking for matching examples)
+find schema/v1/m.* | while read line
+do
+    split_path=(${line///// })
+    event_type=(${split_path[2]})
+    echo "Checking $event_type"
+    echo "--------------------"
+    # match exact name or exact name with a #<something>
+    find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline
+    do
+        echo "    against $exline"
+        # run z-schema and dump stdout/err to the terminal (good for Jenkin's Console Output) and grep for fail messages
+        if [[ -n $(z-schema schema/v1/$event_type $exline 2>&1 | tee /dev/tty | grep -Ei "error|failed") ]]; then
+            echo "    Failed."
+            exit 1
+        fi
+    done
+done

event-schemas/examples/v1/m.room.member

@@ -1,7 +1,9 @@
 {
   "age": 242352,
   "content": {
-    "membership": "join"
+    "membership": "join",
+    "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
+    "displayname": "Alice Margatroid"
   },
   "state_key": "@alice:localhost",
   "origin_server_ts": 1431961217939,

event-schemas/examples/v1/m.room.redaction

@@ -1,6 +1,8 @@
 {
   "age": 242352,
-  "content": {},
+  "content": {
+    "reason": "Spamming"
+  },
   "origin_server_ts": 1431961217939,
   "event_id": "$WLGTSEFSEF:localhost",
   "type": "m.room.redaction",

event-schemas/schema/v1/m.room.aliases

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "Informs the room about what room aliases it has been given.",
+    "description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],

event-schemas/schema/v1/m.room.create

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "The first event in the room.",
+    "description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],

event-schemas/schema/v1/m.room.join_rules

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "Describes how users are allowed to join the room.",
+    "description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],

event-schemas/schema/v1/m.room.member

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "The current membership state of a user in the room.",
+    "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms/<room id>/invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],
@@ -11,6 +13,12 @@
                 "membership": {
                     "type": "string",
                     "enum": ["invite","join","knock","leave","ban"]
+                },
+                "avatar_url": {
+                    "type": "string"
+                },
+                "displayname": {
+                    "type": "string"
                 }
             },
             "required": ["membership"]

event-schemas/schema/v1/m.room.message

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "A human-readable message in the room.",
+    "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.",
     "allOf": [{
         "$ref": "core#/definitions/room_event"
     }],

event-schemas/schema/v1/m.room.message.feedback

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "A receipt for a message. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.",
+    "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.",
     "allOf": [{
         "$ref": "core#/definitions/room_event"
     }],

event-schemas/schema/v1/m.room.name

@@ -1,5 +1,7 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "Set the human-readable name for the room.",
+    "description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.",
     "type": "object",
     "allOf": [{
         "$ref": "core#/definitions/state_event"

event-schemas/schema/v1/m.room.power_levels

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "Defines the power levels (privileges) of users in the room.",
+    "description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],
@@ -28,7 +30,7 @@
                 }
             },
             "required": ["ban","events","events_default","kick","redact",
-                         "state_default","users","users_default"]
+                         "state_default","users"]
         },
         "state_key": {
             "type": "string",

event-schemas/schema/v1/m.room.redaction

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "Indicates a previous event has been redacted.",
+    "description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data.  There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.",
     "allOf": [{
         "$ref": "core#/definitions/room_event"
     }],

event-schemas/schema/v1/m.room.topic

@@ -1,6 +1,8 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
+    "title": "Set a topic for the room.",
+    "description": "A topic is a short message detailing what is currently being discussed in the room.  It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.",
     "allOf": [{
         "$ref": "core#/definitions/state_event"
     }],

templating/README.md

@@ -0,0 +1,21 @@
+This folder contains the templates and templating system for creating the spec.
+We use the templating system Jinja2 in Python. This was chosen over other
+systems such as Handlebars.js and Templetor because we already have a Python
+dependency on the spec build system, and Jinja provides a rich set of template
+operations beyond basic control flow.
+
+Installation
+------------
+```
+ $ pip install Jinja2
+```
+
+Running
+-------
+To build the spec:
+```
+ $ python build.py
+```
+
+This will output ``spec.rst`` which can then be fed into the RST->HTML
+converter located in ``matrix-doc/scripts``.

templating/build.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+""" 
+Builds the Matrix Specification as restructed text (RST).
+
+Architecture
+============
++-------+                            +----------+
+| units |-+                          | sections |-+
++-------+ |-+ === used to create ==> +----------- | === used to create ==> SPEC
+  +-------+ |                          +----------+
+   +--------+
+RAW DATA (e.g. json)                  Blobs of RST
+
+Units
+=====
+Units are random bits of unprocessed data, e.g. schema JSON files. Anything can
+be done to them, from processing it with Jinja to arbitrary python processing.
+They are dicts.
+
+Sections
+========
+Sections are short segments of RST. They will be in the final spec, but they
+are unordered. They typically use a combination of templates + units to
+construct bits of RST.
+
+Skeleton
+========
+The skeleton is a single RST file which is passed through a templating system to
+replace variable names with sections.
+
+Processing
+==========
+- Execute all unit functions to load units into memory and process them.
+- Execute all section functions (which can now be done because the units exist)
+- Execute the skeleton function to bring it into a single file.
+
+Checks
+======
+- Any units made which were not used at least once will produce a warning.
+- Any sections made but not used in the skeleton will produce a warning.
+"""
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
+from argparse import ArgumentParser, FileType
+import json
+import os
+import sys
+import textwrap
+
+import internal.units
+import internal.sections
+
+def load_units():
+    print "Loading units..."
+    return internal.units.load()
+
+def load_sections(env, units):
+    print "\nLoading sections..."
+    return internal.sections.load(env, units)
+
+def create_from_template(template, sections):
+    return template.render(sections.data)
+
+def check_unaccessed(name, store):
+    unaccessed_keys = store.get_unaccessed_set()
+    if len(unaccessed_keys) > 0:
+        print "Found %s unused %s keys." % (len(unaccessed_keys), name)
+        print unaccessed_keys
+
+def main(file_stream=None, out_dir=None):
+    if out_dir and not os.path.exists(out_dir):
+        os.makedirs(out_dir)
+
+    # add a template filter to produce pretty pretty JSON
+    def jsonify(input, indent=None, pre_whitespace=0):
+        code = json.dumps(input, indent=indent)
+        if pre_whitespace:
+            code = code.replace("\n", ("\n" +" "*pre_whitespace))
+
+        return code
+
+    def indent(input, indent):
+        return input.replace("\n", ("\n" + " "*indent))
+
+    def wrap(input, wrap=80):
+        return '\n'.join(textwrap.wrap(input, wrap))
+
+    # make Jinja aware of the templates and filters
+    env = Environment(
+        loader=FileSystemLoader("templates"),
+        undefined=StrictUndefined
+    )
+    env.filters["jsonify"] = jsonify
+    env.filters["indent"] = indent
+    env.filters["wrap"] = wrap
+
+    # load up and parse the lowest single units possible: we don't know or care
+    # which spec section will use it, we just need it there in memory for when
+    # they want it.
+    units = load_units()
+
+    # use the units to create RST sections
+    sections = load_sections(env, units)
+
+    # print out valid section keys if no file supplied
+    if not file_stream:
+        print "\nValid template variables:"
+        for key in sections.keys():
+            print "  %s" % key
+        return
+
+    # check the input files and substitute in sections where required
+    print "Parsing input template: %s" % file_stream.name
+    temp = Template(file_stream.read())
+    print "Creating output for: %s" % file_stream.name
+    output = create_from_template(temp, sections)
+    with open(os.path.join(out_dir, file_stream.name), "w") as f:
+        f.write(output)
+    print "Output file for: %s" % file_stream.name
+
+    check_unaccessed("units", units)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(
+        "Process a file (typically .rst) and replace templated areas with spec"+
+        " info. For a list of possible template variables, add"+
+        " --show-template-vars."
+    )
+    parser.add_argument(
+        "file", nargs="?", type=FileType('r'),
+        help="The input file to process."
+    )
+    parser.add_argument(
+        "--out-directory", "-o", help="The directory to output the file to."+
+        " Default: /out",
+        default="out"
+    )
+    parser.add_argument(
+        "--show-template-vars", "-s", action="store_true",
+        help="Show a list of all possible variables you can use in the"+
+        " input file."
+    )
+    args = parser.parse_args()
+
+    if (args.show_template_vars):
+        main()
+        sys.exit(0)
+
+    if not args.file:
+        print "No file supplied."
+        parser.print_help()
+        sys.exit(1)
+
+    main(file_stream=args.file, out_dir=args.out_directory)

templating/internal/__init__.py

@@ -0,0 +1,24 @@
+from sets import Set
+
+
+class AccessKeyStore(object):
+    """Storage for arbitrary data. Monitors get calls so we know if they
+    were used or not."""
+
+    def __init__(self):
+        self.data = {}
+        self.accessed_set = Set()
+
+    def keys(self):
+        return self.data.keys()
+
+    def add(self, key, unit_dict):
+        self.data[key] = unit_dict
+
+    def get(self, key):
+        self.accessed_set.add(key)
+        return self.data[key]
+
+    def get_unaccessed_set(self):
+        data_list = Set(self.data.keys())
+        return data_list - self.accessed_set

templating/internal/sections.py

@@ -0,0 +1,31 @@
+"""Contains all the sections for the spec."""
+from . import AccessKeyStore
+import os
+
+def _render_section_room_events(env, units):
+    template = env.get_template("events.tmpl")
+    examples = units.get("event-examples")
+    schemas = units.get("event-schemas")
+    sections = []
+    for event_name in schemas:
+        if not event_name.startswith("m.room"):
+            continue
+        sections.append(template.render(
+            example=examples[event_name], 
+            event=schemas[event_name]
+        ))
+    return "\n\n".join(sections)
+
+SECTION_DICT = {
+    "room_events": _render_section_room_events
+}
+
+def load(env, units):
+    store = AccessKeyStore()
+    for section_key in SECTION_DICT:
+        section = SECTION_DICT[section_key](env, units)
+        print "Generated section '%s' : %s" % (
+            section_key, section[:60].replace("\n","")
+        )
+        store.add(section_key, section)
+    return store

templating/internal/units.py

@@ -0,0 +1,118 @@
+"""Contains all the units for the spec."""
+from . import AccessKeyStore
+import json
+import os
+
+def prop(obj, path):
+    # Helper method to extract nested property values
+    nested_keys = path.split("/")
+    val = obj
+    for key in nested_keys:
+        val = val.get(key, {})
+    return val
+
+def _load_examples():
+    path = "../event-schemas/examples/v1"
+    examples = {}
+    for filename in os.listdir(path):
+        if not filename.startswith("m."):
+            continue
+        with open(os.path.join(path, filename), "r") as f:
+            examples[filename] = json.loads(f.read())
+            if filename == "m.room.message_m.text":
+                examples["m.room.message"] = examples[filename]
+    return examples
+
+def _load_schemas():
+    path = "../event-schemas/schema/v1"
+    schemata = {}
+
+    def format_for_obj(obj):
+        obj_type = "<%s>" % obj.get("type")
+        if obj_type == "<object>":
+            if obj.get("properties"):
+                format = {}
+                for key in obj.get("properties"):
+                    format[key] = format_for_obj(obj.get("properties")[key])
+                return format
+            elif obj.get("additionalProperties"):
+                return {
+                    "<string>": (
+                        "<%s>" % obj.get("additionalProperties").get("type")
+                    )
+                }
+        elif obj_type == "<array>" and obj.get("items"):
+            return [
+                format_for_obj(obj.get("items"))
+            ]
+
+        enum_text = ""
+        # add on enum info
+        enum = obj.get("enum")
+        if enum:
+            if len(enum) > 1:
+                obj_type = "<enum>"
+                enum_text = " (" + "|".join(enum) + ")"
+            else:
+                obj_type = enum[0]
+
+        return obj_type + enum_text
+
+    for filename in os.listdir(path):
+        if not filename.startswith("m."):
+            continue
+        print "Reading %s" % os.path.join(path, filename)
+        with open(os.path.join(path, filename), "r") as f:
+            json_schema = json.loads(f.read())
+            schema = {
+                "typeof": None,
+                "type": None,
+                "summary": None,
+                "desc": None,
+                "json_format": None,
+                "required_keys": None
+            }
+
+            # add typeof
+            base_defs = {
+                "core#/definitions/room_event": "Room Event",
+                "core#/definitions/state_event": "State Event"
+            }
+            if type(json_schema.get("allOf")) == list:
+                schema["typeof"] = base_defs.get(
+                    json_schema["allOf"][0].get("$ref")
+                )
+
+            # add type
+            schema["type"] = prop(json_schema, "properties/type/enum")[0]
+
+            # add summary and desc
+            schema["summary"] = json_schema.get("title")
+            schema["desc"] = json_schema.get("description")
+
+            # add json_format
+            content_props = prop(json_schema, "properties/content")
+            if content_props:
+                schema["json_format"] = format_for_obj(content_props)
+
+            # add required_keys
+            schema["required_keys"] = prop(
+                json_schema, "properties/content/required"
+            )
+            schemata[filename] = schema
+    return schemata
+
+UNIT_DICT = {
+    "event-examples": _load_examples,
+    "event-schemas": _load_schemas
+}
+
+def load():
+    store = AccessKeyStore()
+    for unit_key in UNIT_DICT:
+        unit = UNIT_DICT[unit_key]()
+        print "Generated unit '%s' : %s" % (
+            unit_key, json.dumps(unit)[:50].replace("\n","")
+        )
+        store.add(unit_key, unit)
+    return store

templating/templates/events.tmpl

@@ -0,0 +1,17 @@
+``{{event.type}}``
+  Summary:
+    {{event.summary | wrap(76) | indent(4)}}
+  Type:
+    {{event.typeof}}
+  Description:
+    {{event.desc | wrap(76) | indent(4)}}
+  Required Keys:
+    ``{{event.required_keys | jsonify}}``
+
+  JSON Format::
+
+    {{event.json_format | jsonify(4, 4)}}
+
+  Example::
+
+    {{example.content | jsonify(4, 4)}}