Merge pull request #19 from matrix-org/templating

Implement templating system
pull/977/head
Kegsay 10 years ago
commit 668cbdf7d0

1
.gitignore vendored

@ -1 +1,2 @@
scripts/gen scripts/gen
*.pyc

@ -0,0 +1,19 @@
#!/bin/bash -e
# Runs z-schema over all of the schema files (looking for matching examples)
find schema/v1/m.* | while read line
do
split_path=(${line///// })
event_type=(${split_path[2]})
echo "Checking $event_type"
echo "--------------------"
# match exact name or exact name with a #<something>
find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline
do
echo " against $exline"
# run z-schema and dump stdout/err to the terminal (good for Jenkin's Console Output) and grep for fail messages
if [[ -n $(z-schema schema/v1/$event_type $exline 2>&1 | tee /dev/tty | grep -Ei "error|failed") ]]; then
echo " Failed."
exit 1
fi
done
done

@ -1,7 +1,9 @@
{ {
"age": 242352, "age": 242352,
"content": { "content": {
"membership": "join" "membership": "join",
"avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
"displayname": "Alice Margatroid"
}, },
"state_key": "@alice:localhost", "state_key": "@alice:localhost",
"origin_server_ts": 1431961217939, "origin_server_ts": 1431961217939,

@ -1,6 +1,8 @@
{ {
"age": 242352, "age": 242352,
"content": {}, "content": {
"reason": "Spamming"
},
"origin_server_ts": 1431961217939, "origin_server_ts": 1431961217939,
"event_id": "$WLGTSEFSEF:localhost", "event_id": "$WLGTSEFSEF:localhost",
"type": "m.room.redaction", "type": "m.room.redaction",

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "Informs the room about what room aliases it has been given.",
"description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "The first event in the room.",
"description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "Describes how users are allowed to join the room.",
"description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "The current membership state of a user in the room.",
"description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms/<room id>/invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],
@ -11,6 +13,12 @@
"membership": { "membership": {
"type": "string", "type": "string",
"enum": ["invite","join","knock","leave","ban"] "enum": ["invite","join","knock","leave","ban"]
},
"avatar_url": {
"type": "string"
},
"displayname": {
"type": "string"
} }
}, },
"required": ["membership"] "required": ["membership"]

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "A human-readable message in the room.",
"description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/room_event" "$ref": "core#/definitions/room_event"
}], }],

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "A receipt for a message. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.",
"description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/room_event" "$ref": "core#/definitions/room_event"
}], }],

@ -1,5 +1,7 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"title": "Set the human-readable name for the room.",
"description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.",
"type": "object", "type": "object",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "Defines the power levels (privileges) of users in the room.",
"description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],
@ -28,7 +30,7 @@
} }
}, },
"required": ["ban","events","events_default","kick","redact", "required": ["ban","events","events_default","kick","redact",
"state_default","users","users_default"] "state_default","users"]
}, },
"state_key": { "state_key": {
"type": "string", "type": "string",

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "Indicates a previous event has been redacted.",
"description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data. There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/room_event" "$ref": "core#/definitions/room_event"
}], }],

@ -1,6 +1,8 @@
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object", "type": "object",
"title": "Set a topic for the room.",
"description": "A topic is a short message detailing what is currently being discussed in the room. It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.",
"allOf": [{ "allOf": [{
"$ref": "core#/definitions/state_event" "$ref": "core#/definitions/state_event"
}], }],

@ -0,0 +1,21 @@
This folder contains the templates and templating system for creating the spec.
We use the templating system Jinja2 in Python. This was chosen over other
systems such as Handlebars.js and Templetor because we already have a Python
dependency on the spec build system, and Jinja provides a rich set of template
operations beyond basic control flow.
Installation
------------
```
$ pip install Jinja2
```
Running
-------
To build the spec:
```
$ python build.py
```
This will output ``spec.rst`` which can then be fed into the RST->HTML
converter located in ``matrix-doc/scripts``.

@ -0,0 +1,155 @@
#!/usr/bin/env python
"""
Builds the Matrix Specification as restructed text (RST).
Architecture
============
+-------+ +----------+
| units |-+ | sections |-+
+-------+ |-+ === used to create ==> +----------- | === used to create ==> SPEC
+-------+ | +----------+
+--------+
RAW DATA (e.g. json) Blobs of RST
Units
=====
Units are random bits of unprocessed data, e.g. schema JSON files. Anything can
be done to them, from processing it with Jinja to arbitrary python processing.
They are dicts.
Sections
========
Sections are short segments of RST. They will be in the final spec, but they
are unordered. They typically use a combination of templates + units to
construct bits of RST.
Skeleton
========
The skeleton is a single RST file which is passed through a templating system to
replace variable names with sections.
Processing
==========
- Execute all unit functions to load units into memory and process them.
- Execute all section functions (which can now be done because the units exist)
- Execute the skeleton function to bring it into a single file.
Checks
======
- Any units made which were not used at least once will produce a warning.
- Any sections made but not used in the skeleton will produce a warning.
"""
from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
from argparse import ArgumentParser, FileType
import json
import os
import sys
import textwrap
import internal.units
import internal.sections
def load_units():
print "Loading units..."
return internal.units.load()
def load_sections(env, units):
print "\nLoading sections..."
return internal.sections.load(env, units)
def create_from_template(template, sections):
return template.render(sections.data)
def check_unaccessed(name, store):
unaccessed_keys = store.get_unaccessed_set()
if len(unaccessed_keys) > 0:
print "Found %s unused %s keys." % (len(unaccessed_keys), name)
print unaccessed_keys
def main(file_stream=None, out_dir=None):
if out_dir and not os.path.exists(out_dir):
os.makedirs(out_dir)
# add a template filter to produce pretty pretty JSON
def jsonify(input, indent=None, pre_whitespace=0):
code = json.dumps(input, indent=indent)
if pre_whitespace:
code = code.replace("\n", ("\n" +" "*pre_whitespace))
return code
def indent(input, indent):
return input.replace("\n", ("\n" + " "*indent))
def wrap(input, wrap=80):
return '\n'.join(textwrap.wrap(input, wrap))
# make Jinja aware of the templates and filters
env = Environment(
loader=FileSystemLoader("templates"),
undefined=StrictUndefined
)
env.filters["jsonify"] = jsonify
env.filters["indent"] = indent
env.filters["wrap"] = wrap
# load up and parse the lowest single units possible: we don't know or care
# which spec section will use it, we just need it there in memory for when
# they want it.
units = load_units()
# use the units to create RST sections
sections = load_sections(env, units)
# print out valid section keys if no file supplied
if not file_stream:
print "\nValid template variables:"
for key in sections.keys():
print " %s" % key
return
# check the input files and substitute in sections where required
print "Parsing input template: %s" % file_stream.name
temp = Template(file_stream.read())
print "Creating output for: %s" % file_stream.name
output = create_from_template(temp, sections)
with open(os.path.join(out_dir, file_stream.name), "w") as f:
f.write(output)
print "Output file for: %s" % file_stream.name
check_unaccessed("units", units)
if __name__ == '__main__':
parser = ArgumentParser(
"Process a file (typically .rst) and replace templated areas with spec"+
" info. For a list of possible template variables, add"+
" --show-template-vars."
)
parser.add_argument(
"file", nargs="?", type=FileType('r'),
help="The input file to process."
)
parser.add_argument(
"--out-directory", "-o", help="The directory to output the file to."+
" Default: /out",
default="out"
)
parser.add_argument(
"--show-template-vars", "-s", action="store_true",
help="Show a list of all possible variables you can use in the"+
" input file."
)
args = parser.parse_args()
if (args.show_template_vars):
main()
sys.exit(0)
if not args.file:
print "No file supplied."
parser.print_help()
sys.exit(1)
main(file_stream=args.file, out_dir=args.out_directory)

@ -0,0 +1,24 @@
from sets import Set
class AccessKeyStore(object):
"""Storage for arbitrary data. Monitors get calls so we know if they
were used or not."""
def __init__(self):
self.data = {}
self.accessed_set = Set()
def keys(self):
return self.data.keys()
def add(self, key, unit_dict):
self.data[key] = unit_dict
def get(self, key):
self.accessed_set.add(key)
return self.data[key]
def get_unaccessed_set(self):
data_list = Set(self.data.keys())
return data_list - self.accessed_set

@ -0,0 +1,31 @@
"""Contains all the sections for the spec."""
from . import AccessKeyStore
import os
def _render_section_room_events(env, units):
template = env.get_template("events.tmpl")
examples = units.get("event-examples")
schemas = units.get("event-schemas")
sections = []
for event_name in schemas:
if not event_name.startswith("m.room"):
continue
sections.append(template.render(
example=examples[event_name],
event=schemas[event_name]
))
return "\n\n".join(sections)
SECTION_DICT = {
"room_events": _render_section_room_events
}
def load(env, units):
store = AccessKeyStore()
for section_key in SECTION_DICT:
section = SECTION_DICT[section_key](env, units)
print "Generated section '%s' : %s" % (
section_key, section[:60].replace("\n","")
)
store.add(section_key, section)
return store

@ -0,0 +1,118 @@
"""Contains all the units for the spec."""
from . import AccessKeyStore
import json
import os
def prop(obj, path):
# Helper method to extract nested property values
nested_keys = path.split("/")
val = obj
for key in nested_keys:
val = val.get(key, {})
return val
def _load_examples():
path = "../event-schemas/examples/v1"
examples = {}
for filename in os.listdir(path):
if not filename.startswith("m."):
continue
with open(os.path.join(path, filename), "r") as f:
examples[filename] = json.loads(f.read())
if filename == "m.room.message_m.text":
examples["m.room.message"] = examples[filename]
return examples
def _load_schemas():
path = "../event-schemas/schema/v1"
schemata = {}
def format_for_obj(obj):
obj_type = "<%s>" % obj.get("type")
if obj_type == "<object>":
if obj.get("properties"):
format = {}
for key in obj.get("properties"):
format[key] = format_for_obj(obj.get("properties")[key])
return format
elif obj.get("additionalProperties"):
return {
"<string>": (
"<%s>" % obj.get("additionalProperties").get("type")
)
}
elif obj_type == "<array>" and obj.get("items"):
return [
format_for_obj(obj.get("items"))
]
enum_text = ""
# add on enum info
enum = obj.get("enum")
if enum:
if len(enum) > 1:
obj_type = "<enum>"
enum_text = " (" + "|".join(enum) + ")"
else:
obj_type = enum[0]
return obj_type + enum_text
for filename in os.listdir(path):
if not filename.startswith("m."):
continue
print "Reading %s" % os.path.join(path, filename)
with open(os.path.join(path, filename), "r") as f:
json_schema = json.loads(f.read())
schema = {
"typeof": None,
"type": None,
"summary": None,
"desc": None,
"json_format": None,
"required_keys": None
}
# add typeof
base_defs = {
"core#/definitions/room_event": "Room Event",
"core#/definitions/state_event": "State Event"
}
if type(json_schema.get("allOf")) == list:
schema["typeof"] = base_defs.get(
json_schema["allOf"][0].get("$ref")
)
# add type
schema["type"] = prop(json_schema, "properties/type/enum")[0]
# add summary and desc
schema["summary"] = json_schema.get("title")
schema["desc"] = json_schema.get("description")
# add json_format
content_props = prop(json_schema, "properties/content")
if content_props:
schema["json_format"] = format_for_obj(content_props)
# add required_keys
schema["required_keys"] = prop(
json_schema, "properties/content/required"
)
schemata[filename] = schema
return schemata
UNIT_DICT = {
"event-examples": _load_examples,
"event-schemas": _load_schemas
}
def load():
store = AccessKeyStore()
for unit_key in UNIT_DICT:
unit = UNIT_DICT[unit_key]()
print "Generated unit '%s' : %s" % (
unit_key, json.dumps(unit)[:50].replace("\n","")
)
store.add(unit_key, unit)
return store

@ -0,0 +1,17 @@
``{{event.type}}``
Summary:
{{event.summary | wrap(76) | indent(4)}}
Type:
{{event.typeof}}
Description:
{{event.desc | wrap(76) | indent(4)}}
Required Keys:
``{{event.required_keys | jsonify}}``
JSON Format::
{{event.json_format | jsonify(4, 4)}}
Example::
{{example.content | jsonify(4, 4)}}
Loading…
Cancel
Save