Kegsay
9 years ago
commit
668cbdf7d0
@ -1 +1,2 @@
|
||||
scripts/gen
|
||||
*.pyc
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
#!/bin/bash -e
|
||||
# Runs z-schema over all of the schema files (looking for matching examples)
|
||||
find schema/v1/m.* | while read line
|
||||
do
|
||||
split_path=(${line///// })
|
||||
event_type=(${split_path[2]})
|
||||
echo "Checking $event_type"
|
||||
echo "--------------------"
|
||||
# match exact name or exact name with a #<something>
|
||||
find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline
|
||||
do
|
||||
echo " against $exline"
|
||||
# run z-schema and dump stdout/err to the terminal (good for Jenkin's Console Output) and grep for fail messages
|
||||
if [[ -n $(z-schema schema/v1/$event_type $exline 2>&1 | tee /dev/tty | grep -Ei "error|failed") ]]; then
|
||||
echo " Failed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
@ -0,0 +1,21 @@
|
||||
This folder contains the templates and templating system for creating the spec.
|
||||
We use the templating system Jinja2 in Python. This was chosen over other
|
||||
systems such as Handlebars.js and Templetor because we already have a Python
|
||||
dependency on the spec build system, and Jinja provides a rich set of template
|
||||
operations beyond basic control flow.
|
||||
|
||||
Installation
|
||||
------------
|
||||
```
|
||||
$ pip install Jinja2
|
||||
```
|
||||
|
||||
Running
|
||||
-------
|
||||
To build the spec:
|
||||
```
|
||||
$ python build.py
|
||||
```
|
||||
|
||||
This will output ``spec.rst`` which can then be fed into the RST->HTML
|
||||
converter located in ``matrix-doc/scripts``.
|
||||
@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Builds the Matrix Specification as restructed text (RST).
|
||||
|
||||
Architecture
|
||||
============
|
||||
+-------+ +----------+
|
||||
| units |-+ | sections |-+
|
||||
+-------+ |-+ === used to create ==> +----------- | === used to create ==> SPEC
|
||||
+-------+ | +----------+
|
||||
+--------+
|
||||
RAW DATA (e.g. json) Blobs of RST
|
||||
|
||||
Units
|
||||
=====
|
||||
Units are random bits of unprocessed data, e.g. schema JSON files. Anything can
|
||||
be done to them, from processing it with Jinja to arbitrary python processing.
|
||||
They are dicts.
|
||||
|
||||
Sections
|
||||
========
|
||||
Sections are short segments of RST. They will be in the final spec, but they
|
||||
are unordered. They typically use a combination of templates + units to
|
||||
construct bits of RST.
|
||||
|
||||
Skeleton
|
||||
========
|
||||
The skeleton is a single RST file which is passed through a templating system to
|
||||
replace variable names with sections.
|
||||
|
||||
Processing
|
||||
==========
|
||||
- Execute all unit functions to load units into memory and process them.
|
||||
- Execute all section functions (which can now be done because the units exist)
|
||||
- Execute the skeleton function to bring it into a single file.
|
||||
|
||||
Checks
|
||||
======
|
||||
- Any units made which were not used at least once will produce a warning.
|
||||
- Any sections made but not used in the skeleton will produce a warning.
|
||||
"""
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
|
||||
from argparse import ArgumentParser, FileType
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import internal.units
|
||||
import internal.sections
|
||||
|
||||
def load_units():
|
||||
print "Loading units..."
|
||||
return internal.units.load()
|
||||
|
||||
def load_sections(env, units):
|
||||
print "\nLoading sections..."
|
||||
return internal.sections.load(env, units)
|
||||
|
||||
def create_from_template(template, sections):
|
||||
return template.render(sections.data)
|
||||
|
||||
def check_unaccessed(name, store):
|
||||
unaccessed_keys = store.get_unaccessed_set()
|
||||
if len(unaccessed_keys) > 0:
|
||||
print "Found %s unused %s keys." % (len(unaccessed_keys), name)
|
||||
print unaccessed_keys
|
||||
|
||||
def main(file_stream=None, out_dir=None):
|
||||
if out_dir and not os.path.exists(out_dir):
|
||||
os.makedirs(out_dir)
|
||||
|
||||
# add a template filter to produce pretty pretty JSON
|
||||
def jsonify(input, indent=None, pre_whitespace=0):
|
||||
code = json.dumps(input, indent=indent)
|
||||
if pre_whitespace:
|
||||
code = code.replace("\n", ("\n" +" "*pre_whitespace))
|
||||
|
||||
return code
|
||||
|
||||
def indent(input, indent):
|
||||
return input.replace("\n", ("\n" + " "*indent))
|
||||
|
||||
def wrap(input, wrap=80):
|
||||
return '\n'.join(textwrap.wrap(input, wrap))
|
||||
|
||||
# make Jinja aware of the templates and filters
|
||||
env = Environment(
|
||||
loader=FileSystemLoader("templates"),
|
||||
undefined=StrictUndefined
|
||||
)
|
||||
env.filters["jsonify"] = jsonify
|
||||
env.filters["indent"] = indent
|
||||
env.filters["wrap"] = wrap
|
||||
|
||||
# load up and parse the lowest single units possible: we don't know or care
|
||||
# which spec section will use it, we just need it there in memory for when
|
||||
# they want it.
|
||||
units = load_units()
|
||||
|
||||
# use the units to create RST sections
|
||||
sections = load_sections(env, units)
|
||||
|
||||
# print out valid section keys if no file supplied
|
||||
if not file_stream:
|
||||
print "\nValid template variables:"
|
||||
for key in sections.keys():
|
||||
print " %s" % key
|
||||
return
|
||||
|
||||
# check the input files and substitute in sections where required
|
||||
print "Parsing input template: %s" % file_stream.name
|
||||
temp = Template(file_stream.read())
|
||||
print "Creating output for: %s" % file_stream.name
|
||||
output = create_from_template(temp, sections)
|
||||
with open(os.path.join(out_dir, file_stream.name), "w") as f:
|
||||
f.write(output)
|
||||
print "Output file for: %s" % file_stream.name
|
||||
|
||||
check_unaccessed("units", units)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(
|
||||
"Process a file (typically .rst) and replace templated areas with spec"+
|
||||
" info. For a list of possible template variables, add"+
|
||||
" --show-template-vars."
|
||||
)
|
||||
parser.add_argument(
|
||||
"file", nargs="?", type=FileType('r'),
|
||||
help="The input file to process."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out-directory", "-o", help="The directory to output the file to."+
|
||||
" Default: /out",
|
||||
default="out"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-template-vars", "-s", action="store_true",
|
||||
help="Show a list of all possible variables you can use in the"+
|
||||
" input file."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if (args.show_template_vars):
|
||||
main()
|
||||
sys.exit(0)
|
||||
|
||||
if not args.file:
|
||||
print "No file supplied."
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
main(file_stream=args.file, out_dir=args.out_directory)
|
||||
@ -0,0 +1,24 @@
|
||||
from sets import Set
|
||||
|
||||
|
||||
class AccessKeyStore(object):
|
||||
"""Storage for arbitrary data. Monitors get calls so we know if they
|
||||
were used or not."""
|
||||
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
self.accessed_set = Set()
|
||||
|
||||
def keys(self):
|
||||
return self.data.keys()
|
||||
|
||||
def add(self, key, unit_dict):
|
||||
self.data[key] = unit_dict
|
||||
|
||||
def get(self, key):
|
||||
self.accessed_set.add(key)
|
||||
return self.data[key]
|
||||
|
||||
def get_unaccessed_set(self):
|
||||
data_list = Set(self.data.keys())
|
||||
return data_list - self.accessed_set
|
||||
@ -0,0 +1,31 @@
|
||||
"""Contains all the sections for the spec."""
|
||||
from . import AccessKeyStore
|
||||
import os
|
||||
|
||||
def _render_section_room_events(env, units):
|
||||
template = env.get_template("events.tmpl")
|
||||
examples = units.get("event-examples")
|
||||
schemas = units.get("event-schemas")
|
||||
sections = []
|
||||
for event_name in schemas:
|
||||
if not event_name.startswith("m.room"):
|
||||
continue
|
||||
sections.append(template.render(
|
||||
example=examples[event_name],
|
||||
event=schemas[event_name]
|
||||
))
|
||||
return "\n\n".join(sections)
|
||||
|
||||
SECTION_DICT = {
|
||||
"room_events": _render_section_room_events
|
||||
}
|
||||
|
||||
def load(env, units):
|
||||
store = AccessKeyStore()
|
||||
for section_key in SECTION_DICT:
|
||||
section = SECTION_DICT[section_key](env, units)
|
||||
print "Generated section '%s' : %s" % (
|
||||
section_key, section[:60].replace("\n","")
|
||||
)
|
||||
store.add(section_key, section)
|
||||
return store
|
||||
@ -0,0 +1,118 @@
|
||||
"""Contains all the units for the spec."""
|
||||
from . import AccessKeyStore
|
||||
import json
|
||||
import os
|
||||
|
||||
def prop(obj, path):
|
||||
# Helper method to extract nested property values
|
||||
nested_keys = path.split("/")
|
||||
val = obj
|
||||
for key in nested_keys:
|
||||
val = val.get(key, {})
|
||||
return val
|
||||
|
||||
def _load_examples():
|
||||
path = "../event-schemas/examples/v1"
|
||||
examples = {}
|
||||
for filename in os.listdir(path):
|
||||
if not filename.startswith("m."):
|
||||
continue
|
||||
with open(os.path.join(path, filename), "r") as f:
|
||||
examples[filename] = json.loads(f.read())
|
||||
if filename == "m.room.message_m.text":
|
||||
examples["m.room.message"] = examples[filename]
|
||||
return examples
|
||||
|
||||
def _load_schemas():
|
||||
path = "../event-schemas/schema/v1"
|
||||
schemata = {}
|
||||
|
||||
def format_for_obj(obj):
|
||||
obj_type = "<%s>" % obj.get("type")
|
||||
if obj_type == "<object>":
|
||||
if obj.get("properties"):
|
||||
format = {}
|
||||
for key in obj.get("properties"):
|
||||
format[key] = format_for_obj(obj.get("properties")[key])
|
||||
return format
|
||||
elif obj.get("additionalProperties"):
|
||||
return {
|
||||
"<string>": (
|
||||
"<%s>" % obj.get("additionalProperties").get("type")
|
||||
)
|
||||
}
|
||||
elif obj_type == "<array>" and obj.get("items"):
|
||||
return [
|
||||
format_for_obj(obj.get("items"))
|
||||
]
|
||||
|
||||
enum_text = ""
|
||||
# add on enum info
|
||||
enum = obj.get("enum")
|
||||
if enum:
|
||||
if len(enum) > 1:
|
||||
obj_type = "<enum>"
|
||||
enum_text = " (" + "|".join(enum) + ")"
|
||||
else:
|
||||
obj_type = enum[0]
|
||||
|
||||
return obj_type + enum_text
|
||||
|
||||
for filename in os.listdir(path):
|
||||
if not filename.startswith("m."):
|
||||
continue
|
||||
print "Reading %s" % os.path.join(path, filename)
|
||||
with open(os.path.join(path, filename), "r") as f:
|
||||
json_schema = json.loads(f.read())
|
||||
schema = {
|
||||
"typeof": None,
|
||||
"type": None,
|
||||
"summary": None,
|
||||
"desc": None,
|
||||
"json_format": None,
|
||||
"required_keys": None
|
||||
}
|
||||
|
||||
# add typeof
|
||||
base_defs = {
|
||||
"core#/definitions/room_event": "Room Event",
|
||||
"core#/definitions/state_event": "State Event"
|
||||
}
|
||||
if type(json_schema.get("allOf")) == list:
|
||||
schema["typeof"] = base_defs.get(
|
||||
json_schema["allOf"][0].get("$ref")
|
||||
)
|
||||
|
||||
# add type
|
||||
schema["type"] = prop(json_schema, "properties/type/enum")[0]
|
||||
|
||||
# add summary and desc
|
||||
schema["summary"] = json_schema.get("title")
|
||||
schema["desc"] = json_schema.get("description")
|
||||
|
||||
# add json_format
|
||||
content_props = prop(json_schema, "properties/content")
|
||||
if content_props:
|
||||
schema["json_format"] = format_for_obj(content_props)
|
||||
|
||||
# add required_keys
|
||||
schema["required_keys"] = prop(
|
||||
json_schema, "properties/content/required"
|
||||
)
|
||||
schemata[filename] = schema
|
||||
return schemata
|
||||
|
||||
UNIT_DICT = {
|
||||
"event-examples": _load_examples,
|
||||
"event-schemas": _load_schemas
|
||||
}
|
||||
|
||||
def load():
|
||||
store = AccessKeyStore()
|
||||
for unit_key in UNIT_DICT:
|
||||
unit = UNIT_DICT[unit_key]()
|
||||
print "Generated unit '%s' : %s" % (
|
||||
unit_key, json.dumps(unit)[:50].replace("\n","")
|
||||
)
|
||||
store.add(unit_key, unit)
|
||||
return store
|
||||
@ -0,0 +1,17 @@
|
||||
``{{event.type}}``
|
||||
Summary:
|
||||
{{event.summary | wrap(76) | indent(4)}}
|
||||
Type:
|
||||
{{event.typeof}}
|
||||
Description:
|
||||
{{event.desc | wrap(76) | indent(4)}}
|
||||
Required Keys:
|
||||
``{{event.required_keys | jsonify}}``
|
||||
|
||||
JSON Format::
|
||||
|
||||
{{event.json_format | jsonify(4, 4)}}
|
||||
|
||||
Example::
|
||||
|
||||
{{example.content | jsonify(4, 4)}}
|
||||
Loading…
Reference in New Issue