Merge pull request #1055 from matrix-org/rav/clean_up_event_schema

Clean up event schema processing

event-schemas/schema/core-event-schema/event.yaml

@@ -3,11 +3,12 @@ properties:
   content:
     description: The fields in this object will vary depending on the type of event.
       When interacting with the REST API, this is the HTTP body.
-    title: EventContent
     type: object
   type:
     description: The type of event. This SHOULD be namespaced similar to Java package
       naming conventions e.g. 'com.example.subdomain.event.type'
     type: string
+required:
+  - type
 title: Event
 type: object

event-schemas/schema/core-event-schema/room_event.yaml

@@ -4,17 +4,17 @@ description: In addition to the Event fields, Room Events have the following add
   fields.
 properties:
   event_id:
-    description: Required. The globally unique event identifier.
+    description: The globally unique event identifier.
     type: string
   room_id:
-    description: Required. The ID of the room associated with this event.
+    description: The ID of the room associated with this event.
     type: string
   sender:
-    description: Required. Contains the fully-qualified ID of the user who *sent*
+    description: Contains the fully-qualified ID of the user who *sent*
       this event.
     type: string
   origin_server_ts:
-    description: Required. Timestamp in milliseconds on originating homeserver
+    description: Timestamp in milliseconds on originating homeserver
       when this event was sent.
     type: number
   unsigned:

templating/batesian/units.py

@@ -12,11 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Parent class for writing units."""
-from . import AccessKeyStore
 import inspect
-import json
+
-import os
-import subprocess
 
 class Units(object):
 
@@ -57,9 +54,6 @@ class Units(object):
                 unit_dict[unit_key] = func(self.substitutions)
             else:
                 unit_dict[unit_key] = func()
-            self.log("Generated unit '%s' : %s" % (
+            self.log("Generated unit '%s'" % unit_key)
-                unit_key, json.dumps(unit_dict[unit_key])[:50].replace(
+
-                    "\n",""
-                )
-            ))
         return unit_dict

templating/build.py

@@ -59,10 +59,12 @@ import importlib
 import json
 import logging
 import os
-import re
 import sys
 from textwrap import TextWrapper
 
+from matrix_templates.units import TypeTableRow
+
+
 def create_from_template(template, sections):
     return template.render(sections)
 
@@ -117,15 +119,23 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
         Given a list of rows, returns a list giving the maximum length of the
         values in each column.
 
-        :param list[dict[str, str]] input: a list of rows. Each row should be a
+        :param list[TypeTableRow|dict[str,str]] input:
-           dict with the keys given in ``keys``.
+            a list of rows
         :param list[str] keys: the keys corresponding to the table columns
         :param list[int] defaults: for each column, the default column width.
         :param int default_width: if ``defaults`` is shorter than ``keys``, this
            will be used as a fallback
         """
+        def getrowattribute(row, k):
+            # the row may be a dict (particularly the title row, which is
+            # generated by the template
+            if not isinstance(row, TypeTableRow):
+                return row[k]
+            return getattr(row, k)
+
         def colwidth(key, default):
-            return reduce(max, (len(row[key]) for row in input),
+            rowwidths = (len(getrowattribute(row, key)) for row in input)
+            return reduce(max, rowwidths,
                           default if default is not None else default_width)
 
         results = map(colwidth, keys, defaults)

templating/matrix_templates/templates/common-event-fields.tmpl

@@ -3,6 +3,11 @@
 {{common_event.title}} Fields
 {{(7 + common_event.title | length) * title_kind}}
 
-{{common_event.desc | wrap(80)}}
+{{common_event.desc}}
 
-{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }}
+{% for table in common_event.tables %}
+{{"``"+table.title+"``" if table.title else "" }}
+
+{{ tables.paramtable(table.rows, ["Key", "Type", "Description"]) }}
+
+{% endfor %}

templating/matrix_templates/templates/http-api.tmpl

@@ -27,10 +27,10 @@ Request format:
 `No parameters`
 {% endif %}
 
-{% if endpoint.res_headers|length > 0 -%}
+{% if endpoint.res_headers is not none -%}
 Response headers:
 
-{{ tables.paramtable(endpoint.res_headers) }}
+{{ tables.paramtable(endpoint.res_headers.rows) }}
 {% endif -%}
 
 {% if endpoint.res_tables|length > 0 -%}

templating/matrix_templates/templates/tables.tmpl

@@ -6,8 +6,7 @@
 {#
  # write a table for a list of parameters.
  #
- # 'rows' is the list of parameters. Each row should have the keys
+ # 'rows' is the list of parameters. Each row should be a TypeTableRow.
- # 'key', 'type', and 'desc'.
  #}
 {% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%}
 {{ split_paramtable({None: rows}, titles) }}
@@ -21,11 +20,11 @@
  # As a special case, if a key of 'rows_by_loc' is 'None', no title row is
  # written for that location. This is used by the standard 'paramtable' macro.
  #}
-{% macro split_paramtable(rows_by_loc, 
+{% macro split_paramtable(rows_by_loc,
       titles=["Parameter", "Type", "Description"]) -%}
 
-{% set rowkeys = ['key', 'type', 'desc'] %}
+{% set rowkeys = ['key', 'title', 'desc'] %}
-{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %}
+{% set titlerow = {'key': titles[0], 'title': titles[1], 'desc': titles[2]} %}
 
 {# We need the rows flattened into a single list. Abuse the 'sum' filter to
  # join arrays instead of add numbers. -#}
@@ -34,7 +33,7 @@
 {# Figure out the widths of the columns. The last column is always 50 characters
  # wide; the others default to 10, but stretch if there is wider text in the
  # column. -#}
-{% set fieldwidths = (([titlerow] + flatrows) | 
+{% set fieldwidths = (([titlerow] + flatrows) |
         fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%}
 
 {{ tableheader(fieldwidths) }}
@@ -57,7 +56,7 @@
 
 
 {#
- # Write a table header row, for the given column widths 
+ # Write a table header row, for the given column widths
  #}
 {% macro tableheader(widths) -%}
 {% for arg in widths -%}
@@ -67,7 +66,7 @@
 
 
 {#
- # Write a normal table row. Each of 'widths' and 'keys' should be sequences 
+ # Write a normal table row. Each of 'widths' and 'keys' should be sequences
  # of the same length; 'widths' defines the column widths, and 'keys' the
  # attributes of 'row' to look up for values to put in the columns.
  #}
@@ -81,7 +80,7 @@
    {# the last column needs wrapping and indenting (by the sum of the widths of
    the preceding columns, plus the number of preceding columns (for the
    separators)) -#}
-   {{ value | wrap(widths[loop.index0]) | 
+   {{ value | wrap(widths[loop.index0]) |
          indent_block(widths[0:-1]|sum + loop.index0) -}}
 {% endif -%}
 {% endfor -%}
@@ -90,7 +89,7 @@
 
 
 
-{# 
+{#
  # write a tablespan row. This is a single value which spans the entire table.
  #}
 {% macro tablespan(widths, value) -%}

templating/matrix_templates/units.py

@@ -62,6 +62,50 @@ OrderedLoader.add_constructor(
     yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
     construct_mapping)
 
+
+class TypeTable(object):
+    """Describes a table documenting an object type
+
+    Attributes:
+        title(str|None): Title of the table - normally the object type
+        desc(str|None): description of the object
+        rows(list[TypeTableRow]): the rows in the table
+    """
+    def __init__(self, title=None, desc=None, rows=[]):
+        self.title=title
+        self.desc=desc
+        self._rows = []
+        for row in rows:
+            self.add_row(row)
+
+    def add_row(self, row):
+        if not isinstance(row, TypeTableRow):
+            raise ValueError("Can only add TypeTableRows to TypeTable")
+
+        self._rows.append(row)
+
+    def __getattr__(self, item):
+        if item == 'rows':
+            return list(self._rows)
+        return super(TypeTable, self).__getattr__(item)
+
+    def __repr__(self):
+        return "TypeTable[%s, rows=%s]" % (self.title, self._rows)
+
+
+class TypeTableRow(object):
+    """Describes an object field defined in the json schema
+    """
+    def __init__(self, key, title, desc, required=False):
+        self.key = key
+        self.title = title
+        self.desc = desc
+        self.required = required
+
+    def __repr__(self):
+        return "TypeTableRow[%s: %s]" % (self.key, self.desc)
+
+
 def resolve_references(path, schema):
     if isinstance(schema, dict):
         # do $ref first
@@ -99,10 +143,16 @@ def inherit_parents(obj):
     # iterate through the parents first, and then overwrite with the settings
     # from the child.
     for p in map(inherit_parents, parents) + [obj]:
-        for key in ('title', 'type', 'required', 'description'):
+        # child blats out type, title and description
+        for key in ('type', 'title', 'description'):
             if p.get(key):
                 result[key] = p[key]
 
+        # other fields get merged
+        for key in ('required', ):
+            if p.get(key):
+                result.setdefault(key, []).extend(p[key])
+
         for key in ('properties', 'additionalProperties', 'patternProperties'):
             if p.get(key):
                 result.setdefault(key, OrderedDict()).update(p[key])
@@ -111,6 +161,21 @@ def inherit_parents(obj):
 
 
 def get_json_schema_object_fields(obj, enforce_title=False):
+    """Parse a JSON schema object definition
+
+    Args:
+        obj(dict): definition from the JSON schema file. $refs should already
+            have been resolved.
+        enforce_title (bool): if True, and the definition has no "title",
+            the 'title' result will be set to 'NO_TITLE' (otherwise it will be
+            set to None)
+
+    Returns:
+        dict: with the following fields:
+          - title (str): title (normally the type name) for the object
+          - tables (list[TypeTable]): list of the tables for the type
+                definition
+    """
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
     if obj.get("type") != "object":
@@ -131,7 +196,7 @@ def get_json_schema_object_fields(obj, enforce_title=False):
         key_type = additionalProps.get("x-pattern", "string")
         res = process_data_type(additionalProps)
         return {
-            "type": "{%s: %s}" % (key_type, res["type"]),
+            "title": "{%s: %s}" % (key_type, res["title"]),
             "tables": res["tables"],
         }
 
@@ -151,7 +216,7 @@ def get_json_schema_object_fields(obj, enforce_title=False):
     # doing all the keys.
     if not props:
         return {
-            "type": obj_title if obj_title else 'object',
+            "title": obj_title if obj_title else 'object',
             "tables": [],
         }
 
@@ -171,12 +236,12 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             required = key_name in required_keys
             res = process_data_type(props[key_name], required)
 
-            first_table_rows.append({
+            first_table_rows.append(TypeTableRow(
-                "key": key_name,
+                key=key_name,
-                "type": res["type"],
+                title=res["title"],
-                "required": required,
+                required=required,
-                "desc": res["desc"],
+                desc=res["desc"],
-            })
+            ))
             tables.extend(res["tables"])
             logger.debug("Done property %s" % key_name)
 
@@ -187,19 +252,19 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             # we don't lose information about where the error occurred.
             raise e2, None, sys.exc_info()[2]
 
-    tables.insert(0, {
+    tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows))
-        "title": obj_title,
+
-        "rows": first_table_rows,
+    for table in tables:
-    })
+        assert isinstance(table, TypeTable)
 
     return {
-        "type": obj_title,
+        "title": obj_title,
         "tables": tables,
     }
 
 
 # process a data type definition. returns a dictionary with the keys:
-# type:     stringified type name
+# title:     stringified type name
 # desc:     description
 # enum_desc: description of permissible enum fields
 # is_object: true if the data type is an object
@@ -217,19 +282,22 @@ def process_data_type(prop, required=False, enforce_title=True):
             prop,
             enforce_title=enforce_title,
         )
-        prop_type = res["type"]
+        prop_title = res["title"]
         tables = res["tables"]
         is_object = True
 
     elif prop_type == "array":
         nested = process_data_type(prop["items"])
-        prop_type = "[%s]" % nested["type"]
+        prop_title = "[%s]" % nested["title"]
         tables = nested["tables"]
         enum_desc = nested["enum_desc"]
 
+    else:
+        prop_title = prop_type
+
     if prop.get("enum"):
         if len(prop["enum"]) > 1:
-            prop_type = "enum"
+            prop_title = "enum"
             enum_desc = (
                 "One of: %s" % json.dumps(prop["enum"])
             )
@@ -238,15 +306,17 @@ def process_data_type(prop, required=False, enforce_title=True):
                 "Must be '%s'." % prop["enum"][0]
             )
 
-    if isinstance(prop_type, list):
+    if isinstance(prop_title, list):
-        prop_type = " or ".join(prop_type)
+        prop_title = " or ".join(prop_title)
-
 
     rq = "**Required.**" if required else None
     desc = " ".join(x for x in [rq, prop.get("description"), enum_desc] if x)
 
+    for table in tables:
+        assert isinstance(table, TypeTable)
+
     return {
-        "type": prop_type,
+        "title": prop_title,
         "desc": desc,
         "enum_desc": enum_desc,
         "is_object": is_object,
@@ -263,13 +333,10 @@ def deduplicate_tables(tables):
     titles = set()
     filtered = []
     for table in reversed(tables):
-        if table.get("no-table"):
+        if table.title in titles:
-            continue
-
-        if table.get("title") in titles:
             continue
 
-        titles.add(table.get("title"))
+        titles.add(table.title)
         filtered.append(table)
     filtered.reverse()
 
@@ -286,14 +353,10 @@ def get_tables_for_response(schema):
     # make up the first table, with just the 'body' row in, unless the response
     # is an object, in which case there's little point in having one.
     if not pv["is_object"]:
-        tables = [{
+        first_table_row = TypeTableRow(
-            "title": None,
+            key="<body>", title=pv["title"], desc=pv["desc"],
-            "rows": [{
+        )
-                "key": "<body>",
+        tables.insert(0, TypeTable(None, rows=[first_table_row]))
-                "type": pv["type"],
-                "desc": pv["desc"],
-            }]
-        }] + tables
 
     logger.debug("response: %r" % tables)
 
@@ -383,9 +446,9 @@ class MatrixUnits(Units):
 
                     endpoints.append(endpoint)
                 except Exception as e:
-                    raise Exception(
+                    logger.error("Error handling endpoint %s %s: %s",
-                        "Error handling endpoint %s %s: %s" % (method, path, e),
+                                 method, path, e)
-                    )
+                    raise
         return {
             "base": api.get("basePath").rstrip("/"),
             "group": group_name,
@@ -404,7 +467,7 @@ class MatrixUnits(Units):
             "rate_limited": 429 in endpoint_swagger.get("responses", {}),
             "req_param_by_loc": {},
             "req_body_tables": [],
-            "res_headers": [],
+            "res_headers": None,
             "res_tables": [],
             "responses": [],
             "example": {
@@ -440,11 +503,9 @@ class MatrixUnits(Units):
                         " One of: %s" % json.dumps(param.get("enum"))
                     )
 
-                endpoint["req_param_by_loc"].setdefault(param_loc, []).append({
+                endpoint["req_param_by_loc"].setdefault(param_loc, []).append(
-                    "key": param_name,
+                    TypeTableRow(key=param_name, title=val_type, desc=desc),
-                    "type": val_type,
+                )
-                    "desc": desc
-                })
 
                 example = get_example_for_param(param)
                 if example is None:
@@ -484,14 +545,12 @@ class MatrixUnits(Units):
                     good_response["schema"]
                 )
             if "headers" in good_response:
-                headers = []
+                headers = TypeTable()
-                for (header_name, header) in good_response[
+                for (header_name, header) in good_response["headers"].iteritems():
-                    "headers"].iteritems():
+                    headers.add_row(
-                    headers.append({
+                        TypeTableRow(key=header_name, title=header["type"],
-                        "key": header_name,
+                                     desc=header["description"]),
-                        "type": header["type"],
+                    )
-                        "desc": header["description"],
-                    })
                 endpoint["res_headers"] = headers
         query_string = "" if len(
             example_query_params) == 0 else "?" + urllib.urlencode(
@@ -531,7 +590,7 @@ class MatrixUnits(Units):
             # put the top-level parameters into 'req_param_by_loc', and the others
             # into 'req_body_tables'
             body_params = endpoint_data['req_param_by_loc'].setdefault("JSON body",[])
-            body_params.extend(req_body_tables[0]["rows"])
+            body_params.extend(req_body_tables[0].rows)
 
             body_tables = req_body_tables[1:]
             endpoint_data['req_body_tables'].extend(body_tables)
@@ -565,70 +624,64 @@ class MatrixUnits(Units):
         return apis
 
     def load_common_event_fields(self):
+        """Parse the core event schema files
+
+        Returns:
+            dict: with the following properties:
+                "title": Event title (from the 'title' field of the schema)
+                "desc": desc
+                "tables": list[TypeTable]
+        """
         path = CORE_EVENT_SCHEMA
         event_types = {}
 
-        for (root, dirs, files) in os.walk(path):
+        for filename in os.listdir(path):
-            for filename in files:
+            if not filename.endswith(".yaml"):
-                if not filename.endswith(".yaml"):
+                continue
-                    continue
+
+            filepath = os.path.join(path, filename)
+
+            event_type = filename[:-5]  # strip the ".yaml"
+            logger.info("Reading event schema: %s" % filepath)
+
+            with open(filepath) as f:
+                event_schema = yaml.load(f, OrderedLoader)
 
-                event_type = filename[:-5]  # strip the ".yaml"
+            schema_info = process_data_type(
-                filepath = os.path.join(root, filename)
+                event_schema,
-                with open(filepath) as f:
+                enforce_title=True,
-                    try:
+            )
-                        event_info = yaml.load(f, OrderedLoader)
+            event_types[event_type] = schema_info
-                    except Exception as e:
-                        raise ValueError(
-                            "Error reading file %r" % (filepath,), e
-                        )
-
-                if "event" not in event_type:
-                    continue  # filter ImageInfo and co
-
-                table = {
-                    "title": event_info["title"],
-                    "desc": event_info["description"],
-                    "rows": []
-                }
-
-                for prop in sorted(event_info["properties"]):
-                    row = {
-                        "key": prop,
-                        "type": event_info["properties"][prop]["type"],
-                        "desc": event_info["properties"][prop].get("description","")
-                    }
-                    table["rows"].append(row)
-
-                event_types[event_type] = table
         return event_types
 
     def load_apis(self, substitutions):
         cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable")
         fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable")
-        return {
+
-            "rows": [{
+        # we abuse the typetable to return this info to the templates
-                "key": "`Client-Server API <client_server/"+cs_ver+".html>`_",
+        return TypeTable(rows=[
-                "type": cs_ver,
+            TypeTableRow(
-                "desc": "Interaction between clients and servers",
+                "`Client-Server API <client_server/"+cs_ver+".html>`_",
-            }, {
+                cs_ver,
-                "key": "`Server-Server API <server_server/"+fed_ver+".html>`_",
+                "Interaction between clients and servers",
-                "type": fed_ver,
+            ), TypeTableRow(
-                "desc": "Federation between servers",
+                "`Server-Server API <server_server/"+fed_ver+".html>`_",
-            }, {
+                fed_ver,
-                "key": "`Application Service API <application_service/unstable.html>`_",
+                "Federation between servers",
-                "type": "unstable",
+            ), TypeTableRow(
-                "desc": "Privileged server plugins",
+                "`Application Service API <application_service/unstable.html>`_",
-            }, {
+                "unstable",
-                "key": "`Identity Service API <identity_service/unstable.html>`_",
+                "Privileged server plugins",
-                "type": "unstable",
+            ), TypeTableRow(
-                "desc": "Mapping of third party IDs to Matrix IDs",
+                "`Identity Service API <identity_service/unstable.html>`_",
-            }, {
+                "unstable",
-                "key": "`Push Gateway API <push_gateway/unstable.html>`_",
+                "Mapping of third party IDs to Matrix IDs",
-                "type": "unstable",
+            ), TypeTableRow(
-                "desc": "Push notifications for Matrix events",
+                "`Push Gateway API <push_gateway/unstable.html>`_",
-            }]
+                "unstable",
-        }
+                "Push notifications for Matrix events",
+            ),
+        ])
 
     def load_event_examples(self):
         path = EVENT_EXAMPLES
@@ -673,24 +726,23 @@ class MatrixUnits(Units):
             json_schema = yaml.load(f, OrderedLoader)
 
         schema = {
+            # one of "Message Event" or "State Event"
             "typeof": "",
             "typeof_info": "",
+
+            # event type, eg "m.room.member". Note *not* the type of the
+            # event object (which should always be 'object').
             "type": None,
             "title": None,
             "desc": None,
             "msgtype": None,
             "content_fields": [
-                # {
+                # <TypeTable>
-                #   title: "<title> key"
-                #   rows: [
-                #       { key: <key_name>, type: <string>,
-                #         desc: <desc>, required: <bool> }
-                #   ]
-                # }
             ]
         }
 
-        # add typeof
+        # before we resolve the references, see if the first reference is to
+        # the message event or state event schemas, and add typeof info if so.
         base_defs = {
             ROOM_EVENT: "Message Event",
             STATE_EVENT: "State Event"