Merge pull request #59 from matrix-org/spec-restructure-modules

Add spec build targets; restructure spec

scripts/gendoc.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python
 
 from docutils.core import publish_file
+import copy
 import fileinput
 import glob
 import os
@@ -8,6 +9,7 @@ import re
 import shutil
 import subprocess
 import sys
+import yaml
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
@@ -15,67 +17,216 @@ stylesheets = {
     "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"]
 }
 
-title_style_matchers = {
-    "=": re.compile("^=+$"),
-    "-": re.compile("^-+$")
-}
-TOP_LEVEL = "="
-SECOND_LEVEL = "-"
-FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$")
 
+"""
+Read a RST file and replace titles with a different title level if required.
+Args:
+    filename: The name of the file being read (for debugging)
+    file_stream: The open file stream to read from.
+    title_level: The integer which determines the offset to *start* from.
+    title_styles: An array of characters detailing the right title styles to use
+                  e.g. ["=", "-", "~", "+"]
+Returns:
+    string: The file contents with titles adjusted.
+Example:
+    Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file
+    when read line-by-line encounters the titles "===", "---", "---", "===", "---".
+    This function will bump every title encountered down a sub-heading e.g.
+    "=" to "-" and "-" to "~" because title_level = 1, so the output would be
+    "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level.
+"""
+def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
+    rst_lines = []
+    title_chars = "".join(title_styles)
+    title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$")
+
+    prev_line_title_level = 0 # We expect the file to start with '=' titles
+    file_offset = None
+    prev_non_title_line = None
+    for i, line in enumerate(file_stream, 1):
+        # ignore anything which isn't a title (e.g. '===============')
+        if not title_regex.match(line):
+            rst_lines.append(line)
+            prev_non_title_line = line
+            continue
+        # The title underline must match at a minimum the length of the title
+        if len(prev_non_title_line) > len(line):
+            rst_lines.append(line)
+            prev_non_title_line = line
+            continue
+
+        line_title_style = line[0]
+        line_title_level = title_styles.index(line_title_style)
+
+        # Not all files will start with "===" and we should be flexible enough
+        # to allow that. The first title we encounter sets the "file offset"
+        # which is added to the title_level desired.
+        if file_offset is None:
+            file_offset = line_title_level
+            if file_offset != 0:
+                print ("     WARNING: %s starts with a title style of '%s' but '%s' " +
+                    "is preferable.") % (filename, line_title_style, title_styles[0])
 
-def check_valid_section(filename, section):
+        # Sanity checks: Make sure that this file is obeying the title levels
-    if not re.match(FILE_FORMAT_MATCHER, filename):
+        # specified and bail if it isn't.
+        # The file is allowed to go 1 deeper or any number shallower
+        if prev_line_title_level - line_title_level < -1:
             raise Exception(
-            "The filename of " + filename + " does not match the expected format " +
+                ("File '%s' line '%s' has a title " +
-            "of '##_##_words-go-here.rst'"
+                "style '%s' which doesn't match one of the " +
+                "allowed title styles of %s because the " +
+                "title level before this line was '%s'") %
+                (filename, (i + 1), line_title_style, title_styles,
+                title_styles[prev_line_title_level])
             )
+        prev_line_title_level = line_title_level
 
-    # we need TWO new lines else the next file's title gets merged
+        adjusted_level = (
-    # the last paragraph *WITHOUT RST PRODUCING A WARNING*
+            title_level + line_title_level - file_offset
-    if not section[-2:] == "\n\n":
-        raise Exception(
-            "The file " + filename + " does not end with 2 new lines."
         )
 
-    # Enforce some rules to reduce the risk of having mismatched title
+        # Sanity check: Make sure we can bump down the title and we aren't at the
-    # styles.
+        # lowest level already
-    title_line = section.split("\n")[1]
+        if adjusted_level >= len(title_styles):
-    if title_line != (len(title_line) * title_line[0]):
             raise Exception(
-            "The file " + filename + " doesn't have a title style line on line 2"
+                ("Files '%s' line '%s' has a sub-title level too low and it " +
+                "cannot be adjusted to fit. You can add another level to the " +
+                "'title_styles' key in targets.yaml to fix this.") %
+                (filename, (i + 1))
             )
 
-    # anything marked as xx_00_ is the start of a new top-level section
+        if adjusted_level == line_title_level:
-    if re.match("^[0-9]+_00_", filename):
+            # no changes required
-        if not title_style_matchers[TOP_LEVEL].match(title_line):
+            rst_lines.append(line)
+            continue
+
+        # Adjusting line levels
+        # print (
+        #     "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" %
+        #     (filename, line_title_style,
+        #         title_styles[adjusted_level],
+        #         file_offset, title_level)
+        # )
+        rst_lines.append(line.replace(
+            line_title_style,
+            title_styles[adjusted_level]
+        ))
+            
+    return "".join(rst_lines)
+
+
+def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
+    # string are file paths to RST blobs
+    if isinstance(file_info, basestring):
+        print "%s %s" % (">" * (1 + title_level), file_info)
+        with open(os.path.join(spec_dir, file_info), "r") as f:
+            rst = None
+            if adjust_titles:
+                rst = load_with_adjusted_titles(
+                    file_info, f, title_level, title_styles
+                )
+            else:
+                rst = f.read()
+            if rst[-2:] != "\n\n":
                 raise Exception(
-                "The file " + filename + " is a top-level section because it matches " +
+                    ("File %s should end with TWO new-line characters to ensure " +
-                "the filename format ##_00_something.rst but has the wrong title " +
+                    "file concatenation works correctly.") % (file_info,)
-                "style: expected '" + TOP_LEVEL + "' but got '" +
+                )
-                title_line[0] + "'"
+            return rst
-            )
+    # dicts look like {0: filepath, 1: filepath} where the key is the title level
-    # anything marked as xx_xx_ is the start of a sub-section
+    elif isinstance(file_info, dict):
-    elif re.match("^[0-9]+_[0-9]{2}_", filename):
+        levels = sorted(file_info.keys())
-        if not title_style_matchers[SECOND_LEVEL].match(title_line):
+        rst = []
+        for l in levels:
+            rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles))
+        return "".join(rst)
+    # lists are multiple file paths e.g. [filepath, filepath]
+    elif isinstance(file_info, list):
+        rst = []
+        for f in file_info:
+            rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles))
+        return "".join(rst)
     raise Exception(
-                "The file " + filename + " is a 2nd-level section because it matches " +
+        "The following 'file' entry in this target isn't a string, list or dict. " +
-                "the filename format ##_##_something.rst but has the wrong title " +
+        "It really really should be. Entry: %s" % (file_info,)
-                "style: expected '" + SECOND_LEVEL + "' but got '" +
+    )
-                title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " +
+
-                "then use the form '##_##b_something.rst' which will not apply this " +
+
-                "check."
+def build_spec(target, out_filename):
-            )
+    with open(out_filename, "wb") as outfile:
-
+        for file_info in target["files"]:
-def cat_spec_sections_to(out_file_name):
+            section = get_rst(
-    with open(out_file_name, "wb") as outfile:
+                file_info=file_info,
-        for f in sorted(glob.glob("../specification/*.rst")):
+                title_level=0,
-            with open(f, "rb") as infile:
+                title_styles=target["title_styles"],
-                section = infile.read()
+                spec_dir="../specification/",
-                check_valid_section(os.path.basename(f), section)
+                adjust_titles=True
+            )
             outfile.write(section)
 
 
+"""
+Replaces relative title styles with actual title styles.
+
+The templating system has no idea what the right title style is when it produces
+RST because it depends on the build target. As a result, it uses relative title
+styles defined in targets.yaml to say "down a level, up a level, same level".
+
+This function replaces these relative titles with actual title styles from the
+array in targets.yaml.
+"""
+def fix_relative_titles(target, filename, out_filename):
+    title_styles = target["title_styles"]
+    relative_title_chars = [
+        target["relative_title_styles"]["subtitle"],
+        target["relative_title_styles"]["sametitle"],
+        target["relative_title_styles"]["supertitle"]
+    ]
+    relative_title_matcher = re.compile(
+        "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$"
+    )
+    title_matcher = re.compile(
+        "^[" + re.escape("".join(title_styles)) + "]{3,}$"
+    )
+    current_title_style = None
+    with open(filename, "r") as infile:
+        with open(out_filename, "w") as outfile:
+            for line in infile.readlines():
+                if not relative_title_matcher.match(line):
+                    if title_matcher.match(line):
+                        current_title_style = line[0]
+                    outfile.write(line)
+                    continue
+                line_char = line[0]
+                replacement_char = None
+                current_title_level = title_styles.index(current_title_style)
+                if line_char == target["relative_title_styles"]["subtitle"]:
+                    if (current_title_level + 1) == len(title_styles):
+                        raise Exception(
+                            "Encountered sub-title line style but we can't go " +
+                            "any lower."
+                        )
+                    replacement_char = title_styles[current_title_level + 1]
+                elif line_char == target["relative_title_styles"]["sametitle"]:
+                    replacement_char = title_styles[current_title_level]
+                elif line_char == target["relative_title_styles"]["supertitle"]:
+                    if (current_title_level - 1) < 0:
+                        raise Exception(
+                            "Encountered super-title line style but we can't go " +
+                            "any higher."
+                        )
+                    replacement_char = title_styles[current_title_level - 1]
+                else:
+                    raise Exception(
+                        "Unknown relative line char %s" % (line_char,)
+                    )
+
+                outfile.write(
+                    line.replace(line_char, replacement_char)
+                )
+
+
+
 def rst2html(i, o):
     with open(i, "r") as in_file:
         with open(o, "w") as out_file:
@@ -88,13 +239,14 @@ def rst2html(i, o):
                 settings_overrides=stylesheets
             )
 
+
 def run_through_template(input):
     tmpfile = './tmp/output'
     try:
         with open(tmpfile, 'w') as out:
-            subprocess.check_output(
+            print subprocess.check_output(
                 [
-                    'python', 'build.py',
+                    'python', 'build.py', "-v",
                     "-i", "matrix_templates",
                     "-o", "../scripts/tmp",
                     "../scripts/"+input
@@ -107,6 +259,95 @@ def run_through_template(input):
             sys.stderr.write(f.read() + "\n")
         raise
 
+
+"""
+Extract and resolve groups for the given target in the given targets listing.
+Args:
+    targets_listing (str): The path to a YAML file containing a list of targets
+    target_name (str): The name of the target to extract from the listings.
+Returns:
+    dict: Containing "filees" (a list of file paths), "relative_title_styles"
+          (a dict of relative style keyword to title character) and "title_styles"
+          (a list of characters which represent the global title style to follow,
+           with the top section title first, the second section second, and so on.)
+"""
+def get_build_target(targets_listing, target_name):
+    build_target = {
+        "title_styles": [],
+        "relative_title_styles": {},
+        "files": []
+    }
+    with open(targets_listing, "r") as targ_file:
+        all_targets = yaml.load(targ_file.read())
+
+    build_target["title_styles"] = all_targets["title_styles"]
+    build_target["relative_title_styles"] = all_targets["relative_title_styles"]
+    target = all_targets["targets"].get(target_name)
+    if not target:
+        raise Exception(
+            "No target by the name '" + target_name + "' exists in '" +
+            targets_listing + "'."
+        )
+    if not isinstance(target.get("files"), list):
+        raise Exception(
+            "Found target but 'files' key is not a list."
+        )
+
+    def get_group(group_id, depth):
+        group_name = group_id[len("group:"):]
+        group = all_targets.get("groups", {}).get(group_name)
+        if not group:
+            raise Exception(
+                "Tried to find group '%s' but it doesn't exist." % group_name
+            )
+        if not isinstance(group, list):
+            raise Exception(
+                "Expected group '%s' to be a list but it isn't." % group_name
+            )
+        # deep copy so changes to depths don't contaminate multiple uses of this group
+        group = copy.deepcopy(group)
+        # swap relative depths for absolute ones
+        for i, entry in enumerate(group):
+            if isinstance(entry, dict):
+                group[i] = {
+                    (rel_depth + depth): v for (rel_depth, v) in entry.items()
+                }
+        return group
+
+    resolved_files = []
+    for file_entry in target["files"]:
+        # file_entry is a group id
+        if isinstance(file_entry, basestring) and file_entry.startswith("group:"):
+            group = get_group(file_entry, 0)
+            # The group may be resolved to a list of file entries, in which case
+            # we want to extend the array to insert each of them rather than
+            # insert the entire list as a single element (which is what append does)
+            if isinstance(group, list):
+                resolved_files.extend(group)
+            else:
+                resolved_files.append(group)
+        # file_entry is a dict which has more file entries as values
+        elif isinstance(file_entry, dict):
+            resolved_entry = {}
+            for (depth, entry) in file_entry.iteritems():
+                if not isinstance(entry, basestring):
+                    raise Exception(
+                        "Double-nested depths are not supported. Entry: %s" % (file_entry,)
+                    )
+                if entry.startswith("group:"):
+                    resolved_entry[depth] = get_group(entry, depth)
+                else:
+                    # map across without editing (e.g. normal file path)
+                    resolved_entry[depth] = entry
+            resolved_files.append(resolved_entry)
+            continue
+        # file_entry is just a plain ol' file path
+        else:
+            resolved_files.append(file_entry)
+    build_target["files"] = resolved_files
+    return build_target
+
+
 def prepare_env():
     try:
         os.makedirs("./gen")
@@ -117,13 +358,21 @@ def prepare_env():
     except OSError:
         pass
 
+
 def cleanup_env():
     shutil.rmtree("./tmp")
 
-def main():
+
+def main(target_name):
     prepare_env()
-    cat_spec_sections_to("tmp/full_spec.rst")
+    print "Building spec [target=%s]" % target_name
-    run_through_template("tmp/full_spec.rst")
+    target = get_build_target("../specification/targets.yaml", target_name)
+    build_spec(target=target, out_filename="tmp/templated_spec.rst")
+    run_through_template("tmp/templated_spec.rst")
+    fix_relative_titles(
+        target=target, filename="tmp/templated_spec.rst",
+        out_filename="tmp/full_spec.rst"
+    )
     shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
     run_through_template("tmp/howto.rst")
     rst2html("tmp/full_spec.rst", "gen/specification.html")
@@ -131,6 +380,7 @@ def main():
     if "--nodelete" not in sys.argv:
         cleanup_env()
 
+
 if __name__ == '__main__':
     if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]:
         # we accept almost no args, so they don't know what they're doing!
@@ -145,4 +395,4 @@ if __name__ == '__main__':
         print "Requirements:"
         print " - This script requires Jinja2 and rst2html (docutils)."
         sys.exit(0)
-    main()
+    main("main")

specification/0-events.rst

@@ -0,0 +1,38 @@
+Events
+======
+
+All communication in Matrix is expressed in the form of data objects called
+Events. These are the fundamental building blocks common to the client-server,
+server-server and application-service APIs, and are described below.
+
+{{common_event_fields}}
+
+{{common_room_event_fields}}
+
+{{common_state_event_fields}}
+
+
+Room Events
+-----------
+.. NOTE::
+  This section is a work in progress.
+
+This specification outlines several standard event types, all of which are
+prefixed with ``m.``
+
+{{m_room_aliases_event}}
+
+{{m_room_canonical_alias_event}}
+
+{{m_room_create_event}}
+
+{{m_room_history_visibility_event}}
+
+{{m_room_join_rules_event}}
+
+{{m_room_member_event}}
+
+{{m_room_power_levels_event}}
+
+{{m_room_redaction_event}}
+

specification/0-feature_profiles.rst

@@ -0,0 +1,3 @@
+Feature Profiles
+================
+

specification/1-client_server_api.rst

@@ -1054,12 +1054,6 @@ medium
 address
   The textual address of the 3pid, eg. the email address
 
-Presence
---------
-.. TODO-spec
-  - Define how users receive presence invites, and how they accept/decline them
-
-{{presence_http_api}}
 
 Profiles
 --------

specification/2-modules.rst

@@ -0,0 +1,3 @@
+Modules
+=======
+

specification/3-application_service_api.rst

@@ -401,24 +401,3 @@ client from which the event originated.  For instance, this could contain the
 message-ID for emails/nntp posts, or a link to a blog comment when gatewaying
 blog comment traffic in & out of matrix
 
-Active Application Services
-----------------------------
-.. NOTE::
-  This section is a work in progress.
-
-.. TODO-spec
-  API that provides hooks into the server so that you can intercept and
-  manipulate events, and/or insert virtual users & rooms into the server.
-
-Policy Servers
-==============
-.. NOTE::
-  This section is a work in progress.
-
-.. TODO-spec
-  We should mention them in the Architecture section at least: how they fit
-  into the picture.
-
-Enforcing policies
-------------------
-

specification/4-server_server_api.rst

@@ -92,7 +92,7 @@ server by querying other servers.
 .. _Perspectives Project: http://perspectives-project.org/
 
 Publishing Keys
-_______________
+^^^^^^^^^^^^^^^
 
 Home servers publish the allowed TLS fingerprints and signing keys in a JSON
 object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of
@@ -178,7 +178,7 @@ events sent by that server can still be checked.
     }
 
 Querying Keys Through Another Server
-____________________________________
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys
 for another server. This API can be used to GET at list of JSON objects for a

specification/modules/instant_messaging.rst

@@ -0,0 +1,27 @@
+Instant Messaging
+=================
+
+Events
+------
+
+{{m_room_message_event}}
+
+{{m_room_message_feedback_event}}
+
+{{m_room_name_event}}
+
+{{m_room_topic_event}}
+
+m.room.message msgtypes
+-----------------------
+
+.. TODO-spec
+   How a client should handle unknown message types.
+
+
+Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
+of message being sent. Each type has their own required and optional keys, as
+outlined below.
+
+{{msgtype_events}}
+

specification/modules/presence.rst

@@ -1,44 +1,5 @@
-Events
+Presence
-======
+========
-
-All communication in Matrix is expressed in the form of data objects called
-Events. These are the fundamental building blocks common to the client-server,
-server-server and application-service APIs, and are described below.
-
-{{common_event_fields}}
-
-{{common_room_event_fields}}
-
-{{common_state_event_fields}}
-
-
-Room Events
------------
-.. NOTE::
-  This section is a work in progress.
-
-This specification outlines several standard event types, all of which are
-prefixed with ``m.``
-
-{{room_events}}
-
-m.room.message msgtypes
-~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO-spec
-   How a client should handle unknown message types.
-
-
-Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
-of message being sent. Each type has their own required and optional keys, as
-outlined below.
-
-{{msgtype_events}}
-
-Presence Events
-~~~~~~~~~~~~~~~
-
-{{presence_events}}
  
 Each user has the concept of presence information. This encodes the
 "availability" of that user, suitable for display on other user's clients.
@@ -66,9 +27,21 @@ is presented via a key called ``last_active_ago``, which gives the relative
 number of milliseconds since the message is generated/emitted that the user
 was last seen active.
 
+Events
+------
+
+{{presence_events}}
+
+Presence HTTP API
+-----------------
+.. TODO-spec
+  - Define how users receive presence invites, and how they accept/decline them
+
+{{presence_http_api}}
+    
 
 Events on Change of Profile Information
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+---------------------------------------
 Because the profile displayname and avatar information are likely to be used in
 many places of a client's display, changes to these fields cause an automatic
 propagation event to occur, informing likely-interested parties of the new

specification/targets.yaml

@@ -0,0 +1,40 @@
+targets:
+  main:  # arbitrary name to identify this build target
+    files:  # the sort order of files to cat
+      - 0-intro.rst
+      - { 1: 0-feature_profiles.rst }
+      - 1-client_server_api.rst
+      - { 1: 0-events.rst }
+      - { 1: 0-event_signing.rst }
+      - 2-modules.rst
+      - { 1: "group:modules" }  # reference a group of files
+      - 3-application_service_api.rst
+      - 4-server_server_api.rst
+      - 5-identity_servers.rst
+      - 6-appendices.rst
+groups:  # reusable blobs of files when prefixed with 'group:'
+  modules:
+    - modules/instant_messaging.rst
+    - modules/voip_events.rst
+    - modules/typing_notifications.rst
+    - modules/receipts.rst
+    - modules/presence.rst
+    - modules/content_repo.rst
+    - modules/end_to_end_encryption.rst
+    - modules/history_visibility.rst
+    - modules/push_overview.rst
+      # relative depth
+    - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] }
+
+title_styles: ["=", "-", "~", "+", "^"]
+
+# The templating system doesn't know the right title style to use when generating
+# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title
+# at the same level (0)", or "make a title one above (+1)". The gendoc script
+# will inspect this file and replace these relative styles with actual title
+# styles. The templating system will also inspect this file to know which symbols
+# to inject.
+relative_title_styles:
+  subtitle: "<"
+  sametitle: "/"
+  supertitle: ">"

templating/batesian/sections.py

@@ -27,12 +27,38 @@ class Sections(object):
             section_key = func_name[len("render_"):]
             self.log("Generating section '%s'" % section_key)
             section = func()
-            if not isinstance(section, basestring):
+            if isinstance(section, basestring):
+                if section_key in section_dict:
                     raise Exception(
-                    "Section function '%s' didn't return a string!" % func_name
+                        ("%s : Section %s already exists. It must have been " +
+                        "generated dynamically. Check which render_ methods " +
+                        "return a dict.") %
+                        (func_name, section_key)
                     )
                 section_dict[section_key] = section
                 self.log(
                     "  Generated. Snippet => %s" % section[:60].replace("\n","")
                 )
+            elif isinstance(section, dict):
+                self.log("  Generated multiple sections:")
+                for (k, v) in section.iteritems():
+                    if not isinstance(k, basestring) or not isinstance(v, basestring):
+                        raise Exception(
+                            ("Method %s returned multiple sections as a dict but " +
+                            "expected the dict elements to be strings but they aren't.") %
+                            (func_name, )
+                        )
+                    if k in section_dict:
+                        raise Exception(
+                            "%s tried to produce section %s which already exists." %
+                            (func_name, k)
+                        )
+                    section_dict[k] = v
+                    self.log(
+                        "  %s  =>  %s" % (k, v[:60].replace("\n",""))
+                    )
+            else:
+                raise Exception(
+                    "Section function '%s' didn't return a string/dict!" % func_name
+                )
         return section_dict

templating/matrix_templates/sections.py

@@ -23,10 +23,13 @@ class MatrixSections(Sections):
         spec_meta = self.units.get("spec_meta")
         return spec_meta["changelog"]
 
-    def _render_events(self, filterFn, sortFn, title_kind="~"):
+    def _render_events(self, filterFn, sortFn):
         template = self.env.get_template("events.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         for event_name in sortFn(schemas):
             if not filterFn(event_name):
@@ -34,14 +37,16 @@ class MatrixSections(Sections):
             sections.append(template.render(
                 example=examples[event_name], 
                 event=schemas[event_name],
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
-    def _render_http_api_group(self, group, sortFnOrPathList=None, 
+    def _render_http_api_group(self, group, sortFnOrPathList=None):
-                               title_kind="-"):
         template = self.env.get_template("http-api.tmpl")
         http_api = self.units.get("swagger_apis")[group]["__meta"]
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         endpoints = []
         if sortFnOrPathList:
@@ -67,46 +72,40 @@ class MatrixSections(Sections):
         for endpoint in endpoints:
             sections.append(template.render(
                 endpoint=endpoint,
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
-    def render_profile_http_api(self):
-        return self._render_http_api_group(
-            "profile", 
-            sortFnOrPathList=["displayname", "avatar_url"],
-            title_kind="~"
-        )
-
-    def render_sync_http_api(self):
-        return self._render_http_api_group(
-            "sync"
-        )
-
-    def render_presence_http_api(self):
-        return self._render_http_api_group(
-            "presence",
-            sortFnOrPathList=["status"],
-            title_kind="~"
-        )
-
-    def render_membership_http_api(self):
-        return self._render_http_api_group(
-            "membership",
-            title_kind="~"
-        )
 
-    def render_login_http_api(self):
+    # Special function: Returning a dict will specify multiple sections where
-        return self._render_http_api_group(
+    # the key is the section name and the value is the value of the section
-            "login",
+    def render_group_http_apis(self):
-            title_kind="~"
+        # map all swagger_apis to the form $GROUP_http_api
+        swagger_groups = self.units.get("swagger_apis").keys()
+        renders = {}
+        for group in swagger_groups:
+            sortFnOrPathList = None
+            if group == "presence":
+                sortFnOrPathList = ["status"]
+            elif group == "profile":
+                sortFnOrPathList=["displayname", "avatar_url"]
+            renders[group + "_http_api"] = self._render_http_api_group(
+                group, sortFnOrPathList
             )
+        return renders
 
-    def render_rooms_http_api(self):
+    # Special function: Returning a dict will specify multiple sections where
-        return self._render_http_api_group(
+    # the key is the section name and the value is the value of the section
-            "rooms",
+    def render_group_events(self):
-            title_kind="+"
+        # map all event schemata to the form $EVENTTYPE_event with s/./_/g
+        # e.g. m_room_topic_event
+        schemas = self.units.get("event_schemas")
+        renders = {}
+        for event_type in schemas:
+            renders[event_type.replace(".", "_") + "_event"] = self._render_events(
+                lambda x: x == event_type, sorted
             )
+        return renders
 
     def render_room_events(self):
         def filterFn(eventType):
@@ -120,6 +119,9 @@ class MatrixSections(Sections):
         template = self.env.get_template("msgtypes.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         msgtype_order = [
             "m.room.message#m.text", "m.room.message#m.emote",
@@ -135,7 +137,8 @@ class MatrixSections(Sections):
                 continue
             sections.append(template.render(
                 example=examples[event_name], 
-                event=schemas[event_name]
+                event=schemas[event_name],
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
@@ -156,12 +159,17 @@ class MatrixSections(Sections):
     def render_presence_events(self):
         def filterFn(eventType):
             return eventType.startswith("m.presence")
-        return self._render_events(filterFn, sorted, title_kind="+")
+        return self._render_events(filterFn, sorted)
 
     def _render_ce_type(self, type):
         template = self.env.get_template("common-event-fields.tmpl")
         ce_types = self.units.get("common_event_fields")
-        return template.render(common_event=ce_types[type])
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
+        return template.render(
+            common_event=ce_types[type], title_kind=subtitle_title_char
+        )
 
     def render_common_event_fields(self):
         return self._render_ce_type("event")
@@ -171,3 +179,4 @@ class MatrixSections(Sections):
 
     def render_common_state_event_fields(self):
         return self._render_ce_type("state_event")
+

templating/matrix_templates/templates/common-event-fields.tmpl

@@ -1,5 +1,5 @@
 {{common_event.title}} Fields
-{{(7 + common_event.title | length) * '-'}}
+{{(7 + common_event.title | length) * title_kind}}
 
 {{common_event.desc | wrap(80)}}
 

templating/matrix_templates/templates/msgtypes.tmpl

@@ -1,5 +1,5 @@
 ``{{event.msgtype}}``
-{{(4 + event.msgtype | length) * '+'}}
+{{(4 + event.msgtype | length) * title_kind}}
 {{event.desc | wrap(80)}}
 {% for table in event.content_fields -%}
 {{"``"+table.title+"``" if table.title else "" }}

templating/matrix_templates/units.py

@@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1"
 V1_EVENT_SCHEMA = "../event-schemas/schema/v1"
 CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema"
 CHANGELOG = "../CHANGELOG.rst"
+TARGETS = "../specification/targets.yaml"
 
 ROOM_EVENT = "core-event-schema/room_event.json"
 STATE_EVENT = "core-event-schema/state_event.json"
@@ -485,6 +486,12 @@ class MatrixUnits(Units):
             "changelog": "".join(changelog_lines)
         }
 
+
+    def load_spec_targets(self):
+        with open(TARGETS, "r") as f:
+            return yaml.load(f.read())
+
+
     def load_git_version(self):
         null = open(os.devnull, 'w')
         cwd = os.path.dirname(os.path.abspath(__file__))