Implement relative title styles

Templates don't know at what level they will be inserted. Previously, we
hard-coded the title style which is not compatible with the build target
system. Define a set of styles which will be replaced by the gendoc script
when it encounters them:
 '<' : Make this title a sub-heading
 '/' : Make this title a heading at the same level
 '>' : Make this title a super-heading

The build target system is now basically complete and functioning.

scripts/gendoc.py

@@ -17,57 +17,133 @@ stylesheets = {
 }
 
 
-def _list_get(l, index, default=None):
+"""
-    try:
+Read a RST file and replace titles with a different title level if required.
-        return l[index]
+Args:
-    except IndexError:
+    filename: The name of the file being read (for debugging)
-        return default
+    file_stream: The open file stream to read from.
-
+    title_level: The integer which determines the offset to *start* from.
-
+    title_styles: An array of characters detailing the right title styles to use
+                  e.g. ["=", "-", "~", "+"]
+Returns:
+    string: The file contents with titles adjusted.
+Example:
+    Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file
+    when read line-by-line encounters the titles "===", "---", "---", "===", "---".
+    This function will bump every title encountered down a sub-heading e.g.
+    "=" to "-" and "-" to "~" because title_level = 1, so the output would be
+    "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level.
+"""
 def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
     rst_lines = []
     title_chars = "".join(title_styles)
-    title_regex = re.compile("^[" + re.escape(title_chars) + "]+$")
+    title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$")
 
-    curr_title_level = title_level
+    prev_line_title_level = 0 # We expect the file to start with '=' titles
+    file_offset = None
+    prev_non_title_line = None
     for i, line in enumerate(file_stream, 1):
-        if title_regex.match(line):
+        # ignore anything which isn't a title (e.g. '===============')
-            line_title_level = title_styles.index(line[0])
+        if not title_regex.match(line):
-            # Allowed to go 1 deeper or any number shallower
-            if curr_title_level - line_title_level < -1:
-                raise Exception(
-                    ("File '%s' line '%s' has a title " +
-                    "style '%s' which doesn't match one of the " +
-                    "allowed title styles of %s because the " +
-                    "title level before this line was '%s'") %
-                    (filename, (i + 1), line[0], title_styles,
-                    title_styles[curr_title_level])
-                )
-            curr_title_level = line_title_level
             rst_lines.append(line)
-        else:
+            prev_non_title_line = line
+            continue
+        # The title underline must match at a minimum the length of the title
+        if len(prev_non_title_line) > len(line):
             rst_lines.append(line)
+            prev_non_title_line = line
+            continue
+
+        line_title_style = line[0]
+        line_title_level = title_styles.index(line_title_style)
+
+        # Not all files will start with "===" and we should be flexible enough
+        # to allow that. The first title we encounter sets the "file offset"
+        # which is added to the title_level desired.
+        if file_offset is None:
+            file_offset = line_title_level
+            if file_offset != 0:
+                print ("     WARNING: %s starts with a title style of '%s' but '%s' " +
+                    "is preferable.") % (filename, line_title_style, title_styles[0])
+
+        # Sanity checks: Make sure that this file is obeying the title levels
+        # specified and bail if it isn't.
+        # The file is allowed to go 1 deeper or any number shallower
+        if prev_line_title_level - line_title_level < -1:
+            raise Exception(
+                ("File '%s' line '%s' has a title " +
+                "style '%s' which doesn't match one of the " +
+                "allowed title styles of %s because the " +
+                "title level before this line was '%s'") %
+                (filename, (i + 1), line_title_style, title_styles,
+                title_styles[prev_line_title_level])
+            )
+        prev_line_title_level = line_title_level
+
+        adjusted_level = (
+            title_level + line_title_level - file_offset
+        )
+
+        # Sanity check: Make sure we can bump down the title and we aren't at the
+        # lowest level already
+        if adjusted_level >= len(title_styles):
+            raise Exception(
+                ("Files '%s' line '%s' has a sub-title level too low and it " +
+                "cannot be adjusted to fit. You can add another level to the " +
+                "'title_styles' key in targets.yaml to fix this.") %
+                (filename, (i + 1))
+            )
+
+        if adjusted_level == line_title_level:
+            # no changes required
+            rst_lines.append(line)
+            continue
+
+        # Adjusting line levels
+        # print (
+        #     "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" %
+        #     (filename, line_title_style,
+        #         title_styles[adjusted_level],
+        #         file_offset, title_level)
+        # )
+        rst_lines.append(line.replace(
+            line_title_style,
+            title_styles[adjusted_level]
+        ))
+            
     return "".join(rst_lines)
 
 
-def get_rst(file_info, title_level, title_styles, spec_dir):
+def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
     # string are file paths to RST blobs
     if isinstance(file_info, basestring):
         print "%s %s" % (">" * (1 + title_level), file_info)
         with open(spec_dir + file_info, "r") as f:
-            return load_with_adjusted_titles(file_info, f, title_level, title_styles)
+            rst = None
+            if adjust_titles:
+                rst = load_with_adjusted_titles(
+                    file_info, f, title_level, title_styles
+                )
+            else:
+                rst = f.read()
+            if rst[-2:] != "\n\n":
+                raise Exception(
+                    ("File %s should end with TWO new-line characters to ensure " +
+                    "file concatenation works correctly.") % (file_info,)
+                )
+            return rst
     # dicts look like {0: filepath, 1: filepath} where the key is the title level
     elif isinstance(file_info, dict):
         levels = sorted(file_info.keys())
         rst = []
         for l in levels:
-            rst.append(get_rst(file_info[l], l, title_styles, spec_dir))
+            rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles))
         return "".join(rst)
     # lists are multiple file paths e.g. [filepath, filepath]
     elif isinstance(file_info, list):
         rst = []
         for f in file_info:
-            rst.append(get_rst(f, title_level, title_styles, spec_dir))
+            rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles))
         return "".join(rst)
     raise Exception(
         "The following 'file' entry in this target isn't a string, list or dict. " +
@@ -82,11 +158,74 @@ def build_spec(target, out_filename):
                 file_info=file_info,
                 title_level=0,
                 title_styles=target["title_styles"],
-                spec_dir="../specification/"
+                spec_dir="../specification/",
+                adjust_titles=True
             )
             outfile.write(section)
 
 
+"""
+Replaces relative title styles with actual title styles.
+
+The templating system has no idea what the right title style is when it produces
+RST because it depends on the build target. As a result, it uses relative title
+styles defined in targets.yaml to say "down a level, up a level, same level".
+
+This function replaces these relative titles with actual title styles from the
+array in targets.yaml.
+"""
+def fix_relative_titles(target, filename, out_filename):
+    title_styles = target["title_styles"] # ["=", "-", "~", "+"]
+    relative_title_chars = [ # ["<", "/", ">"]
+        target["relative_title_styles"]["subtitle"],
+        target["relative_title_styles"]["sametitle"],
+        target["relative_title_styles"]["supertitle"]
+    ]
+    relative_title_matcher = re.compile(
+        "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$"
+    )
+    title_matcher = re.compile(
+        "^[" + re.escape("".join(title_styles)) + "]{3,}$"
+    )
+    current_title_style = None
+    with open(filename, "r") as infile:
+        with open(out_filename, "w") as outfile:
+            for line in infile.readlines():
+                if not relative_title_matcher.match(line):
+                    if title_matcher.match(line):
+                        current_title_style = line[0]
+                    outfile.write(line)
+                    continue
+                line_char = line[0]
+                replacement_char = None
+                current_title_level = title_styles.index(current_title_style)
+                if line_char == target["relative_title_styles"]["subtitle"]:
+                    if (current_title_level + 1) == len(title_styles):
+                        raise Exception(
+                            "Encountered sub-title line style but we can't go " +
+                            "any lower."
+                        )
+                    replacement_char = title_styles[current_title_level + 1]
+                elif line_char == target["relative_title_styles"]["sametitle"]:
+                    replacement_char = title_styles[current_title_level]
+                elif line_char == target["relative_title_styles"]["supertitle"]:
+                    if (current_title_level - 1) < 0:
+                        raise Exception(
+                            "Encountered super-title line style but we can't go " +
+                            "any higher."
+                        )
+                    replacement_char = title_styles[current_title_level - 1]
+                else:
+                    raise Exception(
+                        "Unknown relative line char %s" % (line_char,)
+                    )
+
+                outfile.write(
+                    line.replace(line_char, replacement_char)
+                )
+
+
+
 def rst2html(i, o):
     with open(i, "r") as in_file:
         with open(o, "w") as out_file:
@@ -123,11 +262,13 @@ def run_through_template(input):
 def get_build_target(targets_listing, target_name):
     build_target = {
         "title_styles": [],
+        "relative_title_styles": {},
         "files": []
     }
     with open(targets_listing, "r") as targ_file:
         all_targets = yaml.load(targ_file.read())
         build_target["title_styles"] = all_targets["title_styles"]
+        build_target["relative_title_styles"] = all_targets["relative_title_styles"]
         target = all_targets["targets"].get(target_name)
         if not target:
             raise Exception(
@@ -138,17 +279,30 @@ def get_build_target(targets_listing, target_name):
             raise Exception(
                 "Found target but 'files' key is not a list."
             )
+
+        def get_group(group_id):
+            group_name = group_id[len("group:"):]
+            group = all_targets.get("groups", {}).get(group_name)
+            if not group:
+                raise Exception(
+                    "Tried to find group '" + group_name + "' but it " +
+                    "doesn't exist."
+                )
+            return group
+
         resolved_files = []
         for f in target["files"]:
+            group = None
             if isinstance(f, basestring) and f.startswith("group:"):
-                # copy across the group of files specified
+                group = get_group(f)
-                group_name = f[len("group:"):]
+            elif isinstance(f, dict):
-                group = all_targets.get("groups", {}).get(group_name)
+                for (k, v) in f.iteritems():
-                if not group:
+                    if isinstance(v, basestring) and v.startswith("group:"):
-                    raise Exception(
+                        f[k] = get_group(v)
-                        "Tried to find group '" + group_name + "' but it " +
+                resolved_files.append(f)
-                        "doesn't exist."
+                continue
-                    )
+
+            if group:
                 if isinstance(group, list):
                     resolved_files.extend(group)
                 else:
@@ -178,8 +332,12 @@ def main(target_name):
     prepare_env()
     print "Building spec [target=%s]" % target_name
     target = get_build_target("../specification/targets.yaml", target_name)
-    build_spec(target=target, out_filename="tmp/full_spec.rst")
+    build_spec(target=target, out_filename="tmp/templated_spec.rst")
-    run_through_template("tmp/full_spec.rst")
+    run_through_template("tmp/templated_spec.rst")
+    fix_relative_titles(
+        target=target, filename="tmp/templated_spec.rst",
+        out_filename="tmp/full_spec.rst"
+    )
     shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
     run_through_template("tmp/howto.rst")
     rst2html("tmp/full_spec.rst", "gen/specification.html")

specification/00_01_feature_profiles.rst

@@ -0,0 +1,5 @@
+Feature Profiles
+================
+
+Feature profiles blurb goes here.
+

specification/02_00_modules.rst

@@ -0,0 +1,5 @@
+Modules
+=======
+
+Modules intro here.
+

specification/03_00_application_service_api.rst

@@ -400,3 +400,4 @@ in their content to provide a way for Matrix clients to link into the 'native'
 client from which the event originated.  For instance, this could contain the
 message-ID for emails/nntp posts, or a link to a blog comment when gatewaying
 blog comment traffic in & out of matrix
+

specification/04_00_server_server_api.rst

@@ -92,7 +92,7 @@ server by querying other servers.
 .. _Perspectives Project: http://perspectives-project.org/
 
 Publishing Keys
-_______________
+^^^^^^^^^^^^^^^
 
 Home servers publish the allowed TLS fingerprints and signing keys in a JSON
 object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of
@@ -178,7 +178,7 @@ events sent by that server can still be checked.
     }
 
 Querying Keys Through Another Server
-____________________________________
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys
 for another server. This API can be used to GET at list of JSON objects for a

specification/modules/00_modules_intro.rst

@@ -0,0 +1,5 @@
+Modules
+=======
+
+Modules blurb goes here.
+

specification/targets.yaml

@@ -2,26 +2,37 @@ targets:
   main:  # arbitrary name to identify this build target
     files:  # the sort order of files to cat
       - 00_00_intro.rst
-      - 00_01_feature_profiles.rst
+      - { 1: 00_01_feature_profiles.rst }
-      - 00_02a_events.rst
+      - { 1: 00_02a_events.rst }
-      - 00_02b_event_signing.rst
+      - { 1: 00_02b_event_signing.rst }
       - 01_00_client_server_api.rst
       - 02_00_modules.rst
-      - "group:modules"  # reference a group of files
+      - { 1: "group:modules" }  # reference a group of files
       - 03_00_application_service_api.rst
       - 04_00_server_server_api.rst
       - 05_00_identity_servers.rst
       - 06_00_appendices.rst
 groups:  # reusable blobs of files when prefixed with 'group:'
   modules:
-      0: modules/00_modules_intro.rst
+    - modules/00_modules_intro.rst
-      1:
+    - modules/01_00_voip_events.rst
-         - modules/01_00_voip_events.rst
+    - modules/02_00_typing_notifications.rst
-         - modules/02_00_typing_notifications.rst
+    - modules/03_00_receipts.rst
-         - modules/03_00_receipts.rst
+    - modules/04_00_content_repo.rst
-         - modules/04_00_content_repo.rst
+    - modules/05_00_end_to_end_encryption.rst
-         - modules/05_00_end_to_end_encryption.rst
+    - modules/06_00_history_visibility.rst
-         - modules/06_00_history_visibility.rst
+    - modules/07_00_push_overview.rst
-         - 1: modules/07_00_push_overview.rst # Mark a nested file dependency
+    - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] }
-           2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst]
+
-title_styles: ["=", "-", "~", "+"]
+title_styles: ["=", "-", "~", "+", "^"]
+
+# The templating system doesn't know the right title style to use when generating
+# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title
+# at the same level (0)", or "make a title one above (+1)". The gendoc script
+# will inspect this file and replace these relative styles with actual title
+# styles. The templating system will also inspect this file to know which symbols
+# to inject.
+relative_title_styles:
+  subtitle: "<"
+  sametitle: "/"
+  supertitle: ">"

templating/matrix_templates/sections.py

@@ -23,10 +23,13 @@ class MatrixSections(Sections):
         spec_meta = self.units.get("spec_meta")
         return spec_meta["changelog"]
 
-    def _render_events(self, filterFn, sortFn, title_kind="~"):
+    def _render_events(self, filterFn, sortFn):
         template = self.env.get_template("events.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         for event_name in sortFn(schemas):
             if not filterFn(event_name):
@@ -34,14 +37,16 @@ class MatrixSections(Sections):
             sections.append(template.render(
                 example=examples[event_name], 
                 event=schemas[event_name],
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
-    def _render_http_api_group(self, group, sortFnOrPathList=None, 
+    def _render_http_api_group(self, group, sortFnOrPathList=None):
-                               title_kind="-"):
         template = self.env.get_template("http-api.tmpl")
         http_api = self.units.get("swagger_apis")[group]["__meta"]
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         endpoints = []
         if sortFnOrPathList:
@@ -67,15 +72,14 @@ class MatrixSections(Sections):
         for endpoint in endpoints:
             sections.append(template.render(
                 endpoint=endpoint,
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
     def render_profile_http_api(self):
         return self._render_http_api_group(
             "profile", 
-            sortFnOrPathList=["displayname", "avatar_url"],
+            sortFnOrPathList=["displayname", "avatar_url"]
-            title_kind="~"
         )
 
     def render_sync_http_api(self):
@@ -86,20 +90,17 @@ class MatrixSections(Sections):
     def render_presence_http_api(self):
         return self._render_http_api_group(
             "presence",
-            sortFnOrPathList=["status"],
+            sortFnOrPathList=["status"]
-            title_kind="~"
         )
 
     def render_membership_http_api(self):
         return self._render_http_api_group(
-            "membership",
+            "membership"
-            title_kind="~"
         )
 
     def render_login_http_api(self):
         return self._render_http_api_group(
-            "login",
+            "login"
-            title_kind="~"
         )
 
     def render_room_events(self):
@@ -114,6 +115,9 @@ class MatrixSections(Sections):
         template = self.env.get_template("msgtypes.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         msgtype_order = [
             "m.room.message#m.text", "m.room.message#m.emote",
@@ -129,7 +133,8 @@ class MatrixSections(Sections):
                 continue
             sections.append(template.render(
                 example=examples[event_name], 
-                event=schemas[event_name]
+                event=schemas[event_name],
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
@@ -150,7 +155,7 @@ class MatrixSections(Sections):
     def render_presence_events(self):
         def filterFn(eventType):
             return eventType.startswith("m.presence")
-        return self._render_events(filterFn, sorted, title_kind="+")
+        return self._render_events(filterFn, sorted)
 
     def _render_ce_type(self, type):
         template = self.env.get_template("common-event-fields.tmpl")

templating/matrix_templates/templates/msgtypes.tmpl

@@ -1,5 +1,5 @@
 ``{{event.msgtype}}``
-{{(4 + event.msgtype | length) * '+'}}
+{{(4 + event.msgtype | length) * title_kind}}
 {{event.desc | wrap(80)}}
 {% for table in event.content_fields -%}
 {{"``"+table.title+"``" if table.title else "" }}

templating/matrix_templates/units.py

@@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1"
 V1_EVENT_SCHEMA = "../event-schemas/schema/v1"
 CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema"
 CHANGELOG = "../CHANGELOG.rst"
+TARGETS = "../specification/targets.yaml"
 
 ROOM_EVENT = "core-event-schema/room_event.json"
 STATE_EVENT = "core-event-schema/state_event.json"
@@ -466,6 +467,12 @@ class MatrixUnits(Units):
             "changelog": "".join(changelog_lines)
         }
 
+
+    def load_spec_targets(self):
+        with open(TARGETS, "r") as f:
+            return yaml.load(f.read())
+
+
     def load_git_version(self):
         null = open(os.devnull, 'w')
         cwd = os.path.dirname(os.path.abspath(__file__))