diff --git a/specification/08_00_address_book_repo.rst b/drafts/address-book-repo.rst similarity index 100% rename from specification/08_00_address_book_repo.rst rename to drafts/address-book-repo.rst diff --git a/scripts/gendoc.py b/scripts/gendoc.py index a821aea7..364b8e65 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from docutils.core import publish_file +import copy import fileinput import glob import os @@ -8,6 +9,7 @@ import re import shutil import subprocess import sys +import yaml os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -15,65 +17,214 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] } -title_style_matchers = { - "=": re.compile("^=+$"), - "-": re.compile("^-+$") -} -TOP_LEVEL = "=" -SECOND_LEVEL = "-" -FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" +def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): + rst_lines = [] + title_chars = "".join(title_styles) + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") -def check_valid_section(filename, section): - if not re.match(FILE_FORMAT_MATCHER, filename): - raise Exception( - "The filename of " + filename + " does not match the expected format " + - "of '##_##_words-go-here.rst'" - ) + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None + for i, line in enumerate(file_stream, 1): + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): + rst_lines.append(line) + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): + rst_lines.append(line) + prev_non_title_line = line + continue - # we need TWO new lines else the next file's title gets merged - # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == "\n\n": - raise Exception( - "The file " + filename + " does not end with 2 new lines." - ) + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) - # Enforce some rules to reduce the risk of having mismatched title - # styles. - title_line = section.split("\n")[1] - if title_line != (len(title_line) * title_line[0]): - raise Exception( - "The file " + filename + " doesn't have a title style line on line 2" - ) + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + print (" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0]) - # anything marked as xx_00_ is the start of a new top-level section - if re.match("^[0-9]+_00_", filename): - if not title_style_matchers[TOP_LEVEL].match(title_line): + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: raise Exception( - "The file " + filename + " is a top-level section because it matches " + - "the filename format ##_00_something.rst but has the wrong title " + - "style: expected '" + TOP_LEVEL + "' but got '" + - title_line[0] + "'" + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) ) - # anything marked as xx_xx_ is the start of a sub-section - elif re.match("^[0-9]+_[0-9]{2}_", filename): - if not title_style_matchers[SECOND_LEVEL].match(title_line): + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): raise Exception( - "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format ##_##_something.rst but has the wrong title " + - "style: expected '" + SECOND_LEVEL + "' but got '" + - title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " + - "then use the form '##_##b_something.rst' which will not apply this " + - "check." + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) ) -def cat_spec_sections_to(out_file_name): - with open(out_file_name, "wb") as outfile: - for f in sorted(glob.glob("../specification/*.rst")): - with open(f, "rb") as infile: - section = infile.read() - check_valid_section(os.path.basename(f), section) - outfile.write(section) + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + # print ( + # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + # (filename, line_title_style, + # title_styles[adjusted_level], + # file_offset, title_level) + # ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + + return "".join(rst_lines) + + +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): + # string are file paths to RST blobs + if isinstance(file_info, basestring): + print "%s %s" % (">" * (1 + title_level), file_info) + with open(os.path.join(spec_dir, file_info), "r") as f: + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + if rst[-2:] != "\n\n": + raise Exception( + ("File %s should end with TWO new-line characters to ensure " + + "file concatenation works correctly.") % (file_info,) + ) + return rst + # dicts look like {0: filepath, 1: filepath} where the key is the title level + elif isinstance(file_info, dict): + levels = sorted(file_info.keys()) + rst = [] + for l in levels: + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + # lists are multiple file paths e.g. [filepath, filepath] + elif isinstance(file_info, list): + rst = [] + for f in file_info: + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + raise Exception( + "The following 'file' entry in this target isn't a string, list or dict. " + + "It really really should be. Entry: %s" % (file_info,) + ) + + +def build_spec(target, out_filename): + with open(out_filename, "wb") as outfile: + for file_info in target["files"]: + section = get_rst( + file_info=file_info, + title_level=0, + title_styles=target["title_styles"], + spec_dir="../specification/", + adjust_titles=True + ) + outfile.write(section) + + +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] + relative_title_chars = [ + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + def rst2html(i, o): @@ -88,13 +239,14 @@ def rst2html(i, o): settings_overrides=stylesheets ) + def run_through_template(input): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: - subprocess.check_output( + print subprocess.check_output( [ - 'python', 'build.py', + 'python', 'build.py', "-v", "-i", "matrix_templates", "-o", "../scripts/tmp", "../scripts/"+input @@ -107,6 +259,95 @@ def run_through_template(input): sys.stderr.write(f.read() + "\n") raise + +""" +Extract and resolve groups for the given target in the given targets listing. +Args: + targets_listing (str): The path to a YAML file containing a list of targets + target_name (str): The name of the target to extract from the listings. +Returns: + dict: Containing "filees" (a list of file paths), "relative_title_styles" + (a dict of relative style keyword to title character) and "title_styles" + (a list of characters which represent the global title style to follow, + with the top section title first, the second section second, and so on.) +""" +def get_build_target(targets_listing, target_name): + build_target = { + "title_styles": [], + "relative_title_styles": {}, + "files": [] + } + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + + build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + + def get_group(group_id, depth): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '%s' but it doesn't exist." % group_name + ) + if not isinstance(group, list): + raise Exception( + "Expected group '%s' to be a list but it isn't." % group_name + ) + # deep copy so changes to depths don't contaminate multiple uses of this group + group = copy.deepcopy(group) + # swap relative depths for absolute ones + for i, entry in enumerate(group): + if isinstance(entry, dict): + group[i] = { + (rel_depth + depth): v for (rel_depth, v) in entry.items() + } + return group + + resolved_files = [] + for file_entry in target["files"]: + # file_entry is a group id + if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + group = get_group(file_entry, 0) + # The group may be resolved to a list of file entries, in which case + # we want to extend the array to insert each of them rather than + # insert the entire list as a single element (which is what append does) + if isinstance(group, list): + resolved_files.extend(group) + else: + resolved_files.append(group) + # file_entry is a dict which has more file entries as values + elif isinstance(file_entry, dict): + resolved_entry = {} + for (depth, entry) in file_entry.iteritems(): + if not isinstance(entry, basestring): + raise Exception( + "Double-nested depths are not supported. Entry: %s" % (file_entry,) + ) + if entry.startswith("group:"): + resolved_entry[depth] = get_group(entry, depth) + else: + # map across without editing (e.g. normal file path) + resolved_entry[depth] = entry + resolved_files.append(resolved_entry) + continue + # file_entry is just a plain ol' file path + else: + resolved_files.append(file_entry) + build_target["files"] = resolved_files + return build_target + + def prepare_env(): try: os.makedirs("./gen") @@ -116,14 +357,22 @@ def prepare_env(): os.makedirs("./tmp") except OSError: pass - + + def cleanup_env(): shutil.rmtree("./tmp") -def main(): + +def main(target_name): prepare_env() - cat_spec_sections_to("tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + print "Building spec [target=%s]" % target_name + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst") + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") @@ -131,6 +380,7 @@ def main(): if "--nodelete" not in sys.argv: cleanup_env() + if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: # we accept almost no args, so they don't know what they're doing! @@ -145,4 +395,4 @@ if __name__ == '__main__': print "Requirements:" print " - This script requires Jinja2 and rst2html (docutils)." sys.exit(0) - main() + main("main") diff --git a/specification/02_02_event_signing.rst b/specification/0-event_signing.rst similarity index 100% rename from specification/02_02_event_signing.rst rename to specification/0-event_signing.rst diff --git a/specification/0-events.rst b/specification/0-events.rst new file mode 100644 index 00000000..16948462 --- /dev/null +++ b/specification/0-events.rst @@ -0,0 +1,38 @@ +Events +====== + +All communication in Matrix is expressed in the form of data objects called +Events. These are the fundamental building blocks common to the client-server, +server-server and application-service APIs, and are described below. + +{{common_event_fields}} + +{{common_room_event_fields}} + +{{common_state_event_fields}} + + +Room Events +----------- +.. NOTE:: + This section is a work in progress. + +This specification outlines several standard event types, all of which are +prefixed with ``m.`` + +{{m_room_aliases_event}} + +{{m_room_canonical_alias_event}} + +{{m_room_create_event}} + +{{m_room_history_visibility_event}} + +{{m_room_join_rules_event}} + +{{m_room_member_event}} + +{{m_room_power_levels_event}} + +{{m_room_redaction_event}} + diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst new file mode 100644 index 00000000..234e14db --- /dev/null +++ b/specification/0-feature_profiles.rst @@ -0,0 +1,3 @@ +Feature Profiles +================ + diff --git a/specification/00_00_intro.rst b/specification/0-intro.rst similarity index 100% rename from specification/00_00_intro.rst rename to specification/0-intro.rst diff --git a/specification/01_00_client_server_api.rst b/specification/1-client_server_api.rst similarity index 99% rename from specification/01_00_client_server_api.rst rename to specification/1-client_server_api.rst index ad39dafc..93e3cb90 100644 --- a/specification/01_00_client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1054,12 +1054,6 @@ medium address The textual address of the 3pid, eg. the email address -Presence --------- -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them - -{{presence_http_api}} Profiles -------- diff --git a/specification/2-modules.rst b/specification/2-modules.rst new file mode 100644 index 00000000..0aad77e1 --- /dev/null +++ b/specification/2-modules.rst @@ -0,0 +1,3 @@ +Modules +======= + diff --git a/specification/03_00_application_service_api.rst b/specification/3-application_service_api.rst similarity index 96% rename from specification/03_00_application_service_api.rst rename to specification/3-application_service_api.rst index 9b58c861..e982390b 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -401,24 +401,3 @@ client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix -Active Application Services ----------------------------- -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - API that provides hooks into the server so that you can intercept and - manipulate events, and/or insert virtual users & rooms into the server. - -Policy Servers -============== -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - We should mention them in the Architecture section at least: how they fit - into the picture. - -Enforcing policies ------------------- - diff --git a/specification/04_00_server_server_api.rst b/specification/4-server_server_api.rst similarity index 99% rename from specification/04_00_server_server_api.rst rename to specification/4-server_server_api.rst index f5cadabf..8d1f8898 100644 --- a/specification/04_00_server_server_api.rst +++ b/specification/4-server_server_api.rst @@ -92,7 +92,7 @@ server by querying other servers. .. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of @@ -178,7 +178,7 @@ events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a diff --git a/specification/09_00_identity_servers.rst b/specification/5-identity_servers.rst similarity index 100% rename from specification/09_00_identity_servers.rst rename to specification/5-identity_servers.rst diff --git a/specification/10_00_appendices.rst b/specification/6-appendices.rst similarity index 100% rename from specification/10_00_appendices.rst rename to specification/6-appendices.rst diff --git a/specification/05_00_content_repository.rst b/specification/modules/content_repo.rst similarity index 100% rename from specification/05_00_content_repository.rst rename to specification/modules/content_repo.rst diff --git a/specification/06_00_end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst similarity index 100% rename from specification/06_00_end_to_end_encryption.rst rename to specification/modules/end_to_end_encryption.rst diff --git a/specification/01_03_history_visibility.rst b/specification/modules/history_visibility.rst similarity index 100% rename from specification/01_03_history_visibility.rst rename to specification/modules/history_visibility.rst diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst new file mode 100644 index 00000000..7f582ca4 --- /dev/null +++ b/specification/modules/instant_messaging.rst @@ -0,0 +1,27 @@ +Instant Messaging +================= + +Events +------ + +{{m_room_message_event}} + +{{m_room_message_feedback_event}} + +{{m_room_name_event}} + +{{m_room_topic_event}} + +m.room.message msgtypes +----------------------- + +.. TODO-spec + How a client should handle unknown message types. + + +Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type +of message being sent. Each type has their own required and optional keys, as +outlined below. + +{{msgtype_events}} + diff --git a/specification/02_00_events.rst b/specification/modules/presence.rst similarity index 74% rename from specification/02_00_events.rst rename to specification/modules/presence.rst index ce36b040..ddd2adff 100644 --- a/specification/02_00_events.rst +++ b/specification/modules/presence.rst @@ -1,44 +1,5 @@ -Events -====== - -All communication in Matrix is expressed in the form of data objects called -Events. These are the fundamental building blocks common to the client-server, -server-server and application-service APIs, and are described below. - -{{common_event_fields}} - -{{common_room_event_fields}} - -{{common_state_event_fields}} - - -Room Events ------------ -.. NOTE:: - This section is a work in progress. - -This specification outlines several standard event types, all of which are -prefixed with ``m.`` - -{{room_events}} - -m.room.message msgtypes -~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO-spec - How a client should handle unknown message types. - - -Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type -of message being sent. Each type has their own required and optional keys, as -outlined below. - -{{msgtype_events}} - -Presence Events -~~~~~~~~~~~~~~~ - -{{presence_events}} +Presence +======== Each user has the concept of presence information. This encodes the "availability" of that user, suitable for display on other user's clients. @@ -65,10 +26,22 @@ proactive event, whereas in the other direction it will not). This timestamp is presented via a key called ``last_active_ago``, which gives the relative number of milliseconds since the message is generated/emitted that the user was last seen active. + +Events +------ + +{{presence_events}} + +Presence HTTP API +----------------- +.. TODO-spec + - Define how users receive presence invites, and how they accept/decline them + +{{presence_http_api}} Events on Change of Profile Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------- Because the profile displayname and avatar information are likely to be used in many places of a client's display, changes to these fields cause an automatic propagation event to occur, informing likely-interested parties of the new diff --git a/specification/07_01_push_cs_api.rst b/specification/modules/push_cs_api.rst similarity index 100% rename from specification/07_01_push_cs_api.rst rename to specification/modules/push_cs_api.rst diff --git a/specification/07_00_push_overview.rst b/specification/modules/push_overview.rst similarity index 100% rename from specification/07_00_push_overview.rst rename to specification/modules/push_overview.rst diff --git a/specification/07_02_push_push_gw_api.rst b/specification/modules/push_push_gw_api.rst similarity index 100% rename from specification/07_02_push_push_gw_api.rst rename to specification/modules/push_push_gw_api.rst diff --git a/specification/01_02_receipts.rst b/specification/modules/receipts.rst similarity index 100% rename from specification/01_02_receipts.rst rename to specification/modules/receipts.rst diff --git a/specification/01_01_typing_notifications.rst b/specification/modules/typing_notifications.rst similarity index 100% rename from specification/01_01_typing_notifications.rst rename to specification/modules/typing_notifications.rst diff --git a/specification/02_01_voip_events.rst b/specification/modules/voip_events.rst similarity index 100% rename from specification/02_01_voip_events.rst rename to specification/modules/voip_events.rst diff --git a/specification/targets.yaml b/specification/targets.yaml new file mode 100644 index 00000000..62585c69 --- /dev/null +++ b/specification/targets.yaml @@ -0,0 +1,40 @@ +targets: + main: # arbitrary name to identify this build target + files: # the sort order of files to cat + - 0-intro.rst + - { 1: 0-feature_profiles.rst } + - 1-client_server_api.rst + - { 1: 0-events.rst } + - { 1: 0-event_signing.rst } + - 2-modules.rst + - { 1: "group:modules" } # reference a group of files + - 3-application_service_api.rst + - 4-server_server_api.rst + - 5-identity_servers.rst + - 6-appendices.rst +groups: # reusable blobs of files when prefixed with 'group:' + modules: + - modules/instant_messaging.rst + - modules/voip_events.rst + - modules/typing_notifications.rst + - modules/receipts.rst + - modules/presence.rst + - modules/content_repo.rst + - modules/end_to_end_encryption.rst + - modules/history_visibility.rst + - modules/push_overview.rst + # relative depth + - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } + +title_styles: ["=", "-", "~", "+", "^"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 11c34fb3..31849389 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -27,12 +27,38 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if not isinstance(section, basestring): + if isinstance(section, basestring): + if section_key in section_dict: + raise Exception( + ("%s : Section %s already exists. It must have been " + + "generated dynamically. Check which render_ methods " + + "return a dict.") % + (func_name, section_key) + ) + section_dict[section_key] = section + self.log( + " Generated. Snippet => %s" % section[:60].replace("\n","") + ) + elif isinstance(section, dict): + self.log(" Generated multiple sections:") + for (k, v) in section.iteritems(): + if not isinstance(k, basestring) or not isinstance(v, basestring): + raise Exception( + ("Method %s returned multiple sections as a dict but " + + "expected the dict elements to be strings but they aren't.") % + (func_name, ) + ) + if k in section_dict: + raise Exception( + "%s tried to produce section %s which already exists." % + (func_name, k) + ) + section_dict[k] = v + self.log( + " %s => %s" % (k, v[:60].replace("\n","")) + ) + else: raise Exception( - "Section function '%s' didn't return a string!" % func_name + "Section function '%s' didn't return a string/dict!" % func_name ) - section_dict[section_key] = section - self.log( - " Generated. Snippet => %s" % section[:60].replace("\n","") - ) return section_dict \ No newline at end of file diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 2a591c11..e75a75af 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,46 +72,40 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def render_profile_http_api(self): - return self._render_http_api_group( - "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" - ) - - def render_sync_http_api(self): - return self._render_http_api_group( - "sync" - ) - - def render_presence_http_api(self): - return self._render_http_api_group( - "presence", - sortFnOrPathList=["status"], - title_kind="~" - ) - - def render_membership_http_api(self): - return self._render_http_api_group( - "membership", - title_kind="~" - ) - def render_login_http_api(self): - return self._render_http_api_group( - "login", - title_kind="~" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_http_apis(self): + # map all swagger_apis to the form $GROUP_http_api + swagger_groups = self.units.get("swagger_apis").keys() + renders = {} + for group in swagger_groups: + sortFnOrPathList = None + if group == "presence": + sortFnOrPathList = ["status"] + elif group == "profile": + sortFnOrPathList=["displayname", "avatar_url"] + renders[group + "_http_api"] = self._render_http_api_group( + group, sortFnOrPathList + ) + return renders - def render_rooms_http_api(self): - return self._render_http_api_group( - "rooms", - title_kind="+" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_events(self): + # map all event schemata to the form $EVENTTYPE_event with s/./_/g + # e.g. m_room_topic_event + schemas = self.units.get("event_schemas") + renders = {} + for event_type in schemas: + renders[event_type.replace(".", "_") + "_event"] = self._render_events( + lambda x: x == event_type, sorted + ) + return renders def render_room_events(self): def filterFn(eventType): @@ -120,6 +119,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -135,7 +137,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -156,12 +159,17 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") ce_types = self.units.get("common_event_fields") - return template.render(common_event=ce_types[type]) + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + return template.render( + common_event=ce_types[type], title_kind=subtitle_title_char + ) def render_common_event_fields(self): return self._render_ce_type("event") @@ -171,3 +179,4 @@ class MatrixSections(Sections): def render_common_state_event_fields(self): return self._render_ce_type("state_event") + diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 2a3f7ec5..3f16be3d 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,5 +1,5 @@ {{common_event.title}} Fields -{{(7 + common_event.title | length) * '-'}} +{{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index a03ffa7d..eb3f3e64 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -18,7 +18,7 @@ Request format: ================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* --------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} {{param.key}}{{param.type|indent(19-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(37)}} {% endfor -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160..f7862451 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 975f7c80..a5c6a815 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" @@ -485,6 +486,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__))