From f71763b0d3b6d95ed7839f5c9735d21aa59d5d71 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 13:08:15 +0100 Subject: [PATCH] Implement relative title styles Templates don't know at what level they will be inserted. Previously, we hard-coded the title style which is not compatible with the build target system. Define a set of styles which will be replaced by the gendoc script when it encounters them: '<' : Make this title a sub-heading '/' : Make this title a heading at the same level '>' : Make this title a super-heading The build target system is now basically complete and functioning. --- scripts/gendoc.py | 234 +++++++++++++++--- specification/00_01_feature_profiles.rst | 5 + specification/02_00_modules.rst | 5 + .../03_00_application_service_api.rst | 1 + specification/04_00_server_server_api.rst | 4 +- specification/modules/00_modules_intro.rst | 5 + specification/targets.yaml | 41 +-- templating/matrix_templates/sections.py | 35 +-- .../matrix_templates/templates/msgtypes.tmpl | 2 +- templating/matrix_templates/units.py | 7 + 10 files changed, 268 insertions(+), 71 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f7bde162..308d148f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -17,57 +17,133 @@ stylesheets = { } -def _list_get(l, index, default=None): - try: - return l[index] - except IndexError: - return default - - +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): rst_lines = [] title_chars = "".join(title_styles) - title_regex = re.compile("^[" + re.escape(title_chars) + "]+$") + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") - curr_title_level = title_level + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None for i, line in enumerate(file_stream, 1): - if title_regex.match(line): - line_title_level = title_styles.index(line[0]) - # Allowed to go 1 deeper or any number shallower - if curr_title_level - line_title_level < -1: - raise Exception( - ("File '%s' line '%s' has a title " + - "style '%s' which doesn't match one of the " + - "allowed title styles of %s because the " + - "title level before this line was '%s'") % - (filename, (i + 1), line[0], title_styles, - title_styles[curr_title_level]) - ) - curr_title_level = line_title_level + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): rst_lines.append(line) - else: + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): rst_lines.append(line) + prev_non_title_line = line + continue + + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) + + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + print (" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0]) + + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) + ) + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): + raise Exception( + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) + ) + + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + # print ( + # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + # (filename, line_title_style, + # title_styles[adjusted_level], + # file_offset, title_level) + # ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + return "".join(rst_lines) -def get_rst(file_info, title_level, title_styles, spec_dir): +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): print "%s %s" % (">" * (1 + title_level), file_info) with open(spec_dir + file_info, "r") as f: - return load_with_adjusted_titles(file_info, f, title_level, title_styles) + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + if rst[-2:] != "\n\n": + raise Exception( + ("File %s should end with TWO new-line characters to ensure " + + "file concatenation works correctly.") % (file_info,) + ) + return rst # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): levels = sorted(file_info.keys()) rst = [] for l in levels: - rst.append(get_rst(file_info[l], l, title_styles, spec_dir)) + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) return "".join(rst) # lists are multiple file paths e.g. [filepath, filepath] elif isinstance(file_info, list): rst = [] for f in file_info: - rst.append(get_rst(f, title_level, title_styles, spec_dir)) + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) return "".join(rst) raise Exception( "The following 'file' entry in this target isn't a string, list or dict. " + @@ -82,11 +158,74 @@ def build_spec(target, out_filename): file_info=file_info, title_level=0, title_styles=target["title_styles"], - spec_dir="../specification/" + spec_dir="../specification/", + adjust_titles=True ) outfile.write(section) +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] # ["=", "-", "~", "+"] + relative_title_chars = [ # ["<", "/", ">"] + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + + + def rst2html(i, o): with open(i, "r") as in_file: with open(o, "w") as out_file: @@ -123,11 +262,13 @@ def run_through_template(input): def get_build_target(targets_listing, target_name): build_target = { "title_styles": [], + "relative_title_styles": {}, "files": [] } with open(targets_listing, "r") as targ_file: all_targets = yaml.load(targ_file.read()) build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] target = all_targets["targets"].get(target_name) if not target: raise Exception( @@ -138,17 +279,30 @@ def get_build_target(targets_listing, target_name): raise Exception( "Found target but 'files' key is not a list." ) + + def get_group(group_id): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." + ) + return group + resolved_files = [] for f in target["files"]: + group = None if isinstance(f, basestring) and f.startswith("group:"): - # copy across the group of files specified - group_name = f[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." - ) + group = get_group(f) + elif isinstance(f, dict): + for (k, v) in f.iteritems(): + if isinstance(v, basestring) and v.startswith("group:"): + f[k] = get_group(v) + resolved_files.append(f) + continue + + if group: if isinstance(group, list): resolved_files.extend(group) else: @@ -178,8 +332,12 @@ def main(target_name): prepare_env() print "Building spec [target=%s]" % target_name target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst") + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") diff --git a/specification/00_01_feature_profiles.rst b/specification/00_01_feature_profiles.rst index e69de29b..155e51c5 100644 --- a/specification/00_01_feature_profiles.rst +++ b/specification/00_01_feature_profiles.rst @@ -0,0 +1,5 @@ +Feature Profiles +================ + +Feature profiles blurb goes here. + diff --git a/specification/02_00_modules.rst b/specification/02_00_modules.rst index e69de29b..ab35fe9f 100644 --- a/specification/02_00_modules.rst +++ b/specification/02_00_modules.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules intro here. + diff --git a/specification/03_00_application_service_api.rst b/specification/03_00_application_service_api.rst index 2674ba44..e982390b 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/03_00_application_service_api.rst @@ -400,3 +400,4 @@ in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix + diff --git a/specification/04_00_server_server_api.rst b/specification/04_00_server_server_api.rst index f5cadabf..8d1f8898 100644 --- a/specification/04_00_server_server_api.rst +++ b/specification/04_00_server_server_api.rst @@ -92,7 +92,7 @@ server by querying other servers. .. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of @@ -178,7 +178,7 @@ events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/00_modules_intro.rst index e69de29b..fdfcbbdc 100644 --- a/specification/modules/00_modules_intro.rst +++ b/specification/modules/00_modules_intro.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules blurb goes here. + diff --git a/specification/targets.yaml b/specification/targets.yaml index e492ad3c..7bd3d882 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,26 +2,37 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - 00_00_intro.rst - - 00_01_feature_profiles.rst - - 00_02a_events.rst - - 00_02b_event_signing.rst + - { 1: 00_01_feature_profiles.rst } + - { 1: 00_02a_events.rst } + - { 1: 00_02b_event_signing.rst } - 01_00_client_server_api.rst - 02_00_modules.rst - - "group:modules" # reference a group of files + - { 1: "group:modules" } # reference a group of files - 03_00_application_service_api.rst - 04_00_server_server_api.rst - 05_00_identity_servers.rst - 06_00_appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - 0: modules/00_modules_intro.rst - 1: - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - - 1: modules/07_00_push_overview.rst # Mark a nested file dependency - 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] -title_styles: ["=", "-", "~", "+"] + - modules/00_modules_intro.rst + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - modules/07_00_push_overview.rst + - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] } + +title_styles: ["=", "-", "~", "+", "^"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 729157bb..6072222a 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,15 +72,14 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) def render_profile_http_api(self): return self._render_http_api_group( "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" + sortFnOrPathList=["displayname", "avatar_url"] ) def render_sync_http_api(self): @@ -86,20 +90,17 @@ class MatrixSections(Sections): def render_presence_http_api(self): return self._render_http_api_group( "presence", - sortFnOrPathList=["status"], - title_kind="~" + sortFnOrPathList=["status"] ) def render_membership_http_api(self): return self._render_http_api_group( - "membership", - title_kind="~" + "membership" ) def render_login_http_api(self): return self._render_http_api_group( - "login", - title_kind="~" + "login" ) def render_room_events(self): @@ -114,6 +115,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -129,7 +133,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -150,7 +155,7 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160..f7862451 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 1a9d981a..0096bbfa 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" @@ -466,6 +467,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__))