From f2c952ec5855bb120814a736bf482a19e1c43fd3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:13:18 +0100 Subject: [PATCH 01/19] Reshuffle spec into sections roughly right as per PR #52 --- .../address-book-repo.rst | 0 ..._repository.rst => 02_03_content_repo.rst} | 0 ...on.rst => 02_04_end_to_end_encryption.rst} | 0 ...ons.rst => 02_05_typing_notifications.rst} | 0 ...{01_02_receipts.rst => 02_06_receipts.rst} | 0 ...ility.rst => 02_07_history_visibility.rst} | 0 ...h_overview.rst => 02_08_push_overview.rst} | 0 ...push_cs_api.rst => 02_08b_push_cs_api.rst} | 0 ...push_gw_api.rst => 02_08c_push_cs_api.rst} | 0 .../03_00_application_service_api.rst | 22 ------------------- ...servers.rst => 05_00_identity_servers.rst} | 0 ...00_appendices.rst => 06_00_appendices.rst} | 0 12 files changed, 22 deletions(-) rename specification/08_00_address_book_repo.rst => drafts/address-book-repo.rst (100%) rename specification/{05_00_content_repository.rst => 02_03_content_repo.rst} (100%) rename specification/{06_00_end_to_end_encryption.rst => 02_04_end_to_end_encryption.rst} (100%) rename specification/{01_01_typing_notifications.rst => 02_05_typing_notifications.rst} (100%) rename specification/{01_02_receipts.rst => 02_06_receipts.rst} (100%) rename specification/{01_03_history_visibility.rst => 02_07_history_visibility.rst} (100%) rename specification/{07_00_push_overview.rst => 02_08_push_overview.rst} (100%) rename specification/{07_01_push_cs_api.rst => 02_08b_push_cs_api.rst} (100%) rename specification/{07_02_push_push_gw_api.rst => 02_08c_push_cs_api.rst} (100%) rename specification/{09_00_identity_servers.rst => 05_00_identity_servers.rst} (100%) rename specification/{10_00_appendices.rst => 06_00_appendices.rst} (100%) diff --git a/specification/08_00_address_book_repo.rst b/drafts/address-book-repo.rst similarity index 100% rename from specification/08_00_address_book_repo.rst rename to drafts/address-book-repo.rst diff --git a/specification/05_00_content_repository.rst b/specification/02_03_content_repo.rst similarity index 100% rename from specification/05_00_content_repository.rst rename to specification/02_03_content_repo.rst diff --git a/specification/06_00_end_to_end_encryption.rst b/specification/02_04_end_to_end_encryption.rst similarity index 100% rename from specification/06_00_end_to_end_encryption.rst rename to specification/02_04_end_to_end_encryption.rst diff --git a/specification/01_01_typing_notifications.rst b/specification/02_05_typing_notifications.rst similarity index 100% rename from specification/01_01_typing_notifications.rst rename to specification/02_05_typing_notifications.rst diff --git a/specification/01_02_receipts.rst b/specification/02_06_receipts.rst similarity index 100% rename from specification/01_02_receipts.rst rename to specification/02_06_receipts.rst diff --git a/specification/01_03_history_visibility.rst b/specification/02_07_history_visibility.rst similarity index 100% rename from specification/01_03_history_visibility.rst rename to specification/02_07_history_visibility.rst diff --git a/specification/07_00_push_overview.rst b/specification/02_08_push_overview.rst similarity index 100% rename from specification/07_00_push_overview.rst rename to specification/02_08_push_overview.rst diff --git a/specification/07_01_push_cs_api.rst b/specification/02_08b_push_cs_api.rst similarity index 100% rename from specification/07_01_push_cs_api.rst rename to specification/02_08b_push_cs_api.rst diff --git a/specification/07_02_push_push_gw_api.rst b/specification/02_08c_push_cs_api.rst similarity index 100% rename from specification/07_02_push_push_gw_api.rst rename to specification/02_08c_push_cs_api.rst diff --git a/specification/03_00_application_service_api.rst b/specification/03_00_application_service_api.rst index 9b58c861..2674ba44 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/03_00_application_service_api.rst @@ -400,25 +400,3 @@ in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix - -Active Application Services ----------------------------- -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - API that provides hooks into the server so that you can intercept and - manipulate events, and/or insert virtual users & rooms into the server. - -Policy Servers -============== -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - We should mention them in the Architecture section at least: how they fit - into the picture. - -Enforcing policies ------------------- - diff --git a/specification/09_00_identity_servers.rst b/specification/05_00_identity_servers.rst similarity index 100% rename from specification/09_00_identity_servers.rst rename to specification/05_00_identity_servers.rst diff --git a/specification/10_00_appendices.rst b/specification/06_00_appendices.rst similarity index 100% rename from specification/10_00_appendices.rst rename to specification/06_00_appendices.rst From c77ef1a2ccca95149c4d9d9d9ee965068190fd23 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:17:31 +0100 Subject: [PATCH 02/19] Add stub files feature_profiles and modules --- specification/00_01_feature_profiles.rst | 0 specification/{02_00_events.rst => 00_02a_events.rst} | 0 .../{02_02_event_signing.rst => 00_02b_event_signing.rst} | 0 specification/02_00_modules.rst | 0 .../{02_08_push_overview.rst => 02_08a_push_overview.rst} | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 specification/00_01_feature_profiles.rst rename specification/{02_00_events.rst => 00_02a_events.rst} (100%) rename specification/{02_02_event_signing.rst => 00_02b_event_signing.rst} (100%) create mode 100644 specification/02_00_modules.rst rename specification/{02_08_push_overview.rst => 02_08a_push_overview.rst} (100%) diff --git a/specification/00_01_feature_profiles.rst b/specification/00_01_feature_profiles.rst new file mode 100644 index 00000000..e69de29b diff --git a/specification/02_00_events.rst b/specification/00_02a_events.rst similarity index 100% rename from specification/02_00_events.rst rename to specification/00_02a_events.rst diff --git a/specification/02_02_event_signing.rst b/specification/00_02b_event_signing.rst similarity index 100% rename from specification/02_02_event_signing.rst rename to specification/00_02b_event_signing.rst diff --git a/specification/02_00_modules.rst b/specification/02_00_modules.rst new file mode 100644 index 00000000..e69de29b diff --git a/specification/02_08_push_overview.rst b/specification/02_08a_push_overview.rst similarity index 100% rename from specification/02_08_push_overview.rst rename to specification/02_08a_push_overview.rst From f520ac9d828890193772764e902c3ae5b4b77c1d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:25:52 +0100 Subject: [PATCH 03/19] Create a modules folder to group together similar parts of the spec --- .../{02_01_voip_events.rst => modules/01_00_voip_events.rst} | 0 .../02_00_typing_notifications.rst} | 0 specification/{02_06_receipts.rst => modules/03_00_receipts.rst} | 0 .../{02_03_content_repo.rst => modules/04_00_content_repo.rst} | 0 .../05_00_end_to_end_encryption.rst} | 0 .../06_00_history_visibility.rst} | 0 .../{02_08a_push_overview.rst => modules/07_00_push_overview.rst} | 0 .../{02_08b_push_cs_api.rst => modules/07_01_push_cs_api.rst} | 0 .../07_02_push_push_gw_api.rst} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename specification/{02_01_voip_events.rst => modules/01_00_voip_events.rst} (100%) rename specification/{02_05_typing_notifications.rst => modules/02_00_typing_notifications.rst} (100%) rename specification/{02_06_receipts.rst => modules/03_00_receipts.rst} (100%) rename specification/{02_03_content_repo.rst => modules/04_00_content_repo.rst} (100%) rename specification/{02_04_end_to_end_encryption.rst => modules/05_00_end_to_end_encryption.rst} (100%) rename specification/{02_07_history_visibility.rst => modules/06_00_history_visibility.rst} (100%) rename specification/{02_08a_push_overview.rst => modules/07_00_push_overview.rst} (100%) rename specification/{02_08b_push_cs_api.rst => modules/07_01_push_cs_api.rst} (100%) rename specification/{02_08c_push_cs_api.rst => modules/07_02_push_push_gw_api.rst} (100%) diff --git a/specification/02_01_voip_events.rst b/specification/modules/01_00_voip_events.rst similarity index 100% rename from specification/02_01_voip_events.rst rename to specification/modules/01_00_voip_events.rst diff --git a/specification/02_05_typing_notifications.rst b/specification/modules/02_00_typing_notifications.rst similarity index 100% rename from specification/02_05_typing_notifications.rst rename to specification/modules/02_00_typing_notifications.rst diff --git a/specification/02_06_receipts.rst b/specification/modules/03_00_receipts.rst similarity index 100% rename from specification/02_06_receipts.rst rename to specification/modules/03_00_receipts.rst diff --git a/specification/02_03_content_repo.rst b/specification/modules/04_00_content_repo.rst similarity index 100% rename from specification/02_03_content_repo.rst rename to specification/modules/04_00_content_repo.rst diff --git a/specification/02_04_end_to_end_encryption.rst b/specification/modules/05_00_end_to_end_encryption.rst similarity index 100% rename from specification/02_04_end_to_end_encryption.rst rename to specification/modules/05_00_end_to_end_encryption.rst diff --git a/specification/02_07_history_visibility.rst b/specification/modules/06_00_history_visibility.rst similarity index 100% rename from specification/02_07_history_visibility.rst rename to specification/modules/06_00_history_visibility.rst diff --git a/specification/02_08a_push_overview.rst b/specification/modules/07_00_push_overview.rst similarity index 100% rename from specification/02_08a_push_overview.rst rename to specification/modules/07_00_push_overview.rst diff --git a/specification/02_08b_push_cs_api.rst b/specification/modules/07_01_push_cs_api.rst similarity index 100% rename from specification/02_08b_push_cs_api.rst rename to specification/modules/07_01_push_cs_api.rst diff --git a/specification/02_08c_push_cs_api.rst b/specification/modules/07_02_push_push_gw_api.rst similarity index 100% rename from specification/02_08c_push_cs_api.rst rename to specification/modules/07_02_push_push_gw_api.rst From 703f913a47afc3b035a1730e9014e89807370b3e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:31:57 +0100 Subject: [PATCH 04/19] Add a 'targets.yaml' files in /specification We're well beyond the point now where a simple `cat` of .rst files to "build" the spec is practical. We may want to slice and dice the spec in different ways to address various cross-cutting concerns. To this end, there is now a 'targets' file which contains the "build targets" for the spec, which contains the sorting order for the .rst files. For now, we just have a single target: 'main'. --- specification/targets.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 specification/targets.yaml diff --git a/specification/targets.yaml b/specification/targets.yaml new file mode 100644 index 00000000..845da99d --- /dev/null +++ b/specification/targets.yaml @@ -0,0 +1,25 @@ +targets: + main: + files: + - 00_00_intro.rst + - 00_01_feature_profiles.rst + - 00_02a_events.rst + - 00_02b_event_signing.rst + - 01_00_client_server_api.rst + - 02_00_modules.rst + - "group:module" + - 03_00_application_service_api.rst + - 04_00_server_server_api.rst + - 05_00_identity_servers.rst + - 06_00_appendices.rst +groups: + modules: + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - modules/07_00_push_overview.rst + - modules/07_01_push_cs_api.rst + - modules/07_02_push_push_gw_api.rst From 568982ee2e5a3dbf2f76556475dfdd8fae96998d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:49:54 +0100 Subject: [PATCH 05/19] Represent nested deps in targets.yaml along with title styles This will allow us to programatically position .rst snippets *anywhere* which will for once and for all remove the horrid title level mismatch bugs. We require this in order to allow people to re-shuffle the spec without having to adjust the spec itself (e.g. 2 targets with different levels of nesting). --- specification/targets.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/specification/targets.yaml b/specification/targets.yaml index 845da99d..b61e2e20 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -20,6 +20,10 @@ groups: - modules/04_00_content_repo.rst - modules/05_00_end_to_end_encryption.rst - modules/06_00_history_visibility.rst - - modules/07_00_push_overview.rst - - modules/07_01_push_cs_api.rst - - modules/07_02_push_push_gw_api.rst + - 1: modules/07_00_push_overview.rst + 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] +title_styles: + - "=" + - "-" + - "~" + - "+" From 8590cc84b51ba81b14b0f73988879368c26e649f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 15:33:28 +0100 Subject: [PATCH 06/19] Process and extract targets in gendoc.py --- scripts/gendoc.py | 74 +++++++++++++++++++++++++++++++------- specification/targets.yaml | 10 +++--- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 3521efed..69c39dcc 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -8,6 +8,7 @@ import re import shutil import subprocess import sys +import yaml os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -24,7 +25,7 @@ SECOND_LEVEL = "-" FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") -def check_valid_section(filename, section): +def check_valid_section_old(filename, section): if not re.match(FILE_FORMAT_MATCHER, filename): raise Exception( "The filename of " + filename + " does not match the expected format " + @@ -67,13 +68,19 @@ def check_valid_section(filename, section): "check." ) -def cat_spec_sections_to(out_file_name): - with open(out_file_name, "wb") as outfile: - for f in sorted(glob.glob("../specification/*.rst")): - with open(f, "rb") as infile: - section = infile.read() - check_valid_section(os.path.basename(f), section) - outfile.write(section) +def check_valid_section(section): + pass + + +def get_rst(file_info, target): + pass + +def build_spec(target, out_filename): + with open(out_filename, "wb") as outfile: + for file_info in target["files"]: + section = get_rst(file_info, target) + check_valid_section(section) + outfile.write(section) def rst2html(i, o): @@ -88,6 +95,7 @@ def rst2html(i, o): settings_overrides=stylesheets ) + def run_through_template(input): tmpfile = './tmp/output' try: @@ -107,6 +115,43 @@ def run_through_template(input): sys.stderr.write(f.read() + "\n") raise + +def get_build_target(targets_listing, target_name): + build_target = { + "title_styles": [], + "files": [] + } + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + build_target["title_styles"] = all_targets["title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + resolved_files = [] + for f in target["files"]: + if isinstance(f, basestring) and f.startswith("group:"): + # copy across the group of files specified + group_name = f[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not isinstance(group, list): + raise Exception( + "Tried to find group '" + group_name + "' but either " + + "it doesn't exist or it isn't a list of files." + ) + resolved_files.extend(group) + else: + resolved_files.append(f) + build_target["files"] = resolved_files + return build_target + + def prepare_env(): try: os.makedirs("./gen") @@ -116,13 +161,17 @@ def prepare_env(): os.makedirs("./tmp") except OSError: pass - + + def cleanup_env(): shutil.rmtree("./tmp") -def main(): + +def main(target_name): prepare_env() - cat_spec_sections_to("tmp/full_spec.rst") + target = get_build_target("../specification/targets.yaml", target_name) + print target + build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") @@ -131,6 +180,7 @@ def main(): if "--nodelete" not in sys.argv: cleanup_env() + if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: # we accept almost no args, so they don't know what they're doing! @@ -145,4 +195,4 @@ if __name__ == '__main__': print "Requirements:" print " - This script requires Jinja2 and rst2html (docutils)." sys.exit(0) - main() + main("main") diff --git a/specification/targets.yaml b/specification/targets.yaml index b61e2e20..c4828ef1 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,18 +1,18 @@ targets: - main: - files: + main: # arbitrary name to identify this build target + files: # the sort order of files to cat - 00_00_intro.rst - 00_01_feature_profiles.rst - 00_02a_events.rst - 00_02b_event_signing.rst - 01_00_client_server_api.rst - 02_00_modules.rst - - "group:module" + - "group:modules" # reference a group of files - 03_00_application_service_api.rst - 04_00_server_server_api.rst - 05_00_identity_servers.rst - 06_00_appendices.rst -groups: +groups: # reusable blobs of files when prefixed with 'group:' modules: - modules/01_00_voip_events.rst - modules/02_00_typing_notifications.rst @@ -20,6 +20,7 @@ groups: - modules/04_00_content_repo.rst - modules/05_00_end_to_end_encryption.rst - modules/06_00_history_visibility.rst + # Mark a nested file dependency - 1: modules/07_00_push_overview.rst 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] title_styles: @@ -27,3 +28,4 @@ title_styles: - "-" - "~" - "+" + From 65ce95249f99fb67e61377e5b35734522a6d7ba3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 16:54:30 +0100 Subject: [PATCH 07/19] cat the spec according to the build target. Remove old checks as they are now obsolete since we don't care about the filename --- scripts/gendoc.py | 121 ++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 69c39dcc..22aaad11 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -16,70 +16,74 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css"] } -title_style_matchers = { - "=": re.compile("^=+$"), - "-": re.compile("^-+$") -} -TOP_LEVEL = "=" -SECOND_LEVEL = "-" -FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") - - -def check_valid_section_old(filename, section): - if not re.match(FILE_FORMAT_MATCHER, filename): - raise Exception( - "The filename of " + filename + " does not match the expected format " + - "of '##_##_words-go-here.rst'" - ) - - # we need TWO new lines else the next file's title gets merged - # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == "\n\n": - raise Exception( - "The file " + filename + " does not end with 2 new lines." - ) - - # Enforce some rules to reduce the risk of having mismatched title - # styles. - title_line = section.split("\n")[1] - if title_line != (len(title_line) * title_line[0]): - raise Exception( - "The file " + filename + " doesn't have a title style line on line 2" - ) - - # anything marked as xx_00_ is the start of a new top-level section - if re.match("^[0-9]+_00_", filename): - if not title_style_matchers[TOP_LEVEL].match(title_line): - raise Exception( - "The file " + filename + " is a top-level section because it matches " + - "the filename format ##_00_something.rst but has the wrong title " + - "style: expected '" + TOP_LEVEL + "' but got '" + - title_line[0] + "'" - ) - # anything marked as xx_xx_ is the start of a sub-section - elif re.match("^[0-9]+_[0-9]{2}_", filename): - if not title_style_matchers[SECOND_LEVEL].match(title_line): - raise Exception( - "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format ##_##_something.rst but has the wrong title " + - "style: expected '" + SECOND_LEVEL + "' but got '" + - title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " + - "then use the form '##_##b_something.rst' which will not apply this " + - "check." - ) - -def check_valid_section(section): - pass +def _list_get(l, index, default=None): + try: + return l[index] + except IndexError: + return default + + +def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): + rst_lines = [] + title_chars = "".join(title_styles) + title_regex = re.compile("^[" + re.escape(title_chars) + "]+$") + + curr_title_level = title_level + for i, line in enumerate(file_stream, 1): + if title_regex.match(line): + line_title_level = title_styles.index(line[0]) + # Allowed to go 1 deeper or any number shallower + if curr_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line[0], title_styles, + title_styles[curr_title_level]) + ) + curr_title_level = line_title_level + rst_lines.append(line) + else: + rst_lines.append(line) + return "".join(rst_lines) + + + +def get_rst(file_info, title_level, title_styles, spec_dir): + # string are file paths to RST blobs + if isinstance(file_info, basestring): + with open(spec_dir + file_info, "r") as f: + return load_with_adjusted_titles(file_info, f, title_level, title_styles) + # dicts look like {1: filepath, 2: filepath} where the key is the title level + elif isinstance(file_info, dict): + levels = sorted(file_info.keys()) + rst = [] + for l in levels: + rst.append(get_rst(file_info[l], l, title_styles, spec_dir)) + return "".join(rst) + # lists are multiple file paths e.g. [filepath, filepath] + elif isinstance(file_info, list): + rst = [] + for f in file_info: + rst.append(get_rst(f, title_level, title_styles, spec_dir)) + return "".join(rst) + raise Exception( + "The following 'file' entry in this target isn't a string, list or dict. " + + "It really really should be. Entry: %s" % (file_info,) + ) -def get_rst(file_info, target): - pass def build_spec(target, out_filename): with open(out_filename, "wb") as outfile: for file_info in target["files"]: - section = get_rst(file_info, target) - check_valid_section(section) + section = get_rst( + file_info=file_info, + title_level=0, + title_styles=target["title_styles"], + spec_dir="../specification/" + ) outfile.write(section) @@ -170,7 +174,6 @@ def cleanup_env(): def main(target_name): prepare_env() target = get_build_target("../specification/targets.yaml", target_name) - print target build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") From 067363c629bcf140e925980d20bcf8b264407cac Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 17:21:08 +0100 Subject: [PATCH 08/19] Get the desired title levels right; print out the actual level used to stdout --- scripts/gendoc.py | 16 ++++++++----- specification/modules/00_modules_intro.rst | 0 specification/targets.yaml | 26 +++++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 specification/modules/00_modules_intro.rst diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2e303ea2..f7bde162 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -50,13 +50,13 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): return "".join(rst_lines) - def get_rst(file_info, title_level, title_styles, spec_dir): # string are file paths to RST blobs if isinstance(file_info, basestring): + print "%s %s" % (">" * (1 + title_level), file_info) with open(spec_dir + file_info, "r") as f: return load_with_adjusted_titles(file_info, f, title_level, title_styles) - # dicts look like {1: filepath, 2: filepath} where the key is the title level + # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): levels = sorted(file_info.keys()) rst = [] @@ -144,12 +144,15 @@ def get_build_target(targets_listing, target_name): # copy across the group of files specified group_name = f[len("group:"):] group = all_targets.get("groups", {}).get(group_name) - if not isinstance(group, list): + if not group: raise Exception( - "Tried to find group '" + group_name + "' but either " + - "it doesn't exist or it isn't a list of files." + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." ) - resolved_files.extend(group) + if isinstance(group, list): + resolved_files.extend(group) + else: + resolved_files.append(group) else: resolved_files.append(f) build_target["files"] = resolved_files @@ -173,6 +176,7 @@ def cleanup_env(): def main(target_name): prepare_env() + print "Building spec [target=%s]" % target_name target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/00_modules_intro.rst new file mode 100644 index 00000000..e69de29b diff --git a/specification/targets.yaml b/specification/targets.yaml index c4828ef1..e492ad3c 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,18 +14,14 @@ targets: - 06_00_appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - # Mark a nested file dependency - - 1: modules/07_00_push_overview.rst - 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] -title_styles: - - "=" - - "-" - - "~" - - "+" - + 0: modules/00_modules_intro.rst + 1: + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - 1: modules/07_00_push_overview.rst # Mark a nested file dependency + 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] +title_styles: ["=", "-", "~", "+"] From f71763b0d3b6d95ed7839f5c9735d21aa59d5d71 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 13:08:15 +0100 Subject: [PATCH 09/19] Implement relative title styles Templates don't know at what level they will be inserted. Previously, we hard-coded the title style which is not compatible with the build target system. Define a set of styles which will be replaced by the gendoc script when it encounters them: '<' : Make this title a sub-heading '/' : Make this title a heading at the same level '>' : Make this title a super-heading The build target system is now basically complete and functioning. --- scripts/gendoc.py | 234 +++++++++++++++--- specification/00_01_feature_profiles.rst | 5 + specification/02_00_modules.rst | 5 + .../03_00_application_service_api.rst | 1 + specification/04_00_server_server_api.rst | 4 +- specification/modules/00_modules_intro.rst | 5 + specification/targets.yaml | 41 +-- templating/matrix_templates/sections.py | 35 +-- .../matrix_templates/templates/msgtypes.tmpl | 2 +- templating/matrix_templates/units.py | 7 + 10 files changed, 268 insertions(+), 71 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f7bde162..308d148f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -17,57 +17,133 @@ stylesheets = { } -def _list_get(l, index, default=None): - try: - return l[index] - except IndexError: - return default - - +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): rst_lines = [] title_chars = "".join(title_styles) - title_regex = re.compile("^[" + re.escape(title_chars) + "]+$") + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") - curr_title_level = title_level + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None for i, line in enumerate(file_stream, 1): - if title_regex.match(line): - line_title_level = title_styles.index(line[0]) - # Allowed to go 1 deeper or any number shallower - if curr_title_level - line_title_level < -1: - raise Exception( - ("File '%s' line '%s' has a title " + - "style '%s' which doesn't match one of the " + - "allowed title styles of %s because the " + - "title level before this line was '%s'") % - (filename, (i + 1), line[0], title_styles, - title_styles[curr_title_level]) - ) - curr_title_level = line_title_level + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): rst_lines.append(line) - else: + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): rst_lines.append(line) + prev_non_title_line = line + continue + + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) + + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + print (" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0]) + + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) + ) + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): + raise Exception( + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) + ) + + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + # print ( + # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + # (filename, line_title_style, + # title_styles[adjusted_level], + # file_offset, title_level) + # ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + return "".join(rst_lines) -def get_rst(file_info, title_level, title_styles, spec_dir): +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): print "%s %s" % (">" * (1 + title_level), file_info) with open(spec_dir + file_info, "r") as f: - return load_with_adjusted_titles(file_info, f, title_level, title_styles) + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + if rst[-2:] != "\n\n": + raise Exception( + ("File %s should end with TWO new-line characters to ensure " + + "file concatenation works correctly.") % (file_info,) + ) + return rst # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): levels = sorted(file_info.keys()) rst = [] for l in levels: - rst.append(get_rst(file_info[l], l, title_styles, spec_dir)) + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) return "".join(rst) # lists are multiple file paths e.g. [filepath, filepath] elif isinstance(file_info, list): rst = [] for f in file_info: - rst.append(get_rst(f, title_level, title_styles, spec_dir)) + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) return "".join(rst) raise Exception( "The following 'file' entry in this target isn't a string, list or dict. " + @@ -82,11 +158,74 @@ def build_spec(target, out_filename): file_info=file_info, title_level=0, title_styles=target["title_styles"], - spec_dir="../specification/" + spec_dir="../specification/", + adjust_titles=True ) outfile.write(section) +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] # ["=", "-", "~", "+"] + relative_title_chars = [ # ["<", "/", ">"] + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + + + def rst2html(i, o): with open(i, "r") as in_file: with open(o, "w") as out_file: @@ -123,11 +262,13 @@ def run_through_template(input): def get_build_target(targets_listing, target_name): build_target = { "title_styles": [], + "relative_title_styles": {}, "files": [] } with open(targets_listing, "r") as targ_file: all_targets = yaml.load(targ_file.read()) build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] target = all_targets["targets"].get(target_name) if not target: raise Exception( @@ -138,17 +279,30 @@ def get_build_target(targets_listing, target_name): raise Exception( "Found target but 'files' key is not a list." ) + + def get_group(group_id): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." + ) + return group + resolved_files = [] for f in target["files"]: + group = None if isinstance(f, basestring) and f.startswith("group:"): - # copy across the group of files specified - group_name = f[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." - ) + group = get_group(f) + elif isinstance(f, dict): + for (k, v) in f.iteritems(): + if isinstance(v, basestring) and v.startswith("group:"): + f[k] = get_group(v) + resolved_files.append(f) + continue + + if group: if isinstance(group, list): resolved_files.extend(group) else: @@ -178,8 +332,12 @@ def main(target_name): prepare_env() print "Building spec [target=%s]" % target_name target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst") + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") diff --git a/specification/00_01_feature_profiles.rst b/specification/00_01_feature_profiles.rst index e69de29b..155e51c5 100644 --- a/specification/00_01_feature_profiles.rst +++ b/specification/00_01_feature_profiles.rst @@ -0,0 +1,5 @@ +Feature Profiles +================ + +Feature profiles blurb goes here. + diff --git a/specification/02_00_modules.rst b/specification/02_00_modules.rst index e69de29b..ab35fe9f 100644 --- a/specification/02_00_modules.rst +++ b/specification/02_00_modules.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules intro here. + diff --git a/specification/03_00_application_service_api.rst b/specification/03_00_application_service_api.rst index 2674ba44..e982390b 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/03_00_application_service_api.rst @@ -400,3 +400,4 @@ in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix + diff --git a/specification/04_00_server_server_api.rst b/specification/04_00_server_server_api.rst index f5cadabf..8d1f8898 100644 --- a/specification/04_00_server_server_api.rst +++ b/specification/04_00_server_server_api.rst @@ -92,7 +92,7 @@ server by querying other servers. .. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of @@ -178,7 +178,7 @@ events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/00_modules_intro.rst index e69de29b..fdfcbbdc 100644 --- a/specification/modules/00_modules_intro.rst +++ b/specification/modules/00_modules_intro.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules blurb goes here. + diff --git a/specification/targets.yaml b/specification/targets.yaml index e492ad3c..7bd3d882 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,26 +2,37 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - 00_00_intro.rst - - 00_01_feature_profiles.rst - - 00_02a_events.rst - - 00_02b_event_signing.rst + - { 1: 00_01_feature_profiles.rst } + - { 1: 00_02a_events.rst } + - { 1: 00_02b_event_signing.rst } - 01_00_client_server_api.rst - 02_00_modules.rst - - "group:modules" # reference a group of files + - { 1: "group:modules" } # reference a group of files - 03_00_application_service_api.rst - 04_00_server_server_api.rst - 05_00_identity_servers.rst - 06_00_appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - 0: modules/00_modules_intro.rst - 1: - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - - 1: modules/07_00_push_overview.rst # Mark a nested file dependency - 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] -title_styles: ["=", "-", "~", "+"] + - modules/00_modules_intro.rst + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - modules/07_00_push_overview.rst + - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] } + +title_styles: ["=", "-", "~", "+", "^"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 729157bb..6072222a 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,15 +72,14 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) def render_profile_http_api(self): return self._render_http_api_group( "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" + sortFnOrPathList=["displayname", "avatar_url"] ) def render_sync_http_api(self): @@ -86,20 +90,17 @@ class MatrixSections(Sections): def render_presence_http_api(self): return self._render_http_api_group( "presence", - sortFnOrPathList=["status"], - title_kind="~" + sortFnOrPathList=["status"] ) def render_membership_http_api(self): return self._render_http_api_group( - "membership", - title_kind="~" + "membership" ) def render_login_http_api(self): return self._render_http_api_group( - "login", - title_kind="~" + "login" ) def render_room_events(self): @@ -114,6 +115,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -129,7 +133,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -150,7 +155,7 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160..f7862451 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 1a9d981a..0096bbfa 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" @@ -466,6 +467,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__)) From e8cdfcbba28223055a182e06c1ee1a4f04cc3738 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 14:01:55 +0100 Subject: [PATCH 10/19] Rename spec files to not have numbers everywhere; update targets.yaml --- ..._event_signing.rst => 0-event_signing.rst} | 0 .../{00_02a_events.rst => 0-events.rst} | 0 ...re_profiles.rst => 0-feature_profiles.rst} | 0 .../{00_00_intro.rst => 0-intro.rst} | 0 ...server_api.rst => 1-client_server_api.rst} | 0 .../{02_00_modules.rst => 2-modules.rst} | 0 ..._api.rst => 3-application_service_api.rst} | 0 ...server_api.rst => 4-server_server_api.rst} | 0 ...ity_servers.rst => 5-identity_servers.rst} | 0 ...{06_00_appendices.rst => 6-appendices.rst} | 0 ...4_00_content_repo.rst => content_repo.rst} | 0 ...cryption.rst => end_to_end_encryption.rst} | 0 ..._visibility.rst => history_visibility.rst} | 0 .../{00_modules_intro.rst => intro.rst} | 0 ...{07_01_push_cs_api.rst => push_cs_api.rst} | 0 ...00_push_overview.rst => push_overview.rst} | 0 ...h_push_gw_api.rst => push_push_gw_api.rst} | 0 .../{03_00_receipts.rst => receipts.rst} | 0 ...fications.rst => typing_notifications.rst} | 0 ...{01_00_voip_events.rst => voip_events.rst} | 0 specification/targets.yaml | 38 +++++++++---------- 21 files changed, 19 insertions(+), 19 deletions(-) rename specification/{00_02b_event_signing.rst => 0-event_signing.rst} (100%) rename specification/{00_02a_events.rst => 0-events.rst} (100%) rename specification/{00_01_feature_profiles.rst => 0-feature_profiles.rst} (100%) rename specification/{00_00_intro.rst => 0-intro.rst} (100%) rename specification/{01_00_client_server_api.rst => 1-client_server_api.rst} (100%) rename specification/{02_00_modules.rst => 2-modules.rst} (100%) rename specification/{03_00_application_service_api.rst => 3-application_service_api.rst} (100%) rename specification/{04_00_server_server_api.rst => 4-server_server_api.rst} (100%) rename specification/{05_00_identity_servers.rst => 5-identity_servers.rst} (100%) rename specification/{06_00_appendices.rst => 6-appendices.rst} (100%) rename specification/modules/{04_00_content_repo.rst => content_repo.rst} (100%) rename specification/modules/{05_00_end_to_end_encryption.rst => end_to_end_encryption.rst} (100%) rename specification/modules/{06_00_history_visibility.rst => history_visibility.rst} (100%) rename specification/modules/{00_modules_intro.rst => intro.rst} (100%) rename specification/modules/{07_01_push_cs_api.rst => push_cs_api.rst} (100%) rename specification/modules/{07_00_push_overview.rst => push_overview.rst} (100%) rename specification/modules/{07_02_push_push_gw_api.rst => push_push_gw_api.rst} (100%) rename specification/modules/{03_00_receipts.rst => receipts.rst} (100%) rename specification/modules/{02_00_typing_notifications.rst => typing_notifications.rst} (100%) rename specification/modules/{01_00_voip_events.rst => voip_events.rst} (100%) diff --git a/specification/00_02b_event_signing.rst b/specification/0-event_signing.rst similarity index 100% rename from specification/00_02b_event_signing.rst rename to specification/0-event_signing.rst diff --git a/specification/00_02a_events.rst b/specification/0-events.rst similarity index 100% rename from specification/00_02a_events.rst rename to specification/0-events.rst diff --git a/specification/00_01_feature_profiles.rst b/specification/0-feature_profiles.rst similarity index 100% rename from specification/00_01_feature_profiles.rst rename to specification/0-feature_profiles.rst diff --git a/specification/00_00_intro.rst b/specification/0-intro.rst similarity index 100% rename from specification/00_00_intro.rst rename to specification/0-intro.rst diff --git a/specification/01_00_client_server_api.rst b/specification/1-client_server_api.rst similarity index 100% rename from specification/01_00_client_server_api.rst rename to specification/1-client_server_api.rst diff --git a/specification/02_00_modules.rst b/specification/2-modules.rst similarity index 100% rename from specification/02_00_modules.rst rename to specification/2-modules.rst diff --git a/specification/03_00_application_service_api.rst b/specification/3-application_service_api.rst similarity index 100% rename from specification/03_00_application_service_api.rst rename to specification/3-application_service_api.rst diff --git a/specification/04_00_server_server_api.rst b/specification/4-server_server_api.rst similarity index 100% rename from specification/04_00_server_server_api.rst rename to specification/4-server_server_api.rst diff --git a/specification/05_00_identity_servers.rst b/specification/5-identity_servers.rst similarity index 100% rename from specification/05_00_identity_servers.rst rename to specification/5-identity_servers.rst diff --git a/specification/06_00_appendices.rst b/specification/6-appendices.rst similarity index 100% rename from specification/06_00_appendices.rst rename to specification/6-appendices.rst diff --git a/specification/modules/04_00_content_repo.rst b/specification/modules/content_repo.rst similarity index 100% rename from specification/modules/04_00_content_repo.rst rename to specification/modules/content_repo.rst diff --git a/specification/modules/05_00_end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst similarity index 100% rename from specification/modules/05_00_end_to_end_encryption.rst rename to specification/modules/end_to_end_encryption.rst diff --git a/specification/modules/06_00_history_visibility.rst b/specification/modules/history_visibility.rst similarity index 100% rename from specification/modules/06_00_history_visibility.rst rename to specification/modules/history_visibility.rst diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/intro.rst similarity index 100% rename from specification/modules/00_modules_intro.rst rename to specification/modules/intro.rst diff --git a/specification/modules/07_01_push_cs_api.rst b/specification/modules/push_cs_api.rst similarity index 100% rename from specification/modules/07_01_push_cs_api.rst rename to specification/modules/push_cs_api.rst diff --git a/specification/modules/07_00_push_overview.rst b/specification/modules/push_overview.rst similarity index 100% rename from specification/modules/07_00_push_overview.rst rename to specification/modules/push_overview.rst diff --git a/specification/modules/07_02_push_push_gw_api.rst b/specification/modules/push_push_gw_api.rst similarity index 100% rename from specification/modules/07_02_push_push_gw_api.rst rename to specification/modules/push_push_gw_api.rst diff --git a/specification/modules/03_00_receipts.rst b/specification/modules/receipts.rst similarity index 100% rename from specification/modules/03_00_receipts.rst rename to specification/modules/receipts.rst diff --git a/specification/modules/02_00_typing_notifications.rst b/specification/modules/typing_notifications.rst similarity index 100% rename from specification/modules/02_00_typing_notifications.rst rename to specification/modules/typing_notifications.rst diff --git a/specification/modules/01_00_voip_events.rst b/specification/modules/voip_events.rst similarity index 100% rename from specification/modules/01_00_voip_events.rst rename to specification/modules/voip_events.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index 7bd3d882..df61e7a1 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,28 +1,28 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - - 00_00_intro.rst - - { 1: 00_01_feature_profiles.rst } - - { 1: 00_02a_events.rst } - - { 1: 00_02b_event_signing.rst } - - 01_00_client_server_api.rst - - 02_00_modules.rst + - 0-intro.rst + - { 1: 0-feature_profiles.rst } + - { 1: 0-events.rst } + - { 1: 0-event_signing.rst } + - 1-client_server_api.rst + - 2-modules.rst - { 1: "group:modules" } # reference a group of files - - 03_00_application_service_api.rst - - 04_00_server_server_api.rst - - 05_00_identity_servers.rst - - 06_00_appendices.rst + - 3-application_service_api.rst + - 4-server_server_api.rst + - 5-identity_servers.rst + - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/00_modules_intro.rst - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - - modules/07_00_push_overview.rst - - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] } + - modules/intro.rst + - modules/voip_events.rst + - modules/typing_notifications.rst + - modules/receipts.rst + - modules/content_repo.rst + - modules/end_to_end_encryption.rst + - modules/history_visibility.rst + - modules/push_overview.rst + - { 2: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } title_styles: ["=", "-", "~", "+", "^"] From 0c0ac3e814c763e271d7a0c9b5c9b5e824b2f2f3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 14:07:23 +0100 Subject: [PATCH 11/19] Fix common event fields template to use subtitle char --- specification/modules/intro.rst | 5 ----- specification/targets.yaml | 1 - templating/matrix_templates/sections.py | 7 ++++++- .../matrix_templates/templates/common-event-fields.tmpl | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 specification/modules/intro.rst diff --git a/specification/modules/intro.rst b/specification/modules/intro.rst deleted file mode 100644 index fdfcbbdc..00000000 --- a/specification/modules/intro.rst +++ /dev/null @@ -1,5 +0,0 @@ -Modules -======= - -Modules blurb goes here. - diff --git a/specification/targets.yaml b/specification/targets.yaml index df61e7a1..96527dae 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,7 +14,6 @@ targets: - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/intro.rst - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 6072222a..78f011b9 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -160,7 +160,12 @@ class MatrixSections(Sections): def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") ce_types = self.units.get("common_event_fields") - return template.render(common_event=ce_types[type]) + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + return template.render( + common_event=ce_types[type], title_kind=subtitle_title_char + ) def render_common_event_fields(self): return self._render_ce_type("event") diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 2a3f7ec5..3f16be3d 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,5 +1,5 @@ {{common_event.title}} Fields -{{(7 + common_event.title | length) * '-'}} +{{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} From 388aeefac000d89e9b775e1fb7905b8e8f178ae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:07:19 +0100 Subject: [PATCH 12/19] Remove obsolete key --- templating/matrix_templates/sections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 15dca9b1..cacaa06a 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -105,8 +105,7 @@ class MatrixSections(Sections): def render_rooms_http_api(self): return self._render_http_api_group( - "rooms", - title_kind="+" + "rooms" ) def render_room_events(self): From b21859836d1b6ca0df6f4072c393211706eae9a8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:11:16 +0100 Subject: [PATCH 13/19] Print stdout of build.py in verbose mode --- scripts/gendoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 308d148f..6c95e33c 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -243,9 +243,9 @@ def run_through_template(input): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: - subprocess.check_output( + print subprocess.check_output( [ - 'python', 'build.py', + 'python', 'build.py', "-v", "-i", "matrix_templates", "-o", "../scripts/tmp", "../scripts/"+input From 16693a644aead93c67ad1f10d3afb1d7fc0169a5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:59:49 +0100 Subject: [PATCH 14/19] Various review fixes --- scripts/gendoc.py | 105 ++++++++++++++++----------- specification/0-feature_profiles.rst | 2 - specification/2-modules.rst | 2 - 3 files changed, 62 insertions(+), 47 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 6c95e33c..26ee5a48 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -118,7 +118,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): print "%s %s" % (">" * (1 + title_level), file_info) - with open(spec_dir + file_info, "r") as f: + with open(os.path.join(spec_dir, file_info), "r") as f: rst = None if adjust_titles: rst = load_with_adjusted_titles( @@ -175,8 +175,8 @@ This function replaces these relative titles with actual title styles from the array in targets.yaml. """ def fix_relative_titles(target, filename, out_filename): - title_styles = target["title_styles"] # ["=", "-", "~", "+"] - relative_title_chars = [ # ["<", "/", ">"] + title_styles = target["title_styles"] + relative_title_chars = [ target["relative_title_styles"]["subtitle"], target["relative_title_styles"]["sametitle"], target["relative_title_styles"]["supertitle"] @@ -259,6 +259,17 @@ def run_through_template(input): raise +""" +Extract and resolve groups for the given target in the given targets listing. +Args: + targets_listing (str): The path to a YAML file containing a list of targets + target_name (str): The name of the target to extract from the listings. +Returns: + dict: Containing "filees" (a list of file paths), "relative_title_styles" + (a dict of relative style keyword to title character) and "title_styles" + (a list of characters which represent the global title style to follow, + with the top section title first, the second section second, and so on.) +""" def get_build_target(targets_listing, target_name): build_target = { "title_styles": [], @@ -267,49 +278,57 @@ def get_build_target(targets_listing, target_name): } with open(targets_listing, "r") as targ_file: all_targets = yaml.load(targ_file.read()) - build_target["title_styles"] = all_targets["title_styles"] - build_target["relative_title_styles"] = all_targets["relative_title_styles"] - target = all_targets["targets"].get(target_name) - if not target: - raise Exception( - "No target by the name '" + target_name + "' exists in '" + - targets_listing + "'." - ) - if not isinstance(target.get("files"), list): + + build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + + def get_group(group_id): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: raise Exception( - "Found target but 'files' key is not a list." + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." ) - - def get_group(group_id): - group_name = group_id[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." - ) - return group - - resolved_files = [] - for f in target["files"]: - group = None - if isinstance(f, basestring) and f.startswith("group:"): - group = get_group(f) - elif isinstance(f, dict): - for (k, v) in f.iteritems(): - if isinstance(v, basestring) and v.startswith("group:"): - f[k] = get_group(v) - resolved_files.append(f) - continue - - if group: - if isinstance(group, list): - resolved_files.extend(group) - else: - resolved_files.append(group) + return group + + resolved_files = [] + for file_entry in target["files"]: + # file_entry is a group id + if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + group = get_group(file_entry) + # The group may be resolved to a list of file entries, in which case + # we want to extend the array to insert each of them rather than + # insert the entire list as a single element (which is what append does) + if isinstance(group, list): + resolved_files.extend(group) else: - resolved_files.append(f) - build_target["files"] = resolved_files + resolved_files.append(group) + # file_entry is a dict which has more file entries as values + elif isinstance(file_entry, dict): + resolved_entry = {} + for (k, v) in file_entry.iteritems(): + if isinstance(v, basestring) and v.startswith("group:"): + resolved_entry[k] = get_group(v) + else: + # map across without editing (e.g. normal file path) + resolved_entry[k] = v + resolved_files.append(resolved_entry) + continue + # file_entry is just a plain ol' file path + else: + resolved_files.append(file_entry) + build_target["files"] = resolved_files return build_target diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 155e51c5..234e14db 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -1,5 +1,3 @@ Feature Profiles ================ -Feature profiles blurb goes here. - diff --git a/specification/2-modules.rst b/specification/2-modules.rst index ab35fe9f..0aad77e1 100644 --- a/specification/2-modules.rst +++ b/specification/2-modules.rst @@ -1,5 +1,3 @@ Modules ======= -Modules intro here. - From b49cb57fe596eb942759f67a054906386854867a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 17:32:31 +0100 Subject: [PATCH 15/19] Move events sections to CS API --- specification/targets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/targets.yaml b/specification/targets.yaml index 96527dae..92ad9bb4 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -3,9 +3,9 @@ targets: files: # the sort order of files to cat - 0-intro.rst - { 1: 0-feature_profiles.rst } + - 1-client_server_api.rst - { 1: 0-events.rst } - { 1: 0-event_signing.rst } - - 1-client_server_api.rst - 2-modules.rst - { 1: "group:modules" } # reference a group of files - 3-application_service_api.rst From 5b134119bd53200f00da3b52557ff88b739f8ad5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 09:59:54 +0100 Subject: [PATCH 16/19] Add presence module; fix relative title bug If a relative title appeared after an HTTP API table, it would insert the wrong level because it thought that part of the table was a title. --- specification/1-client_server_api.rst | 6 -- specification/modules/presence.rst | 60 +++++++++++++++++++ specification/targets.yaml | 1 + .../matrix_templates/templates/http-api.tmpl | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 specification/modules/presence.rst diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index ad39dafc..93e3cb90 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1054,12 +1054,6 @@ medium address The textual address of the 3pid, eg. the email address -Presence --------- -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them - -{{presence_http_api}} Profiles -------- diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst new file mode 100644 index 00000000..6f644879 --- /dev/null +++ b/specification/modules/presence.rst @@ -0,0 +1,60 @@ +Presence Events +=============== + +{{presence_events}} + +Each user has the concept of presence information. This encodes the +"availability" of that user, suitable for display on other user's clients. +This is transmitted as an ``m.presence`` event and is one of the few events +which are sent *outside the context of a room*. The basic piece of presence +information is represented by the ``presence`` key, which is an enum of one +of the following: + + - ``online`` : The default state when the user is connected to an event + stream. + - ``unavailable`` : The user is not reachable at this time. + - ``offline`` : The user is not connected to an event stream. + - ``free_for_chat`` : The user is generally willing to receive messages + moreso than default. + - ``hidden`` : Behaves as offline, but allows the user to see the client + state anyway and generally interact with client features. (Not yet + implemented in synapse). + +In addition, the server maintains a timestamp of the last time it saw a +pro-active event from the user; either sending a message to a room, or +changing presence state from a lower to a higher level of availability +(thus: changing state from ``unavailable`` to ``online`` counts as a +proactive event, whereas in the other direction it will not). This timestamp +is presented via a key called ``last_active_ago``, which gives the relative +number of milliseconds since the message is generated/emitted that the user +was last seen active. + +Presence HTTP API +----------------- +.. TODO-spec + - Define how users receive presence invites, and how they accept/decline them + +{{presence_http_api}} + + +Events on Change of Profile Information +--------------------------------------- +Because the profile displayname and avatar information are likely to be used in +many places of a client's display, changes to these fields cause an automatic +propagation event to occur, informing likely-interested parties of the new +values. This change is conveyed using two separate mechanisms: + + - a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. + - a ``m.presence`` presence status update is sent, again containing the new values of the + ``displayname`` and ``avatar_url`` keys, in addition to the required + ``presence`` key containing the current presence state of the user. + +Both of these should be done automatically by the home server when a user +successfully changes their displayname or avatar URL fields. + +Additionally, when home servers emit room membership events for their own +users, they should include the displayname and avatar URL fields in these +events so that clients already have these details to hand, and do not have to +perform extra roundtrips to query it. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 92ad9bb4..454f23b9 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -17,6 +17,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst + - modules/presence.rst - modules/content_repo.rst - modules/end_to_end_encryption.rst - modules/history_visibility.rst diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index a03ffa7d..eb3f3e64 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -18,7 +18,7 @@ Request format: ================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* --------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} {{param.key}}{{param.type|indent(19-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(37)}} {% endfor -%} From 51153462971d3c6ffd4fae4672504e857532c5c2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 10:48:49 +0100 Subject: [PATCH 17/19] Add instant_messaging module; modify batesian section rules Previously, all `m.room.*` events were wodged into `{{room_events}}` which isn't great when you want to pull specific ones out. Batesian had a 1:1 mapping of `render_foo()` to a section `{{foo}}`, and having to constantly add functions for new types is a PITA. Batesian now supports returning a `dict` instead of a section `string` where the keys are the `{{foo}}` and the value is what will be inserted. Also add conflicting section key checks to avoid multiple definitions of the same `{{foo}}`. Define dicts for event schemata and swagger HTTP APIs. Using this new feature, split out the instant messaging stuff from the events section, and replace `{{room_events}}` with a list of specific events e.g. `{{m_room_member_event}}`. --- specification/0-events.rst | 82 ++++----------------- specification/modules/instant_messaging.rst | 27 +++++++ specification/modules/presence.rst | 11 ++- specification/targets.yaml | 1 + templating/batesian/sections.py | 38 ++++++++-- templating/matrix_templates/sections.py | 58 +++++++-------- 6 files changed, 111 insertions(+), 106 deletions(-) create mode 100644 specification/modules/instant_messaging.rst diff --git a/specification/0-events.rst b/specification/0-events.rst index ce36b040..a50d01fa 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -20,71 +20,19 @@ Room Events This specification outlines several standard event types, all of which are prefixed with ``m.`` -{{room_events}} - -m.room.message msgtypes -~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO-spec - How a client should handle unknown message types. - - -Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type -of message being sent. Each type has their own required and optional keys, as -outlined below. - -{{msgtype_events}} - -Presence Events -~~~~~~~~~~~~~~~ - -{{presence_events}} - -Each user has the concept of presence information. This encodes the -"availability" of that user, suitable for display on other user's clients. -This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. The basic piece of presence -information is represented by the ``presence`` key, which is an enum of one -of the following: - - - ``online`` : The default state when the user is connected to an event - stream. - - ``unavailable`` : The user is not reachable at this time. - - ``offline`` : The user is not connected to an event stream. - - ``free_for_chat`` : The user is generally willing to receive messages - moreso than default. - - ``hidden`` : Behaves as offline, but allows the user to see the client - state anyway and generally interact with client features. (Not yet - implemented in synapse). - -In addition, the server maintains a timestamp of the last time it saw a -pro-active event from the user; either sending a message to a room, or -changing presence state from a lower to a higher level of availability -(thus: changing state from ``unavailable`` to ``online`` counts as a -proactive event, whereas in the other direction it will not). This timestamp -is presented via a key called ``last_active_ago``, which gives the relative -number of milliseconds since the message is generated/emitted that the user -was last seen active. - - -Events on Change of Profile Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because the profile displayname and avatar information are likely to be used in -many places of a client's display, changes to these fields cause an automatic -propagation event to occur, informing likely-interested parties of the new -values. This change is conveyed using two separate mechanisms: - - - a ``m.room.member`` event is sent to every room the user is a member of, - to update the ``displayname`` and ``avatar_url``. - - a ``m.presence`` presence status update is sent, again containing the new values of the - ``displayname`` and ``avatar_url`` keys, in addition to the required - ``presence`` key containing the current presence state of the user. - -Both of these should be done automatically by the home server when a user -successfully changes their displayname or avatar URL fields. - -Additionally, when home servers emit room membership events for their own -users, they should include the displayname and avatar URL fields in these -events so that clients already have these details to hand, and do not have to -perform extra roundtrips to query it. +{{m_room_aliases_event}} + +{{m_room_canonical_aliases_event}} + +{{m_room_create_event}} + +{{m_room_history_visibility_event}} + +{{m_room_join_rules_event}} + +{{m_room_member_event}} + +{{m_room_power_levels_event}} + +{{m_room_redaction_event}} diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst new file mode 100644 index 00000000..7f582ca4 --- /dev/null +++ b/specification/modules/instant_messaging.rst @@ -0,0 +1,27 @@ +Instant Messaging +================= + +Events +------ + +{{m_room_message_event}} + +{{m_room_message_feedback_event}} + +{{m_room_name_event}} + +{{m_room_topic_event}} + +m.room.message msgtypes +----------------------- + +.. TODO-spec + How a client should handle unknown message types. + + +Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type +of message being sent. Each type has their own required and optional keys, as +outlined below. + +{{msgtype_events}} + diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 6f644879..ddd2adff 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,7 +1,5 @@ -Presence Events -=============== - -{{presence_events}} +Presence +======== Each user has the concept of presence information. This encodes the "availability" of that user, suitable for display on other user's clients. @@ -29,6 +27,11 @@ is presented via a key called ``last_active_ago``, which gives the relative number of milliseconds since the message is generated/emitted that the user was last seen active. +Events +------ + +{{presence_events}} + Presence HTTP API ----------------- .. TODO-spec diff --git a/specification/targets.yaml b/specification/targets.yaml index 454f23b9..e33bc785 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,6 +14,7 @@ targets: - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: + - modules/instant_messaging.rst - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 11c34fb3..31849389 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -27,12 +27,38 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if not isinstance(section, basestring): + if isinstance(section, basestring): + if section_key in section_dict: + raise Exception( + ("%s : Section %s already exists. It must have been " + + "generated dynamically. Check which render_ methods " + + "return a dict.") % + (func_name, section_key) + ) + section_dict[section_key] = section + self.log( + " Generated. Snippet => %s" % section[:60].replace("\n","") + ) + elif isinstance(section, dict): + self.log(" Generated multiple sections:") + for (k, v) in section.iteritems(): + if not isinstance(k, basestring) or not isinstance(v, basestring): + raise Exception( + ("Method %s returned multiple sections as a dict but " + + "expected the dict elements to be strings but they aren't.") % + (func_name, ) + ) + if k in section_dict: + raise Exception( + "%s tried to produce section %s which already exists." % + (func_name, k) + ) + section_dict[k] = v + self.log( + " %s => %s" % (k, v[:60].replace("\n","")) + ) + else: raise Exception( - "Section function '%s' didn't return a string!" % func_name + "Section function '%s' didn't return a string/dict!" % func_name ) - section_dict[section_key] = section - self.log( - " Generated. Snippet => %s" % section[:60].replace("\n","") - ) return section_dict \ No newline at end of file diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index cacaa06a..e75a75af 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -76,37 +76,36 @@ class MatrixSections(Sections): )) return "\n\n".join(sections) - def render_profile_http_api(self): - return self._render_http_api_group( - "profile", - sortFnOrPathList=["displayname", "avatar_url"] - ) - - def render_sync_http_api(self): - return self._render_http_api_group( - "sync" - ) - - def render_presence_http_api(self): - return self._render_http_api_group( - "presence", - sortFnOrPathList=["status"] - ) - - def render_membership_http_api(self): - return self._render_http_api_group( - "membership" - ) - def render_login_http_api(self): - return self._render_http_api_group( - "login" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_http_apis(self): + # map all swagger_apis to the form $GROUP_http_api + swagger_groups = self.units.get("swagger_apis").keys() + renders = {} + for group in swagger_groups: + sortFnOrPathList = None + if group == "presence": + sortFnOrPathList = ["status"] + elif group == "profile": + sortFnOrPathList=["displayname", "avatar_url"] + renders[group + "_http_api"] = self._render_http_api_group( + group, sortFnOrPathList + ) + return renders - def render_rooms_http_api(self): - return self._render_http_api_group( - "rooms" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_events(self): + # map all event schemata to the form $EVENTTYPE_event with s/./_/g + # e.g. m_room_topic_event + schemas = self.units.get("event_schemas") + renders = {} + for event_type in schemas: + renders[event_type.replace(".", "_") + "_event"] = self._render_events( + lambda x: x == event_type, sorted + ) + return renders def render_room_events(self): def filterFn(eventType): @@ -180,3 +179,4 @@ class MatrixSections(Sections): def render_common_state_event_fields(self): return self._render_ce_type("state_event") + From 29bae1579017c05d8ff5f5cf415459cebf0bac12 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 11:30:07 +0100 Subject: [PATCH 18/19] Fix typo --- specification/0-events.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/0-events.rst b/specification/0-events.rst index a50d01fa..16948462 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -22,7 +22,7 @@ prefixed with ``m.`` {{m_room_aliases_event}} -{{m_room_canonical_aliases_event}} +{{m_room_canonical_alias_event}} {{m_room_create_event}} From 1da64db302f07019e410e607775c9efa0702d100 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 14:29:37 +0100 Subject: [PATCH 19/19] Use relative depths for groups instead of absolute ones This means the group can be agnostic to how deeply nested it is, improving reusability of groups. --- scripts/gendoc.py | 32 ++++++++++++++++++++++++-------- specification/targets.yaml | 3 ++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 26ee5a48..364b8e65 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from docutils.core import publish_file +import copy import fileinput import glob import os @@ -292,21 +293,32 @@ def get_build_target(targets_listing, target_name): "Found target but 'files' key is not a list." ) - def get_group(group_id): + def get_group(group_id, depth): group_name = group_id[len("group:"):] group = all_targets.get("groups", {}).get(group_name) if not group: raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." + "Tried to find group '%s' but it doesn't exist." % group_name ) + if not isinstance(group, list): + raise Exception( + "Expected group '%s' to be a list but it isn't." % group_name + ) + # deep copy so changes to depths don't contaminate multiple uses of this group + group = copy.deepcopy(group) + # swap relative depths for absolute ones + for i, entry in enumerate(group): + if isinstance(entry, dict): + group[i] = { + (rel_depth + depth): v for (rel_depth, v) in entry.items() + } return group resolved_files = [] for file_entry in target["files"]: # file_entry is a group id if isinstance(file_entry, basestring) and file_entry.startswith("group:"): - group = get_group(file_entry) + group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than # insert the entire list as a single element (which is what append does) @@ -317,12 +329,16 @@ def get_build_target(targets_listing, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (k, v) in file_entry.iteritems(): - if isinstance(v, basestring) and v.startswith("group:"): - resolved_entry[k] = get_group(v) + for (depth, entry) in file_entry.iteritems(): + if not isinstance(entry, basestring): + raise Exception( + "Double-nested depths are not supported. Entry: %s" % (file_entry,) + ) + if entry.startswith("group:"): + resolved_entry[depth] = get_group(entry, depth) else: # map across without editing (e.g. normal file path) - resolved_entry[k] = v + resolved_entry[depth] = entry resolved_files.append(resolved_entry) continue # file_entry is just a plain ol' file path diff --git a/specification/targets.yaml b/specification/targets.yaml index e33bc785..62585c69 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -23,7 +23,8 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/end_to_end_encryption.rst - modules/history_visibility.rst - modules/push_overview.rst - - { 2: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } + # relative depth + - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } title_styles: ["=", "-", "~", "+", "^"]