From ad26b7f8cb89b2a32e009bdd1d061ea6b7887b9d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 10:03:58 +0100 Subject: [PATCH 1/7] Fix a subtle rendering bug when catting spec sections Throw in gendoc.py if a spec section doesn't end with \n\n There needs to be TWO new lines at the end of each spec section else the title of the next section merges into the last paragraph of the earlier section. This happens without rst2html producing a warning, and results in the section heading of a file disappearing(!) --- scripts/gendoc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed172726..e871055e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -18,7 +18,14 @@ def glob_spec_to(out_file_name): with open(out_file_name, "wb") as outfile: for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: - outfile.write(infile.read()) + section = infile.read() + # we need TWO new lines else the next file's title gets merged + # the last paragraph *WITHOUT RST PRODUCING A WARNING* + if not section[-2:] == '\n\n': + raise Exception( + "The file " + f + " does not end with 2 new lines." + ) + outfile.write(section) def rst2html(i, o): From ef473b4161d029b35ef193309c9b4ad23f43e4f1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:05:48 +0100 Subject: [PATCH 2/7] Start enforcing title styles This depends on the number of the file. --- scripts/gendoc.py | 55 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index e871055e..6bf7c1a6 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -4,6 +4,7 @@ from docutils.core import publish_file import fileinput import glob import os +import re import shutil import subprocess import sys @@ -14,17 +15,55 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css"] } -def glob_spec_to(out_file_name): +title_style_matchers = { + "=": re.compile("^=+$"), + "-": re.compile("^-+$") +} +TOP_LEVEL = "=" +SECOND_LEVEL = "-" + + +def check_valid_section(filename, section): + # we need TWO new lines else the next file's title gets merged + # the last paragraph *WITHOUT RST PRODUCING A WARNING* + if not section[-2:] == '\n\n': + raise Exception( + "The file " + filename + " does not end with 2 new lines." + ) + + # Enforce some rules to reduce the risk of having mismatched title + # styles. + title_line = section.split("\n")[1] + if title_line != (len(title_line) * title_line[0]): + raise Exception( + "The file " + filename + " doesn't have a title style line on line 2" + ) + + # anything marked as x0_ is the start of a new top-level section + if re.match("^[0-9]+0_", filename): + if not title_style_matchers[TOP_LEVEL].match(title_line): + raise Exception( + "The file " + filename + " is a top-level section because it matches " + + "the filename format x0_something.rst but has the wrong title " + + "style: expected '" + TOP_LEVEL + "' but got '" + + title_line[0] + "'" + ) + # anything marked as xx_ is the start of a sub-section + elif re.match("^[0-9]+_", filename): + if not title_style_matchers[SECOND_LEVEL].match(title_line): + raise Exception( + "The file " + filename + " is a 2nd-level section because it matches " + + "the filename format xx_something.rst but has the wrong title " + + "style: expected '" + SECOND_LEVEL + "' but got '" + + title_line[0] + "'" + ) + +def cat_spec_sections_to(out_file_name): with open(out_file_name, "wb") as outfile: for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: section = infile.read() - # we need TWO new lines else the next file's title gets merged - # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == '\n\n': - raise Exception( - "The file " + f + " does not end with 2 new lines." - ) + check_valid_section(f.split("/")[-1], section) outfile.write(section) @@ -74,7 +113,7 @@ def cleanup_env(): def main(): prepare_env() - glob_spec_to("tmp/full_spec.rst") + cat_spec_sections_to("tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") From 06e46c1899755d9ae46b66a8b316b4c41b664328 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:20:34 +0100 Subject: [PATCH 3/7] Fix on a better numbering format --- scripts/gendoc.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 6bf7c1a6..1348679a 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -21,9 +21,16 @@ title_style_matchers = { } TOP_LEVEL = "=" SECOND_LEVEL = "-" +FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}_.*\.rst$") def check_valid_section(filename, section): + if not re.match(FILE_FORMAT_MATCHER, filename): + raise Exception( + "The filename of " + filename +" does not match the expected format " + + "of '##_##_words-go-here.rst'" + ) + # we need TWO new lines else the next file's title gets merged # the last paragraph *WITHOUT RST PRODUCING A WARNING* if not section[-2:] == '\n\n': @@ -39,21 +46,21 @@ def check_valid_section(filename, section): "The file " + filename + " doesn't have a title style line on line 2" ) - # anything marked as x0_ is the start of a new top-level section - if re.match("^[0-9]+0_", filename): + # anything marked as xx_00_ is the start of a new top-level section + if re.match("^[0-9]+_00_", filename): if not title_style_matchers[TOP_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a top-level section because it matches " + - "the filename format x0_something.rst but has the wrong title " + + "the filename format ##_00_something.rst but has the wrong title " + "style: expected '" + TOP_LEVEL + "' but got '" + title_line[0] + "'" ) - # anything marked as xx_ is the start of a sub-section - elif re.match("^[0-9]+_", filename): + # anything marked as xx_x0_ is the start of a sub-section + elif re.match("^[0-9]+_0[0-9]{1}_", filename): if not title_style_matchers[SECOND_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format xx_something.rst but has the wrong title " + + "the filename format ##_#0_something.rst but has the wrong title " + "style: expected '" + SECOND_LEVEL + "' but got '" + title_line[0] + "'" ) From 1f6b12b3e828696a588fed9a0028e59991d28b94 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:43:53 +0100 Subject: [PATCH 4/7] Modify and enforce the file format/structure used Convert the file format to be of the form ##_##_something.rst where the first ## is the top-level section number and the second ## is the second-level section number, e.g. 07_01_push_cs_api.rst means Section 7.1 - This is now enforced in gendoc.py along with the title line style that should be used (= for top-level, - for 2nd level) which will give helpful suggestions if you trip up. This feels much more intuitive now looking in /specification --- scripts/gendoc.py | 12 +++++++----- specification/{00_basis.rst => 00_00_intro.rst} | 0 ..._server_api.rst => 01_00_client_server_api.rst} | 0 ...ications.rst => 01_01_typing_notifications.rst} | 8 ++++---- .../{46_receipts.rst => 01_02_receipts.rst} | 8 ++++---- ...visibility.rst => 01_03_history_visibility.rst} | 2 +- specification/{20_events.rst => 02_00_events.rst} | 0 .../{22_voip_events.rst => 02_01_voip_events.rst} | 0 ...1_event_signing.rst => 02_02_event_signing.rst} | 14 +++++++------- ...e_api.rst => 03_00_application_service_api.rst} | 0 ..._server_api.rst => 04_00_server_server_api.rst} | 0 ...repository.rst => 05_00_content_repository.rst} | 0 ...ryption.rst => 06_00_end_to_end_encryption.rst} | 0 ...2_push_overview.rst => 07_00_push_overview.rst} | 0 .../{43_push_cs_api.rst => 07_01_push_cs_api.rst} | 0 ..._push_gw_api.rst => 07_02_push_push_gw_api.rst} | 0 ...n_core_apis.rst => 08_00_address_book_repo.rst} | 0 ...tity_servers.rst => 09_00_identity_servers.rst} | 0 .../{50_appendices.rst => 10_00_appendices.rst} | 0 19 files changed, 23 insertions(+), 21 deletions(-) rename specification/{00_basis.rst => 00_00_intro.rst} (100%) rename specification/{10_client_server_api.rst => 01_00_client_server_api.rst} (100%) rename specification/{45_typing_notifications.rst => 01_01_typing_notifications.rst} (96%) rename specification/{46_receipts.rst => 01_02_receipts.rst} (94%) rename specification/{47_history_visibility.rst => 01_03_history_visibility.rst} (97%) rename specification/{20_events.rst => 02_00_events.rst} (100%) rename specification/{22_voip_events.rst => 02_01_voip_events.rst} (100%) rename specification/{31_event_signing.rst => 02_02_event_signing.rst} (99%) rename specification/{25_application_service_api.rst => 03_00_application_service_api.rst} (100%) rename specification/{30_server_server_api.rst => 04_00_server_server_api.rst} (100%) rename specification/{40_content_repository.rst => 05_00_content_repository.rst} (100%) rename specification/{41_end_to_end_encryption.rst => 06_00_end_to_end_encryption.rst} (100%) rename specification/{42_push_overview.rst => 07_00_push_overview.rst} (100%) rename specification/{43_push_cs_api.rst => 07_01_push_cs_api.rst} (100%) rename specification/{44_push_push_gw_api.rst => 07_02_push_push_gw_api.rst} (100%) rename specification/{49_other_non_core_apis.rst => 08_00_address_book_repo.rst} (100%) rename specification/{60_identity_servers.rst => 09_00_identity_servers.rst} (100%) rename specification/{50_appendices.rst => 10_00_appendices.rst} (100%) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 1348679a..c97995e8 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -21,7 +21,7 @@ title_style_matchers = { } TOP_LEVEL = "=" SECOND_LEVEL = "-" -FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}_.*\.rst$") +FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") def check_valid_section(filename, section): @@ -55,14 +55,16 @@ def check_valid_section(filename, section): "style: expected '" + TOP_LEVEL + "' but got '" + title_line[0] + "'" ) - # anything marked as xx_x0_ is the start of a sub-section - elif re.match("^[0-9]+_0[0-9]{1}_", filename): + # anything marked as xx_xx_ is the start of a sub-section + elif re.match("^[0-9]+_[0-9]{2}_", filename): if not title_style_matchers[SECOND_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format ##_#0_something.rst but has the wrong title " + + "the filename format ##_##_something.rst but has the wrong title " + "style: expected '" + SECOND_LEVEL + "' but got '" + - title_line[0] + "'" + title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " + + "then use the form '##_##b_something.rst' which will not apply this " + + "check." ) def cat_spec_sections_to(out_file_name): diff --git a/specification/00_basis.rst b/specification/00_00_intro.rst similarity index 100% rename from specification/00_basis.rst rename to specification/00_00_intro.rst diff --git a/specification/10_client_server_api.rst b/specification/01_00_client_server_api.rst similarity index 100% rename from specification/10_client_server_api.rst rename to specification/01_00_client_server_api.rst diff --git a/specification/45_typing_notifications.rst b/specification/01_01_typing_notifications.rst similarity index 96% rename from specification/45_typing_notifications.rst rename to specification/01_01_typing_notifications.rst index ff00165d..25b714ab 100644 --- a/specification/45_typing_notifications.rst +++ b/specification/01_01_typing_notifications.rst @@ -1,8 +1,8 @@ Typing Notifications -==================== +-------------------- Client APIs ------------ +~~~~~~~~~~~ To set "I am typing for the next N msec":: @@ -21,7 +21,7 @@ To set "I am no longer typing":: Content: { "typing": false } Client Events -------------- +~~~~~~~~~~~~~ All room members will receive an event on the event stream:: @@ -39,7 +39,7 @@ users who are not currently typing, as that list gets big quickly. The client should mark as not typing, any user ID who is not in that list. Server APIs ------------ +~~~~~~~~~~~ Servers will emit EDUs in the following form:: diff --git a/specification/46_receipts.rst b/specification/01_02_receipts.rst similarity index 94% rename from specification/46_receipts.rst rename to specification/01_02_receipts.rst index a0a90c16..e2f83eea 100644 --- a/specification/46_receipts.rst +++ b/specification/01_02_receipts.rst @@ -1,5 +1,5 @@ Receipts -======== +-------- Receipts are used to publish which events in a room the user or their devices have interacted with. For example, which events the user has read. For @@ -7,7 +7,7 @@ efficiency this is done as "up to" markers, i.e. marking a particular event as, say, ``read`` indicates the user has read all events *up to* that event. Client-Server API ------------------ +~~~~~~~~~~~~~~~~~ Clients will receive receipts in the following format:: @@ -58,7 +58,7 @@ other users. The server will automatically set the ``ts`` field. Server-Server API ------------------ +~~~~~~~~~~~~~~~~~ Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -73,5 +73,5 @@ format of the EDUs are:: ... } -These are always sent as deltas to previously sent reciepts. +These are always sent as deltas to previously sent receipts. diff --git a/specification/47_history_visibility.rst b/specification/01_03_history_visibility.rst similarity index 97% rename from specification/47_history_visibility.rst rename to specification/01_03_history_visibility.rst index b1630f1e..01c2e419 100644 --- a/specification/47_history_visibility.rst +++ b/specification/01_03_history_visibility.rst @@ -1,5 +1,5 @@ Room History Visibility -======================= +----------------------- Whether a member of a room can see the events that happened in a room from before they joined the room is controlled by the ``history_visibility`` key diff --git a/specification/20_events.rst b/specification/02_00_events.rst similarity index 100% rename from specification/20_events.rst rename to specification/02_00_events.rst diff --git a/specification/22_voip_events.rst b/specification/02_01_voip_events.rst similarity index 100% rename from specification/22_voip_events.rst rename to specification/02_01_voip_events.rst diff --git a/specification/31_event_signing.rst b/specification/02_02_event_signing.rst similarity index 99% rename from specification/31_event_signing.rst rename to specification/02_02_event_signing.rst index f3a22d1d..bde58f0c 100644 --- a/specification/31_event_signing.rst +++ b/specification/02_02_event_signing.rst @@ -1,8 +1,8 @@ Signing Events -============== +-------------- Canonical JSON --------------- +~~~~~~~~~~~~~~ Matrix events are represented using JSON objects. If we want to sign JSON events we need to encode the JSON as a binary string. Unfortunately the same @@ -38,7 +38,7 @@ using this representation. ).encode("UTF-8") Grammar -~~~~~~~ ++++++++ Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing insignificant whitespace, fractions, exponents and redundant character escapes @@ -69,14 +69,14 @@ insignificant whitespace, fractions, exponents and redundant character escapes / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X Signing JSON ------------- +~~~~~~~~~~~~ We can now sign a JSON object by encoding it as a sequence of bytes, computing the signature for that sequence and then adding the signature to the original JSON object. Signing Details -~~~~~~~~~~~~~~~ ++++++++++++++++ JSON is signed by encoding the JSON object without ``signatures`` or keys grouped as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the @@ -133,7 +133,7 @@ and additional signatures. return json_object Checking for a Signature -~~~~~~~~~~~~~~~~~~~~~~~~ +++++++++++++++++++++++++ To check if an entity has signed a JSON object a server does the following @@ -151,7 +151,7 @@ To check if an entity has signed a JSON object a server does the following the check fails. Otherwise the check succeeds. Signing Events --------------- +~~~~~~~~~~~~~~ Signing events is a more complicated process since servers can choose to redact non-essential parts of an event. Before signing the event it is encoded as diff --git a/specification/25_application_service_api.rst b/specification/03_00_application_service_api.rst similarity index 100% rename from specification/25_application_service_api.rst rename to specification/03_00_application_service_api.rst diff --git a/specification/30_server_server_api.rst b/specification/04_00_server_server_api.rst similarity index 100% rename from specification/30_server_server_api.rst rename to specification/04_00_server_server_api.rst diff --git a/specification/40_content_repository.rst b/specification/05_00_content_repository.rst similarity index 100% rename from specification/40_content_repository.rst rename to specification/05_00_content_repository.rst diff --git a/specification/41_end_to_end_encryption.rst b/specification/06_00_end_to_end_encryption.rst similarity index 100% rename from specification/41_end_to_end_encryption.rst rename to specification/06_00_end_to_end_encryption.rst diff --git a/specification/42_push_overview.rst b/specification/07_00_push_overview.rst similarity index 100% rename from specification/42_push_overview.rst rename to specification/07_00_push_overview.rst diff --git a/specification/43_push_cs_api.rst b/specification/07_01_push_cs_api.rst similarity index 100% rename from specification/43_push_cs_api.rst rename to specification/07_01_push_cs_api.rst diff --git a/specification/44_push_push_gw_api.rst b/specification/07_02_push_push_gw_api.rst similarity index 100% rename from specification/44_push_push_gw_api.rst rename to specification/07_02_push_push_gw_api.rst diff --git a/specification/49_other_non_core_apis.rst b/specification/08_00_address_book_repo.rst similarity index 100% rename from specification/49_other_non_core_apis.rst rename to specification/08_00_address_book_repo.rst diff --git a/specification/60_identity_servers.rst b/specification/09_00_identity_servers.rst similarity index 100% rename from specification/60_identity_servers.rst rename to specification/09_00_identity_servers.rst diff --git a/specification/50_appendices.rst b/specification/10_00_appendices.rst similarity index 100% rename from specification/50_appendices.rst rename to specification/10_00_appendices.rst From 427e4c8b1b23ab875b24acad5605848abf0d8ee9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 15:46:37 +0100 Subject: [PATCH 5/7] Log port being listened on --- scripts/continuserv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index e7757c06..dc7705a8 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -56,7 +56,7 @@ func main() { go doPopulate(ch, dir) go watchFS(ch, w) - + fmt.Printf("Listening on port %d\n", *port) http.HandleFunc("/", serve) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From ba4998a62b28fcd13c9085ae58996d1c857b27c5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 15:49:01 +0100 Subject: [PATCH 6/7] Apparently tabs are A Thing --- scripts/continuserv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index dc7705a8..658ae0fb 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -56,7 +56,7 @@ func main() { go doPopulate(ch, dir) go watchFS(ch, w) - fmt.Printf("Listening on port %d\n", *port) + fmt.Printf("Listening on port %d\n", *port) http.HandleFunc("/", serve) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From cf8abdafd43ad67802b20896d53ab977d6d4beee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 15:42:49 +0100 Subject: [PATCH 7/7] Fix PR comments --- scripts/gendoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index c97995e8..e2f51419 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -27,13 +27,13 @@ FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") def check_valid_section(filename, section): if not re.match(FILE_FORMAT_MATCHER, filename): raise Exception( - "The filename of " + filename +" does not match the expected format " + + "The filename of " + filename + " does not match the expected format " + "of '##_##_words-go-here.rst'" ) # we need TWO new lines else the next file's title gets merged # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == '\n\n': + if not section[-2:] == "\n\n": raise Exception( "The file " + filename + " does not end with 2 new lines." ) @@ -72,7 +72,7 @@ def cat_spec_sections_to(out_file_name): for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: section = infile.read() - check_valid_section(f.split("/")[-1], section) + check_valid_section(os.path.basename(f), section) outfile.write(section)