From 81a60a25ccd78b413d5fc0a25e6267dcf05db23d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:46:10 +0000 Subject: [PATCH 01/24] Update 3pid spec based on new implementation --- event-schemas/examples/v1/m.room.member | 15 ---------- event-schemas/schema/v1/m.room.member | 20 ++----------- specification/modules/third_party_invites.rst | 29 ++++++++++--------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a..d7605dcd 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -4,21 +4,6 @@ "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70..1d09a0dd 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -26,18 +26,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -57,13 +45,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31..140bab86 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -15,13 +15,15 @@ The homeserver asks the identity server whether a Matrix user ID is known for that identifier. If it is, an invite is simply issued for that user. If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets +the invitation, and to notify the invitee's homeserver of this pending invitation if it gets a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property whichi contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -39,9 +41,10 @@ Server behaviour All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -91,16 +94,16 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. + When the third party user validates their identity, their homeserver, H3, + is notified, and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. + H3 validates the signature given to it by the identity server. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 + Having validated these things, H1 writes the invite event to the room, and H3 begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking From b92a0f2b4de47093b229e9f0521aec2bb008435c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:49:22 +0000 Subject: [PATCH 02/24] Remove extra trailing comma --- event-schemas/examples/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index d7605dcd..e2ca5668 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,7 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", + "displayname": "Alice Margatroid" }, "invite_room_state": [ { From 176f919fc89f5d01669aea17003aef8da903ce4e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:00:53 +0000 Subject: [PATCH 03/24] Show multiple examples where present --- .../v1/m.room.member#third_party_invite | 41 +++++++++++++++++++ templating/matrix_templates/sections.py | 2 +- .../matrix_templates/templates/events.tmpl | 4 +- templating/matrix_templates/units.py | 11 +++-- 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite new file mode 100644 index 00000000..4ef571ec --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -0,0 +1,41 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid", + "third_party_invite": { + "signed": { + "mxid": "@alice:localhost", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + }, + "token": "abc123" + } + } + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index e75a75af..81b2bb3c 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -35,7 +35,7 @@ class MatrixSections(Sections): if not filterFn(event_name): continue sections.append(template.render( - example=examples[event_name], + examples=examples[event_name], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440..fbefe17f 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -21,8 +21,10 @@ ======================= ================= =========================================== {% endfor %} -Example: +Example{% if examples | length > 1 %}s{% endif %}: +{% for example in examples %} .. code:: json {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c4342141..d33ba81d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -495,9 +495,14 @@ class MatrixUnits(Units): if not filename.startswith("m."): continue with open(os.path.join(path, filename), "r") as f: - examples[filename] = json.loads(f.read()) - if filename == "m.room.message#m.text": - examples["m.room.message"] = examples[filename] + event_name = filename.split("#")[0] + example = json.loads(f.read()) + + examples[filename] = examples.get(event_name, []) + examples[filename].append(example) + if filename != event_name: + examples[event_name] = examples.get(event_name, []) + examples[event_name].append(example) return examples def load_event_schemas(self): From 810922bb381a0f7162941605ec631f00927071ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:06:44 +0000 Subject: [PATCH 04/24] Fix schema validator for multiple examples --- event-schemas/check_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index e54d3a1c..b10b2d0e 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir): continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) + if schemapath.find("#") >= 0: + schemapath = schemapath[:schemapath.find("#")] try: check_example_file(examplepath, schemapath) except Exception as e: From cddfc110edc9018e44ce1a3690f3f6431a0ea9b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 29 Oct 2015 12:48:04 +0000 Subject: [PATCH 05/24] Review comments --- specification/modules/third_party_invites.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 140bab86..6d3f8f6a 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -22,7 +22,7 @@ and public key to the inviting homeserver. When the invitee's homeserver receives the notification of the binding, it should insert an ``m.room.member`` event into the room's graph for that user, with ``content.membership`` = ``invite``, as well as a -``content.third_party_invite`` property whichi contains proof that the invitee +``content.third_party_invite`` property which contains proof that the invitee does indeed own that third party identifier. Events @@ -94,8 +94,8 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, their homeserver, H3, - is notified, and attempts to issue an ``m.room.member`` event to participate + When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate in the room. H3 validates the signature given to it by the identity server. From 9f4d81308d3eca3870f54c196335d2bb5176bd44 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 30 Oct 2015 16:21:50 +0000 Subject: [PATCH 06/24] Pull out separate invite_room_state example --- .../v1/m.room.member#invite_room_state | 30 +++++++++++++++++++ .../v1/m.room.member#third_party_invite | 16 ---------- 2 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/v1/m.room.member#invite_room_state new file mode 100644 index 00000000..e2ca5668 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#invite_room_state @@ -0,0 +1,30 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid" + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite index 4ef571ec..2457302a 100644 --- a/event-schemas/examples/v1/m.room.member#third_party_invite +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -16,22 +16,6 @@ } } }, - "invite_room_state": [ - { - "type": "m.room.name", - "state_key": "", - "content": { - "name": "Forest of Magic" - } - }, - { - "type": "m.room.join_rules", - "state_key": "", - "content": { - "join_rules": "invite" - } - } - ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", From c00abe9f2faa80f6cfcf7fba028de870eca676ac Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:26:51 +0000 Subject: [PATCH 07/24] Fix msgtype display --- templating/matrix_templates/sections.py | 2 +- templating/matrix_templates/units.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 81b2bb3c..78aabca7 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -136,7 +136,7 @@ class MatrixSections(Sections): if not event_name.startswith("m.room.message#m."): continue sections.append(template.render( - example=examples[event_name], + example=examples[event_name][0], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 5ae117ca..94435c52 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -562,7 +562,7 @@ class MatrixUnits(Units): event_name = filename.split("#")[0] example = json.loads(f.read()) - examples[filename] = examples.get(event_name, []) + examples[filename] = examples.get(filename, []) examples[filename].append(example) if filename != event_name: examples[event_name] = examples.get(event_name, []) From 9ad64b02d183e72b6548d7a36f9b96949f624e70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 15:41:15 -0500 Subject: [PATCH 08/24] speculator: guard against concurrent git commands --- scripts/speculator/main.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 97e67c8c..4472164c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -136,17 +137,26 @@ func writeError(w http.ResponseWriter, code int, err error) { } type server struct { + mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL matrixDocCloneURL string } +func (s *server) updateBase() error { + s.mu.Lock() + defer s.mu.Unlock() + return gitFetch(s.matrixDocCloneURL) +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = s.updateBase() if err != nil { return } + s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) + s.mu.Unlock() if err != nil { return } @@ -164,7 +174,10 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - if err := cmd.Run(); err != nil { + s.mu.Lock() + err := cmd.Run() + s.mu.Unlock() + if err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil @@ -174,7 +187,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := gitFetch(s.matrixDocCloneURL); err != nil { + if err := s.updateBase(); err != nil { writeError(w, 500, err) return } @@ -383,7 +396,7 @@ func main() { if err != nil { log.Fatal(err) } - s := server{masterCloneDir} + s := server{matrixDocCloneURL: masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) From 757b0bcd12269b0a9905efee7520e61abc81ec7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:08:37 -0500 Subject: [PATCH 09/24] Try to build continuserv and speculator --- jenkins.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins.sh b/jenkins.sh index 0b217e58..b2aa8489 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -7,3 +7,8 @@ set -ex (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) + +if which go >/dev/null 2>/dev/null; then + (cd scripts/continuserv && go build) + (cd scripts/speculator && go build) +fi From dd53847211c57553180efca9d843c1c8741d6de3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:11:19 -0500 Subject: [PATCH 10/24] Include command stderr in error text --- scripts/speculator/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4472164c..00a59eea 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -70,13 +70,15 @@ const ( func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, directory) + if err := os.MkdirAll(directory, 0755); err != nil { + return "", fmt.Errorf("error making directory %s: %v", directory, err) + } + args := []string{"clone", url, directory} if shared { - cmd.Args = append(cmd.Args, "--shared") + args = append(args, "--shared") } - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("error cloning repo: %v", err) + if err := runGitCommand(directory, args); err != nil { + return "", err } return directory, nil } @@ -92,8 +94,10 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path + var b bytes.Buffer + cmd.Stderr = &b if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String()) } return nil } From 8872e17f9355a47f9b7a2822cfd1303d8e56e2bc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:14:22 -0500 Subject: [PATCH 11/24] Fall back to last known HEAD sha if fetch fails --- scripts/speculator/main.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 00a59eea..380cd2bc 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -87,10 +87,6 @@ func gitCheckout(path, sha string) error { return runGitCommand(path, []string{"checkout", sha}) } -func gitFetch(path string) error { - return runGitCommand(path, []string{"fetch"}) -} - func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path @@ -148,7 +144,7 @@ type server struct { func (s *server) updateBase() error { s.mu.Lock() defer s.mu.Unlock() - return gitFetch(s.matrixDocCloneURL) + return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } // generateAt generates spec from repo at sha. @@ -191,16 +187,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := s.updateBase(); err != nil { + if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return + } else { + sha = headSha } - originHead, err := s.getSHAOf("origin/master") - if err != nil { - writeError(w, 500, err) - return - } - sha = originHead } else { pr, err := lookupPullRequest(*req.URL, "/spec") if err != nil { @@ -237,6 +229,25 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { specCache.Add(sha, b) } +// lookupHeadSHA looks up what origin/master's HEAD SHA is. +// It attempts to `git fetch` before doing so. +// If this fails, it may still return a stale sha, but will also return an error. +func (s *server) lookupHeadSHA() (sha string, retErr error) { + retErr = s.updateBase() + if retErr != nil { + log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr) + } + originHead, err := s.getSHAOf("origin/master") + if err != nil { + retErr = err + } + sha = originHead + if retErr != nil && originHead != "" { + log.Printf("Successfully fell back to possibly stale sha: %s", sha) + } + return +} + func checkAuth(pr *PullRequest) error { if !pr.User.IsTrusted() { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login) From 6f1d00097be403998e5490845835ecb23d2c741c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:15:13 -0500 Subject: [PATCH 12/24] Only bother trying to fetch if we need to --- scripts/speculator/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 380cd2bc..4972210e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -147,12 +147,20 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } +func (s *server) knowsAbout(sha string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = s.updateBase() - if err != nil { - return + if !s.knowsAbout(sha) { + err = s.updateBase() + if err != nil { + return + } } s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) From 858674477111bfc80adb299ad73715a98099c296 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:41:58 -0500 Subject: [PATCH 13/24] Add anchors to spec This is currently done by a script on the prod serving machine. We might as well keep the matrix.org spec and dev spec as similar as possible. --- scripts/gendoc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed36a5a7..28a11528 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -238,6 +238,18 @@ def rst2html(i, o): ) +def addAnchors(path): + with open(path, "r") as f: + lines = f.readlines() + + replacement = replacement = r'

\n\1' + with open(path, "w") as f: + for line in lines: + line = re.sub(r'()', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write(line + "\n") + + def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: @@ -387,6 +399,7 @@ def main(target_name, keep_intermediates): shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") + addAnchors("gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() From e0410330484257e6d0c8da481d8d7e31a2b05053 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:17:50 -0500 Subject: [PATCH 14/24] Rename file --- scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} (100%) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/add-matrix-org-stylings.sh similarity index 100% rename from scripts/matrix-org-gendoc.sh rename to scripts/add-matrix-org-stylings.sh From ca3a9e3562492964e314d7f9173d320281fbddfd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:18:33 -0500 Subject: [PATCH 15/24] exec gendoc outside of script --- scripts/add-matrix-org-stylings.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 5716786d..b02cb306 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -35,8 +35,6 @@ if [ ! -f $FOOTER ]; then exit 1 fi -python gendoc.py - perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header #' gen/specification.html gen/howtos.html From da93317a7874096c2c13974542e2f2272e5d2a60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:13:40 -0500 Subject: [PATCH 16/24] Take dir not files as args --- scripts/add-matrix-org-stylings.sh | 43 +++++++++--------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index b02cb306..83888346 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -1,39 +1,20 @@ -#! /bin/bash +#!/bin/bash -eu -if [ -z "$1" ]; then - echo "Expected /includes/head.html file as 1st arg." - exit 1 -fi - -if [ -z "$2" ]; then - echo "Expected /includes/nav.html file as 2nd arg." - exit 1 +if [[ $# != 1 || ! -d $1 ]]; then + echo >&2 "Usage: $0 include_dir" + exit 1 fi -if [ -z "$3" ]; then - echo "Expected /includes/footer.html file as 3rd arg." - exit 1 -fi - - -HEADER=$1 -NAV_BAR=$2 -FOOTER=$3 +HEADER="$1/head.html" +NAV_BAR="$1/nav.html" +FOOTER="$1/footer.html" -if [ ! -f $HEADER ]; then - echo $HEADER " does not exist" +for f in "${HEADER}"/{head,nav,footer}.html; do + if [[ ! -e "${f}" ]]; then + echo >&2 "Need ${f} to exist" exit 1 -fi - -if [ ! -f $NAV_BAR ]; then - echo $NAV_BAR " does not exist" - exit 1 -fi - -if [ ! -f $FOOTER ]; then - echo $FOOTER " does not exist" - exit 1 -fi + fi +done perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header From 4ac85997f52303b5fa75241f9bc50cdc227af3fa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:16:02 -0500 Subject: [PATCH 17/24] Fix check --- scripts/add-matrix-org-stylings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 83888346..ef4e1014 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -9,7 +9,7 @@ HEADER="$1/head.html" NAV_BAR="$1/nav.html" FOOTER="$1/footer.html" -for f in "${HEADER}"/{head,nav,footer}.html; do +for f in "$1"/{head,nav,footer}.html; do if [[ ! -e "${f}" ]]; then echo >&2 "Need ${f} to exist" exit 1 From e045f28b44137303acccc833ffb486008c46be06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:20:54 +0000 Subject: [PATCH 18/24] Pull out constant for permissions Also, drop permissions from 0755 to 0700 --- scripts/speculator/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4972210e..cfe0f4ee 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -64,13 +64,14 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" - matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + permissionsOwnerFull = 0700 ) func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - if err := os.MkdirAll(directory, 0755); err != nil { + if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil { return "", fmt.Errorf("error making directory %s: %v", directory, err) } args := []string{"clone", url, directory} From 866fa582763279063856ea89c3459e4eeb5ae2b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:22:53 +0000 Subject: [PATCH 19/24] Rename --- scripts/speculator/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index cfe0f4ee..e1874383 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -148,7 +148,8 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } -func (s *server) knowsAbout(sha string) bool { +// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL. +func (s *server) canCheckout(sha string) bool { s.mu.Lock() defer s.mu.Unlock() return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil @@ -157,7 +158,7 @@ func (s *server) knowsAbout(sha string) bool { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - if !s.knowsAbout(sha) { + if !s.canCheckout(sha) { err = s.updateBase() if err != nil { return From c432396079645052c8a9c60eb5be822889d25117 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:26:32 +0000 Subject: [PATCH 20/24] Add comment --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e1874383..0c9a2bfa 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -197,6 +197,8 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. + // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return From 6a6cbd9d24a7cbef9db04a57b6d273e09d51ddf1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:28:58 +0000 Subject: [PATCH 21/24] Always try to build continuserv & speculator on jenkins --- jenkins.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index b2aa8489..f5ed3b15 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -8,7 +8,11 @@ set -ex (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) -if which go >/dev/null 2>/dev/null; then - (cd scripts/continuserv && go build) - (cd scripts/speculator && go build) -fi +: ${GOPATH:=${WORKSPACE}/.gopath} +mkdir -p "${GOPATH}" +export GOPATH +go get github.com/hashicorp/golang-lru +go get gopkg.in/fsnotify.v1 + +(cd scripts/continuserv && go build) +(cd scripts/speculator && go build) From 4e42aab245514a0d7373e23b282c1a284b100dec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 25 Nov 2015 15:13:00 +0000 Subject: [PATCH 22/24] Fix up the multi-hash ratchet thing --- drafts/markjh_end_to_end.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst index 1699f79b..07390b5e 100644 --- a/drafts/markjh_end_to_end.rst +++ b/drafts/markjh_end_to_end.rst @@ -84,18 +84,22 @@ follows: .. math:: \begin{align} - R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ - R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\ R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ - R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ - R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ - R_{i,3} &= H_1\left(R_{(i-1),3}\right) + R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_3\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_3\left(R_{(i-1),3}\right) \end{align} -Where :math:`H_1` and :math:`H_2` are different hash functions. For example -:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and -:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. +Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3` +are different hash functions. For example +:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`. So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. From ec31c0f5184fcf932981b854d770e90011e8fd24 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 12:04:37 +0000 Subject: [PATCH 23/24] speculator: allow styling like matrix.org --- scripts/speculator/main.go | 47 ++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 0c9a2bfa..74731ff6 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -54,9 +54,11 @@ type User struct { } var ( - port = flag.Int("port", 9000, "Port on which to listen for HTTP") - allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") + allowedMembers map[string]bool + specCache *lru.Cache // string -> []byte + styledSpecCache *lru.Cache // string -> []byte ) func (u *User) IsTrusted() bool { @@ -196,6 +198,13 @@ func (s *server) getSHAOf(ref string) (string, error) { func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string + var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != "" + + if styleLikeMatrixDotOrg && *includesDir == "" { + writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified")) + return + } + if strings.ToLower(req.URL.Path) == "/spec/head" { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. @@ -220,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { } sha = pr.Head.SHA } - if cached, ok := specCache.Get(sha); ok { + + var cache = specCache + if styleLikeMatrixDotOrg { + cache = styledSpecCache + } + + if cached, ok := cache.Get(sha); ok { w.Write(cached.([]byte)) return } @@ -232,13 +247,24 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) if err != nil { writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) return } w.Write(b) - specCache.Add(sha, b) + cache.Add(sha, b) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -384,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) { pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } s += `` + if *includesDir != "" { + s += `` + } + io.WriteString(w, s) } @@ -448,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) { } func initCache() error { - c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) - specCache = c + c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + specCache = c1 + + c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + styledSpecCache = c2 return err } From 5e30b5b8d74cdfbd764175fd735b3c39d652453e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 16:46:29 +0000 Subject: [PATCH 24/24] Remove POST version of /send PUT should always be used. --- api/client-server/v1/room_send.yaml | 47 --------------------- specification/modules/instant_messaging.rst | 8 +--- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml index 9c9273df..3ab1ac40 100644 --- a/api/client-server/v1/room_send.yaml +++ b/api/client-server/v1/room_send.yaml @@ -76,50 +76,3 @@ paths: type: string description: |- A unique identifier for the event. - "/rooms/{roomId}/send/{eventType}": - post: - summary: Send a message event to the given room. - description: |- - This endpoint can be used to send a message event to a room; however - the lack of a transaction ID means that it is possible to cause message - duplication if events are resent on error, so it is preferable to use - `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room to send the event to. - required: true - x-example: "!636q39766251:example.com" - - in: path - type: string - name: eventType - description: The type of event to send. - required: true - x-example: "m.room.message" - - in: body - name: body - schema: - type: object - example: |- - { - "msgtype": "m.text", - "body": "hello" - } - responses: - 200: - description: "An ID for the sent event." - examples: - application/json: |- - { - "event_id": "YUwRidLecu" - } - schema: - type: object - properties: - event_id: - type: string - description: |- - A unique identifier for the event. diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index a58c762f..cd385001 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key. Recommendations when sending messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use -``PUT`` requests with `transaction IDs`_ to make requests idempotent. This -ensures that messages are sent exactly once even under poor network conditions. -Clients SHOULD retry requests using an exponential-backoff algorithm for a +In the event of send failure, clients SHOULD retry requests using an +exponential-backoff algorithm for a certain amount of time T. It is recommended that T is no longer than 5 minutes. After this time, the client should stop retrying and mark the message as "unsent". Users should be able to manually resend unsent messages. @@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room rather than a global queue, as ordering is only relevant within a single room rather than between rooms. -.. _`transaction IDs`: `sect:txn_ids`_ - Local echo ~~~~~~~~~~