Merge branch 'master' into markjh/end_to_end_encryption

api/client-server/v1/room_send.yaml

@@ -76,50 +76,3 @@ paths:
                 type: string
                 description: |-
                   A unique identifier for the event.
-  "/rooms/{roomId}/send/{eventType}":
-    post:
-      summary: Send a message event to the given room.
-      description: |-
-        This endpoint can be used to send a message event to a room; however
-        the lack of a transaction ID means that it is possible to cause message
-        duplication if events are resent on error, so it is preferable to use
-        `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_.
-      security:
-        - accessToken: []
-      parameters:
-        - in: path
-          type: string
-          name: roomId
-          description: The room to send the event to.
-          required: true
-          x-example: "!636q39766251:example.com"
-        - in: path
-          type: string
-          name: eventType
-          description: The type of event to send.
-          required: true
-          x-example: "m.room.message"
-        - in: body
-          name: body
-          schema:
-            type: object
-            example: |-
-              {
-                "msgtype": "m.text",
-                "body": "hello"
-              }
-      responses:
-        200:
-          description: "An ID for the sent event."
-          examples:
-            application/json: |-
-              {
-                  "event_id": "YUwRidLecu"
-              }
-          schema:
-            type: object
-            properties:
-              event_id:
-                type: string
-                description: |-
-                  A unique identifier for the event.

drafts/markjh_end_to_end.rst

@@ -84,18 +84,22 @@ follows:
 
 .. math::
     \begin{align}
-    R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\
+    R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\
-    R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\
+    R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\
+    R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\
+    R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\
     R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\
     R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\
-    R_{2^8i,2}  &= H_1\left(R_{2^8(i-1),2}\right) \\
+    R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\
-    R_{2^8i,3}  &= H_2\left(R_{2^8(i-1),2}\right) \\
+    R_{2^8i,2}  &= H_2\left(R_{2^8(i-1),2}\right) \\
-    R_{i,3}     &= H_1\left(R_{(i-1),3}\right)
+    R_{2^8i,3}  &= H_3\left(R_{2^8(i-1),2}\right) \\
+    R_{i,3}     &= H_3\left(R_{(i-1),3}\right)
     \end{align}
 
-Where :math:`H_1` and :math:`H_2` are different hash functions. For example
+Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3`
-:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and
+are different hash functions. For example
-:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`.
+:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and
+:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`.
 
 So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`.
 Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`.

event-schemas/check_examples.py

@@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir):
                 continue
             examplepath = os.path.join(root, filename)
             schemapath = examplepath.replace(exampledir, schemadir)
+            if schemapath.find("#") >= 0:
+                schemapath = schemapath[:schemapath.find("#")]
             try:
                 check_example_file(examplepath, schemapath)
             except Exception as e:

event-schemas/examples/v1/m.room.member#invite_room_state

@@ -0,0 +1,30 @@
+{
+  "age": 242352,
+  "content": {
+    "membership": "join",
+    "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
+    "displayname": "Alice Margatroid"
+  },
+  "invite_room_state": [
+    {
+      "type": "m.room.name",
+      "state_key": "",
+      "content": {
+        "name": "Forest of Magic"
+      }
+    },
+    {
+      "type": "m.room.join_rules",
+      "state_key": "",
+      "content": {
+        "join_rules": "invite"
+      }
+    }
+  ],
+  "state_key": "@alice:localhost",
+  "origin_server_ts": 1431961217939,
+  "event_id": "$WLGTSEFSEF:localhost",
+  "type": "m.room.member",
+  "room_id": "!Cuyf34gef24t:localhost",
+  "user_id": "@example:localhost"
+}

event-schemas/examples/v1/m.room.member#third_party_invite

@@ -0,0 +1,25 @@
+{
+  "age": 242352,
+  "content": {
+    "membership": "join",
+    "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
+    "displayname": "Alice Margatroid",
+    "third_party_invite": {
+      "signed": {
+        "mxid": "@alice:localhost",
+        "signatures": {
+          "magic.forest": {
+            "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
+          }
+        },
+        "token": "abc123"
+      }
+    }
+  },
+  "state_key": "@alice:localhost",
+  "origin_server_ts": 1431961217939,
+  "event_id": "$WLGTSEFSEF:localhost",
+  "type": "m.room.member",
+  "room_id": "!Cuyf34gef24t:localhost",
+  "user_id": "@example:localhost"
+}

jenkins.sh

@@ -7,3 +7,12 @@ set -ex
 (cd scripts && ./gendoc.py -v)
 (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha")
 (cd event-schemas/ && ./check.sh)
+
+: ${GOPATH:=${WORKSPACE}/.gopath}
+mkdir -p "${GOPATH}"
+export GOPATH
+go get github.com/hashicorp/golang-lru
+go get gopkg.in/fsnotify.v1
+
+(cd scripts/continuserv && go build)
+(cd scripts/speculator && go build)

scripts/add-matrix-org-stylings.sh

@@ -1,41 +1,20 @@
-#! /bin/bash
+#!/bin/bash -eu
 
-if [ -z "$1" ]; then
+if [[ $# != 1 || ! -d $1 ]]; then
-    echo "Expected /includes/head.html file as 1st arg."
+  echo >&2 "Usage: $0 include_dir"
   exit 1
 fi
 
-if [ -z "$2" ]; then
+HEADER="$1/head.html"
-    echo "Expected /includes/nav.html file as 2nd arg."
+NAV_BAR="$1/nav.html"
-    exit 1
+FOOTER="$1/footer.html"
-fi
-
-if [ -z "$3" ]; then
-    echo "Expected /includes/footer.html file as 3rd arg."
-    exit 1
-fi
-
-
-HEADER=$1
-NAV_BAR=$2
-FOOTER=$3
 
-if [ ! -f $HEADER ]; then
+for f in "$1"/{head,nav,footer}.html; do
-    echo $HEADER " does not exist"
+  if [[ ! -e "${f}" ]]; then
+    echo >&2 "Need ${f} to exist"
     exit 1
   fi
-
+done
-if [ ! -f $NAV_BAR ]; then
-    echo $NAV_BAR " does not exist"
-    exit 1
-fi
-
-if [ ! -f $FOOTER ]; then
-    echo $FOOTER " does not exist"
-    exit 1
-fi
-
-python gendoc.py
 
 perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s#<head>#<head>$header
   <link rel="stylesheet" href="//matrix.org/docs/guides/css/docs_overrides.css">

scripts/gendoc.py

@@ -238,6 +238,18 @@ def rst2html(i, o):
             )
 
 
+def addAnchors(path):
+    with open(path, "r") as f:
+        lines = f.readlines()
+
+    replacement = replacement = r'<p><a class="anchor" id="\3"></a></p>\n\1'
+    with open(path, "w") as f:
+        for line in lines:
+            line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
+            line = re.sub(r'(<div class="section" (id)="(.*?)">)', replacement, line.rstrip())
+            f.write(line + "\n")
+
+
 def run_through_template(input, set_verbose):
     tmpfile = './tmp/output'
     try:
@@ -387,6 +399,7 @@ def main(target_name, keep_intermediates):
     shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
     run_through_template("tmp/howto.rst", False)  # too spammy to mark -v on this
     rst2html("tmp/full_spec.rst", "gen/specification.html")
+    addAnchors("gen/specification.html")
     rst2html("tmp/howto.rst", "gen/howtos.html")
     if not keep_intermediates:
         cleanup_env()

scripts/speculator/main.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -54,8 +55,10 @@ type User struct {
 
 var (
 	port            = flag.Int("port", 9000, "Port on which to listen for HTTP")
+	includesDir     = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org")
 	allowedMembers  map[string]bool
 	specCache       *lru.Cache // string -> []byte
+	styledSpecCache *lru.Cache // string -> []byte
 )
 
 func (u *User) IsTrusted() bool {
@@ -65,17 +68,20 @@ func (u *User) IsTrusted() bool {
 const (
 	pullsPrefix          = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
 	matrixDocCloneURL    = "https://github.com/matrix-org/matrix-doc.git"
+	permissionsOwnerFull = 0700
 )
 
 func gitClone(url string, shared bool) (string, error) {
 	directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
-	cmd := exec.Command("git", "clone", url, directory)
+	if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil {
+		return "", fmt.Errorf("error making directory %s: %v", directory, err)
+	}
+	args := []string{"clone", url, directory}
 	if shared {
-		cmd.Args = append(cmd.Args, "--shared")
+		args = append(args, "--shared")
 	}
-
+	if err := runGitCommand(directory, args); err != nil {
-	if err := cmd.Run(); err != nil {
+		return "", err
-		return "", fmt.Errorf("error cloning repo: %v", err)
 	}
 	return directory, nil
 }
@@ -84,15 +90,13 @@ func gitCheckout(path, sha string) error {
 	return runGitCommand(path, []string{"checkout", sha})
 }
 
-func gitFetch(path string) error {
-	return runGitCommand(path, []string{"fetch"})
-}
-
 func runGitCommand(path string, args []string) error {
 	cmd := exec.Command("git", args...)
 	cmd.Dir = path
+	var b bytes.Buffer
+	cmd.Stderr = &b
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err)
+		return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String())
 	}
 	return nil
 }
@@ -136,17 +140,35 @@ func writeError(w http.ResponseWriter, code int, err error) {
 }
 
 type server struct {
+	mu                sync.Mutex // Must be locked around any git command on matrixDocCloneURL
 	matrixDocCloneURL string
 }
 
+func (s *server) updateBase() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return runGitCommand(s.matrixDocCloneURL, []string{"fetch"})
+}
+
+// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL.
+func (s *server) canCheckout(sha string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil
+}
+
 // generateAt generates spec from repo at sha.
 // Returns the path where the generation was done.
 func (s *server) generateAt(sha string) (dst string, err error) {
-	err = gitFetch(s.matrixDocCloneURL)
+	if !s.canCheckout(sha) {
+		err = s.updateBase()
 		if err != nil {
 			return
 		}
+	}
+	s.mu.Lock()
 	dst, err = gitClone(s.matrixDocCloneURL, true)
+	s.mu.Unlock()
 	if err != nil {
 		return
 	}
@@ -164,7 +186,10 @@ func (s *server) getSHAOf(ref string) (string, error) {
 	cmd.Dir = path.Join(s.matrixDocCloneURL)
 	var b bytes.Buffer
 	cmd.Stdout = &b
-	if err := cmd.Run(); err != nil {
+	s.mu.Lock()
+	err := cmd.Run()
+	s.mu.Unlock()
+	if err != nil {
 		return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
 	}
 	return strings.TrimSpace(b.String()), nil
@@ -173,17 +198,22 @@ func (s *server) getSHAOf(ref string) (string, error) {
 func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 	var sha string
 
-	if strings.ToLower(req.URL.Path) == "/spec/head" {
+	var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != ""
-		if err := gitFetch(s.matrixDocCloneURL); err != nil {
+
-			writeError(w, 500, err)
+	if styleLikeMatrixDotOrg && *includesDir == "" {
+		writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified"))
 		return
 	}
-		originHead, err := s.getSHAOf("origin/master")
+
-		if err != nil {
+	if strings.ToLower(req.URL.Path) == "/spec/head" {
+		// err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring.
+		// This is to deal with cases like where github is down but we still want to serve the spec.
+		if headSha, err := s.lookupHeadSHA(); headSha == "" {
 			writeError(w, 500, err)
 			return
+		} else {
+			sha = headSha
 		}
-		sha = originHead
 	} else {
 		pr, err := lookupPullRequest(*req.URL, "/spec")
 		if err != nil {
@@ -199,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 		}
 		sha = pr.Head.SHA
 	}
-	if cached, ok := specCache.Get(sha); ok {
+
+	var cache = specCache
+	if styleLikeMatrixDotOrg {
+		cache = styledSpecCache
+	}
+
+	if cached, ok := cache.Get(sha); ok {
 		w.Write(cached.([]byte))
 		return
 	}
@@ -211,13 +247,43 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	if styleLikeMatrixDotOrg {
+		cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
+		cmd.Dir = path.Join(dst, "scripts")
+		var b bytes.Buffer
+		cmd.Stderr = &b
+		if err := cmd.Run(); err != nil {
+			writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
+			return
+		}
+	}
+
 	b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
 	if err != nil {
 		writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
 		return
 	}
 	w.Write(b)
-	specCache.Add(sha, b)
+	cache.Add(sha, b)
+}
+
+// lookupHeadSHA looks up what origin/master's HEAD SHA is.
+// It attempts to `git fetch` before doing so.
+// If this fails, it may still return a stale sha, but will also return an error.
+func (s *server) lookupHeadSHA() (sha string, retErr error) {
+	retErr = s.updateBase()
+	if retErr != nil {
+		log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr)
+	}
+	originHead, err := s.getSHAOf("origin/master")
+	if err != nil {
+		retErr = err
+	}
+	sha = originHead
+	if retErr != nil && originHead != "" {
+		log.Printf("Successfully fell back to possibly stale sha: %s", sha)
+	}
+	return
 }
 
 func checkAuth(pr *PullRequest) error {
@@ -344,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) {
 			pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number)
 	}
 	s += `</ul><div><a href="spec/head">View the spec at head</a></div></body>`
+	if *includesDir != "" {
+		s += `</ul><div><a href="spec/head?matrixdotorgstyle=1">View the spec at head styled like matrix.org</a></div></body>`
+	}
+
 	io.WriteString(w, s)
 }
 
@@ -383,7 +453,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	s := server{masterCloneDir}
+	s := server{matrixDocCloneURL: masterCloneDir}
 	http.HandleFunc("/spec/", forceHTML(s.serveSpec))
 	http.HandleFunc("/diff/rst/", s.serveRSTDiff)
 	http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff))
@@ -408,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) {
 }
 
 func initCache() error {
-	c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
+	c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
-	specCache = c
+	specCache = c1
+
+	c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
+	styledSpecCache = c2
 	return err
 }

specification/modules/instant_messaging.rst

@@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key.
 Recommendations when sending messages
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use
+In the event of send failure, clients SHOULD retry requests using an
-``PUT`` requests with `transaction IDs`_ to make requests idempotent. This
+exponential-backoff algorithm for a
-ensures that messages are sent exactly once even under poor network conditions.
-Clients SHOULD retry requests using an exponential-backoff algorithm for a
 certain amount of time T. It is recommended that T is no longer than 5 minutes.
 After this time, the client should stop retrying and mark the message as "unsent".
 Users should be able to manually resend unsent messages.
@@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room
 rather than a global queue, as ordering is only relevant within a single room
 rather than between rooms.
 
-.. _`transaction IDs`: `sect:txn_ids`_
-
 Local echo
 ~~~~~~~~~~
 

templating/matrix_templates/sections.py

@@ -35,7 +35,7 @@ class MatrixSections(Sections):
             if not filterFn(event_name):
                 continue
             sections.append(template.render(
-                example=examples[event_name], 
+                examples=examples[event_name],
                 event=schemas[event_name],
                 title_kind=subtitle_title_char
             ))
@@ -136,7 +136,7 @@ class MatrixSections(Sections):
             if not event_name.startswith("m.room.message#m."):
                 continue
             sections.append(template.render(
-                example=examples[event_name], 
+                example=examples[event_name][0],
                 event=schemas[event_name],
                 title_kind=subtitle_title_char
             ))

templating/matrix_templates/templates/events.tmpl

@@ -12,8 +12,10 @@
 {{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }}
 
 {% endfor %}
-Example:
+Example{% if examples | length > 1 %}s{% endif %}:
 
+{% for example in examples %}
 .. code:: json
 
     {{example | jsonify(4, 4)}}
+{% endfor %}

templating/matrix_templates/units.py

@@ -572,9 +572,14 @@ class MatrixUnits(Units):
             if not filename.startswith("m."):
                 continue
             with open(os.path.join(path, filename), "r") as f:
-                examples[filename] = json.loads(f.read())
+                event_name = filename.split("#")[0]
-                if filename == "m.room.message#m.text":
+                example = json.loads(f.read())
-                    examples["m.room.message"] = examples[filename]
+
+                examples[filename] = examples.get(filename, [])
+                examples[filename].append(example)
+                if filename != event_name:
+                    examples[event_name] = examples.get(event_name, [])
+                    examples[event_name].append(example)
         return examples
 
     def load_event_schemas(self):