Merge branch 'master' into module-presence

Conflicts:
	specification/modules/presence.rst

api/client-server/v1/sync.yaml

@@ -267,6 +267,12 @@ paths:
                       type: string
                       description: "The user's membership state in this room."
                       enum: ["invite", "join", "leave", "ban"]
+                    invite:
+                        type: object
+                        title: "InviteEvent"
+                        description: "The invite event if ``membership`` is ``invite``"
+                        allOf:
+                          - "$ref": "v1-event-schema/m.room.member"
                     messages:
                       type: object
                       title: PaginationChunk

event-schemas/schema/v1/m.room.member

@@ -32,6 +32,26 @@
         "type": {
             "type": "string",
             "enum": ["m.room.member"]
+        },
+        "invite_room_state": {
+            "type": "array",
+            "description": "A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``",
+            "items": {
+                "type": "object",
+                "title": "StateEvent",
+                "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.",
+                "properties": {
+                    "type": {
+                        "type": "string"
+                    },
+                    "state_key": {
+                        "type": "string"
+                    },
+                    "content": {
+                        "type": "object"
+                    }
+                }
+            }
         }
     }
 }

scripts/continuserv/main.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	fsnotify "gopkg.in/fsnotify.v1"
 )
@@ -67,7 +68,6 @@ func watchFS(ch chan struct{}, w *fsnotify.Watcher) {
 		select {
 		case e := <-w.Events:
 			if filter(e) {
-				wg.Add(1)
 				fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name)
 				ch <- struct{}{}
 			}
@@ -98,6 +98,11 @@ func filter(e fsnotify.Event) bool {
 		return false
 	}
 
+	// Ignore the .git directory - It's very noisy
+	if strings.Contains(e.Name, "/.git/") {
+		return false
+	}
+
 	// Avoid infinite cycles being caused by writing actual output
 	if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") {
 		return false
@@ -133,10 +138,22 @@ func populateOnce(dir string) {
 }
 
 func doPopulate(ch chan struct{}, dir string) {
-	for _ = range ch {
+	var pending int
+	for {
+		select {
+		case <-ch:
+			if pending == 0 {
+				wg.Add(1)
+			}
+			pending++
+		case <-time.After(10 * time.Millisecond):
+			if pending > 0 {
+				pending = 0
 				populateOnce(dir)
 			}
 		}
+	}
+}
 
 func exists(path string) bool {
 	_, err := os.Stat(path)

scripts/speculator/main.go

@@ -24,6 +24,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 )
 
 type PullRequest struct {
@@ -58,16 +59,24 @@ func (u *User) IsTrusted() bool {
 	return allowedMembers[u.Login]
 }
 
-const pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
+const (
+	pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
+	matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
+)
+
+
+func gitClone(url string, shared bool) (string, error) {
+	directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
+	cmd := exec.Command("git", "clone", url, directory)
+	if shared {
+		cmd.Args = append(cmd.Args, "--shared")
+	}
 
-func gitClone(url string) (string, error) {
-	dst := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
-	cmd := exec.Command("git", "clone", url, dst)
 	err := cmd.Run()
 	if err != nil {
 		return "", fmt.Errorf("error cloning repo: %v", err)
 	}
-	return dst, nil
+	return directory, nil
 }
 
 func gitCheckout(path, sha string) error {
@@ -80,6 +89,16 @@ func gitCheckout(path, sha string) error {
 	return nil
 }
 
+func gitFetch(path string) error {
+	cmd := exec.Command("git", "fetch")
+	cmd.Dir = path
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error fetching repo: %v", err)
+	}
+	return nil
+}
+
 func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) {
 	if !strings.HasPrefix(url.Path, pathPrefix+"/") {
 		return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
@@ -119,10 +138,18 @@ func writeError(w http.ResponseWriter, code int, err error) {
 	io.WriteString(w, fmt.Sprintf("%v\n", err))
 }
 
+type server struct {
+	matrixDocCloneURL string
+}
+
 // generateAt generates spec from repo at sha.
 // Returns the path where the generation was done.
-func generateAt(repo, sha string) (dst string, err error) {
+func (s *server) generateAt(sha string) (dst string, err error) {
-	dst, err = gitClone(repo)
+	err = gitFetch(s.matrixDocCloneURL)
+	if err != nil {
+		return
+	}
+	dst, err = gitClone(s.matrixDocCloneURL, true)
 	if err != nil {
 		return
 	}
@@ -135,12 +162,10 @@ func generateAt(repo, sha string) (dst string, err error) {
 	return
 }
 
-func serveSpec(w http.ResponseWriter, req *http.Request) {
+func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
-	var cloneURL string
 	var sha string
 
 	if strings.ToLower(req.URL.Path) == "/spec/head" {
-		cloneURL = "https://github.com/matrix-org/matrix-doc.git"
 		sha = "HEAD"
 	} else {
 		pr, err := lookupPullRequest(*req.URL, "/spec")
@@ -155,11 +180,10 @@ func serveSpec(w http.ResponseWriter, req *http.Request) {
 			writeError(w, 403, err)
 			return
 		}
-		cloneURL = pr.Head.Repo.CloneURL
 		sha = pr.Head.SHA
 	}
 
-	dst, err := generateAt(cloneURL, sha)
+	dst, err := s.generateAt(sha)
 	defer os.RemoveAll(dst)
 	if err != nil {
 		writeError(w, 500, err)
@@ -181,7 +205,7 @@ func checkAuth(pr *PullRequest) error {
 	return nil
 }
 
-func serveRSTDiff(w http.ResponseWriter, req *http.Request) {
+func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) {
 	pr, err := lookupPullRequest(*req.URL, "/diff/rst")
 	if err != nil {
 		writeError(w, 400, err)
@@ -195,14 +219,14 @@ func serveRSTDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA)
+	base, err := s.generateAt(pr.Base.SHA)
 	defer os.RemoveAll(base)
 	if err != nil {
 		writeError(w, 500, err)
 		return
 	}
 
-	head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA)
+	head, err := s.generateAt(pr.Head.SHA)
 	defer os.RemoveAll(head)
 	if err != nil {
 		writeError(w, 500, err)
@@ -219,7 +243,7 @@ func serveRSTDiff(w http.ResponseWriter, req *http.Request) {
 	w.Write(diff.Bytes())
 }
 
-func serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
+func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
 	pr, err := lookupPullRequest(*req.URL, "/diff/html")
 	if err != nil {
 		writeError(w, 400, err)
@@ -233,14 +257,14 @@ func serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA)
+	base, err := s.generateAt(pr.Base.SHA)
 	defer os.RemoveAll(base)
 	if err != nil {
 		writeError(w, 500, err)
 		return
 	}
 
-	head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA)
+	head, err := s.generateAt(pr.Head.SHA)
 	defer os.RemoveAll(head)
 	if err != nil {
 		writeError(w, 500, err)
@@ -327,9 +351,15 @@ func main() {
 		"Kegsay":        true,
 		"NegativeMjark": true,
 	}
-	http.HandleFunc("/spec/", serveSpec)
+	rand.Seed(time.Now().Unix())
-	http.HandleFunc("/diff/rst/", serveRSTDiff)
+	masterCloneDir, err := gitClone(matrixDocCloneURL, false)
-	http.HandleFunc("/diff/html/", serveHTMLDiff)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s := server{masterCloneDir}
+	http.HandleFunc("/spec/", s.serveSpec)
+	http.HandleFunc("/diff/rst/", s.serveRSTDiff)
+	http.HandleFunc("/diff/html/", s.serveHTMLDiff)
 	http.HandleFunc("/healthz", serveText("ok"))
 	http.HandleFunc("/", listPulls)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))

specification/0-feature_profiles.rst

@@ -1,3 +1,93 @@
 Feature Profiles
 ================
 
+.. sect:feature-profiles:
+
+Matrix supports many different kinds of clients: from embedded IoT devices to
+desktop clients. Not all clients can provide the same feature sets as other
+clients e.g. due to lack of physical hardware such as not having a screen.
+Clients can fall into one of several profiles and each profile contains a set
+of features that the client MUST support. This section details a set of
+"feature profiles". Clients are expected to implement a profile in its entirety
+in order for it to be classified as that profile.
+
+Summary
+-------
+
+===================================== ========== ========== ========== ========== ==========
+  Module / Profile                       Web       Mobile    Desktop       CLI     Embedded
+===================================== ========== ========== ========== ========== ==========
+ `Instant Messaging`_                  Required   Required   Required   Required   Optional
+ `Presence`_                           Required   Required   Required   Required   Optional
+ `Push Notifications`_                 Optional   Required   Optional   Optional   Optional
+ `Receipts`_                           Required   Required   Required   Required   Optional
+ `Typing Notifications`_               Required   Required   Required   Required   Optional
+ `VoIP`_                               Required   Required   Required   Optional   Optional
+ `Content Repository`_                 Required   Required   Required   Optional   Optional
+ `Managing History Visibility`_        Required   Required   Required   Required   Optional
+ `End-to-End Encryption`_              Optional   Optional   Optional   Optional   Optional
+===================================== ========== ========== ========== ========== ==========
+
+*Please see each module for more details on what clients need to implement.*
+
+.. _End-to-End Encryption: `module:e2e`_
+.. _Instant Messaging: `module:im`_
+.. _Presence: `module:presence`_
+.. _Push Notifications: `module:push`_
+.. _Receipts: `module:receipts`_
+.. _Typing Notifications: `module:typing`_
+.. _VoIP: `module:voip`_
+.. _Content Repository: `module:content`_
+.. _Managing History Visibility: `module:history-visibility`_
+
+Clients
+-------
+
+Stand-alone web (``Web``)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a web page which heavily uses Matrix for communication. Single-page web
+apps would be classified as a stand-alone web client, as would multi-page web
+apps which use Matrix on nearly every page.
+
+Mobile (``Mobile``)
+~~~~~~~~~~~~~~~~~~~
+
+This is a Matrix client specifically designed for consumption on mobile devices.
+This is typically a mobile app but need not be so provided the feature set can
+be reached (e.g. if a mobile site could display push notifications it could be
+classified as a mobile client).
+
+Desktop (``Desktop``)
+~~~~~~~~~~~~~~~~~~~~~
+
+This is a native GUI application which can run in its own environment outside a
+browser.
+
+Command Line Interface (``CLI``)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a client which is used via a text-based terminal.
+
+Embedded (``Embedded``)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a client which is embedded into another application or an embedded
+device.
+
+Application
++++++++++++
+
+This is a Matrix client which is embedded in another website, e.g. using
+iframes. These embedded clients are typically for a single purpose
+related to the website in question, and are not intended to be fully-fledged
+communication apps.
+
+Device
+++++++
+
+This is a client which is typically running on an embedded device such as a
+kettle, fridge or car. These clients tend to perform a few operations and run
+in a resource constrained environment. Like embedded applications, they are
+not intended to be fully-fledged communication systems.
+

specification/modules/content_repo.rst

@@ -1,6 +1,8 @@
 Content repository
 ==================
 
+.. _module:content:
+
 HTTP API
 --------
 

specification/modules/end_to_end_encryption.rst

@@ -1,6 +1,8 @@
 End-to-End Encryption
 =====================
 
+.. _module:e2e:
+
 .. TODO-doc
   - Why is this needed.
   - Overview of process

specification/modules/history_visibility.rst

@@ -1,6 +1,8 @@
 Room History Visibility
 -----------------------
 
+.. _module:history-visibility:
+
 Whether a member of a room can see the events that happened in a room from
 before they joined the room is controlled by the ``history_visibility`` key
 of the ``m.room.history_visibility`` state event. The valid values for

specification/modules/instant_messaging.rst

@@ -1,6 +1,8 @@
 Instant Messaging
 =================
 
+.. _module:im:
+
 Events
 ------
 

specification/modules/presence.rst

@@ -1,6 +1,8 @@
 Presence
 ========
 
+.. _module:presence:
+
 Each user has the concept of presence information. This encodes:
 
  * Whether the user is currently online

specification/modules/push_overview.rst

@@ -1,6 +1,8 @@
 Push Notifications
 ==================
 
+.. _module:push:
+
 Overview
 --------
 

specification/modules/receipts.rst

@@ -1,6 +1,8 @@
 Receipts
 --------
 
+.. _module:receipts:
+
 Receipts are used to publish which events in a room the user or their devices
 have interacted with. For example, which events the user has read. For
 efficiency this is done as "up to" markers, i.e. marking a particular event

specification/modules/typing_notifications.rst

@@ -1,6 +1,8 @@
 Typing Notifications
 --------------------
 
+.. _module:typing:
+
 Client APIs
 ~~~~~~~~~~~
 

specification/modules/voip_events.rst

@@ -1,5 +1,8 @@
 Voice over IP
 -------------
+
+.. _module:voip:
+
 Matrix can also be used to set up VoIP calls. This is part of the core
 specification, although is at a relatively early stage. Voice (and video) over
 Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like

specification/targets.yaml

@@ -2,11 +2,11 @@ targets:
   main:  # arbitrary name to identify this build target
     files:  # the sort order of files to cat
       - 0-intro.rst
-      - { 1: 0-feature_profiles.rst }
       - 1-client_server_api.rst
       - { 1: 0-events.rst }
       - { 1: 0-event_signing.rst }
       - 2-modules.rst
+      - { 1: 0-feature_profiles.rst }
       - { 1: "group:modules" }  # reference a group of files
       - 3-application_service_api.rst
       - 4-server_server_api.rst

templating/build.py

@@ -122,7 +122,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
 
     # check the input files and substitute in sections where required
     log("Parsing input template: %s" % file_stream.name)
-    temp_str = file_stream.read()
+    temp_str = file_stream.read().decode("utf-8")
     # do sanity checking on the template to make sure they aren't reffing things
     # which will never be replaced with a section.
     ast = env.parse(temp_str)
@@ -140,7 +140,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
     with open(
             os.path.join(out_dir, os.path.basename(file_stream.name)), "w"
             ) as f:
-        f.write(output)
+        f.write(output.encode("utf-8"))
     log("Output file for: %s" % file_stream.name)
     check_unaccessed("units", units)