Merge branch 'master' into module-content-repo

Conflicts:
	templating/matrix_templates/units.py

api/client-server/v1/voip.yaml

@@ -0,0 +1,68 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Voice over IP API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/turnServer":
+    get:
+      summary: Obtain TURN server credentials.
+      description: |-
+        This API provides credentials for the client to use when initiating
+        calls.
+      security:
+        - accessToken: []
+      responses:
+        200:
+          description: The TURN server credentials.
+          examples:
+            application/json: |-
+              {
+                "username":"1443779631:@user:example.com",
+                "password":"JlKfBy1QwLrO20385QyAtEyIv0=",
+                "uris":[
+                  "turn:turn.example.com:3478?transport=udp",
+                  "turn:10.20.30.40:3478?transport=tcp",
+                  "turns:10.20.30.40:443?transport=tcp"
+                ],
+                "ttl":86400
+              }
+          schema:
+            type: object
+            properties:
+              username:
+                type: string
+                description: |-
+                  The username to use.
+              password:
+                type: string
+                description: |-
+                  The password to use.
+              uris:
+                type: array
+                items:
+                  type: string
+                description: A list of TURN URIs
+              ttl:
+                type: integer
+                description: The time-to-live in seconds
+            required: ["username", "password", "uris", "ttl"]
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"
+

api/client-server/v2_alpha/receipts.yaml

@@ -0,0 +1,68 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v2 Receipts API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/v2_alpha
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/rooms/{roomId}/receipt/{receiptType}/{eventId}":
+    post:
+      summary: Send a receipt for the given event ID.
+      description: |-
+        This API updates the marker for the given receipt type to the event ID
+        specified.
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room in which to send the event.
+          required: true
+          x-example: "!wefuh21ffskfuh345:example.com"
+        - in: path
+          type: string
+          name: receiptType
+          description: The type of receipt to send.
+          required: true
+          x-example: "m.read"
+          enum: ["m.read"]
+        - in: path
+          type: string
+          name: eventId
+          description: The event ID to acknowledge up to.
+          required: true
+          x-example: "$1924376522eioj:example.com"
+        - in: body
+          description: |-
+            Extra receipt information to attach to ``content`` if any. The
+            server will automatically set the ``ts`` field.
+          schema:
+            type: object
+            example: |-
+              {}
+      responses:
+        200:
+          description: The receipt was sent.
+          examples:
+            application/json: |-
+              {}
+          schema:
+            type: object  # empty json object
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"

event-schemas/examples/v1/m.receipt

@@ -3,7 +3,7 @@
     "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org",
     "content": {
         "$1435641916114394fHBLK:matrix.org": {
-            "read": {
+            "m.read": {
                 "@rikj:jki.re": {
                     "ts": 1436451550453
                 }

event-schemas/schema/v1/m.receipt

@@ -5,26 +5,32 @@
     "properties": {
         "content": {
             "type": "object",
-            "description": "The event ids which the receipts relate to.",
             "patternProperties": {
                 "^\\$": {
                     "type": "object",
-                    "description": "The types of the receipts.",
+                    "x-pattern": "$EVENT_ID",
-                    "additionalProperties": {
+                    "title": "Receipts",
-                        "type": "object",
+                    "description": "The mapping of event ID to a collection of receipts for this event ID. The event ID is the ID of the event being acknowledged and *not* an ID for the receipt itself.",
-                        "description": "User ids of the receipts",
+                    "properties": {
-                        "patternProperties": {
+                        "m.read": {
-                            "^@": {
+                            "type": "object",
-                                "type": "object",
+                            "title": "Users",
-                                "properties": {
+                            "description": "A collection of users who have sent ``m.read`` receipts for this event.",
-                                    "ts": {
+                            "patternProperties": {
-                                        "type": "number",
+                                "^@": {
-                                        "description": "The timestamp the receipt was sent at"
+                                    "type": "object",
+                                    "title": "Receipt",
+                                    "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.",
+                                    "x-pattern": "$USER_ID",
+                                    "properties": {
+                                        "ts": {
+                                            "type": "number",
+                                            "description": "The timestamp the receipt was sent at."
+                                        }
                                     }
                                 }
                             }
-                        },
+                        }
-                        "additionalProperties": false
                     }
                 }
             },

specification/1-client_server_api.rst

@@ -427,6 +427,8 @@ the complete dataset is provided in "chunk".
 Events
 ------
 
+.. _sect:events:
+
 Overview
 ~~~~~~~~
 

specification/modules/receipts.rst

@@ -1,66 +1,64 @@
 Receipts
---------
+========
 
 .. _module:receipts:
 
-Receipts are used to publish which events in a room the user or their devices
+This module adds in support for receipts. These receipts are a form of
-have interacted with. For example, which events the user has read. For
+acknowledgement of an event. This module defines a single acknowledgement:
-efficiency this is done as "up to" markers, i.e. marking a particular event
+``m.read`` which indicates that the user has read up to a given event.
-as, say, ``read`` indicates the user has read all events *up to* that event.
-
-Client-Server API
-~~~~~~~~~~~~~~~~~
-
-Clients will receive receipts in the following format::
-
-     {
-        "type": "m.receipt",
-        "room_id": <room_id>,
-        "content": {
-            <event_id>: {
-                <receipt_type>: {
-                    <user_id>: { "ts": <ts>, ... },
-                    ...
-                }
-            },
-            ...
-        }
-    }
 
-For example::
+Sending a receipt for each event can result in sending large amounts of traffic
+to a homeserver. To prevent this from becoming a problem, receipts are implemented
+using "up to" markers. This marker indicates that the acknowledgement applies
+to all events "up to and including" the event specified. For example, marking
+an event as "read" would indicate that the user had read all events *up to* the
+referenced event.
 
-    {
+Events
-        "type": "m.receipt",
+------
-        "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org",
+Each ``user_id``, ``receipt_type`` pair must be associated with only a
-        "content": {
+single ``event_id``.
-            "$1435641916114394fHBLK:matrix.org": {
-                "read": {
-                    "@erikj:jki.re": { "ts": 1436451550453 },
-                    ...
-                }
-            },
-            ...
-        }
-    }
 
-For efficiency, receipts are batched into one event per room. In the initialSync
+{{m_receipt_event}}
-and v2 sync APIs the receipts are listed in a separate top level ``receipts``
-key. Each ``user_id``, ``receipt_type`` pair must be associated with only a
-single ``event_id``. New receipts that come down the event streams are deltas.
-Deltas update existing mappings, clobbering based on ``user_id``,
-``receipt_type`` pairs.
 
+Client behaviour
+----------------
 
-A client can update the markers for its user by issuing a request::
+In v1 ``/initialSync``, receipts are listed in a separate top level ``receipts``
+key. In v2 ``/sync``, receipts are contained in the ``ephemeral`` block for a
+room. New receipts that come down the event streams are deltas which update
+existing mappings. Clients should replace older receipt acknowledgements based
+on ``user_id`` and ``receipt_type`` pairs. For example::
 
-    POST /_matrix/client/v2_alpha/rooms/<room_id>/receipt/read/<event_id>
+  Client receives m.receipt:
+    user = @alice:example.com
+    receipt_type = m.read
+    event_id = $aaa:example.com
 
-Where the contents of the ``POST`` will be included in the content sent to
+  Client receives another m.receipt:
-other users. The server will automatically set the ``ts`` field.
+    user = @alice:example.com
+    receipt_type = m.read
+    event_id = $bbb:example.com
 
+  The client should replace the older acknowledgement for $aaa:example.com with
+  this one for $bbb:example.com
 
-Server-Server API
+Clients should send read receipts when there is some certainty that the event in
-~~~~~~~~~~~~~~~~~
+question has been **displayed** to the user. Simply receiving an event does not
+provide enough certainty that the user has seen the event. The user SHOULD need
+to *take some action* such as viewing the room that the event was sent to or
+dismissing a notification in order for the event to count as "read".
+
+A client can update the markers for its user by interacting with the following
+HTTP APIs.
+
+{{v2_receipts_http_api}}
+
+Server behaviour
+----------------
+
+For efficiency, receipts SHOULD be batched into one event per room before
+delivering them to clients.
 
 Receipts are sent across federation as EDUs with type ``m.receipt``. The
 format of the EDUs are::
@@ -75,5 +73,12 @@ format of the EDUs are::
         ...
     }
 
-These are always sent as deltas to previously sent receipts.
+These are always sent as deltas to previously sent receipts. Currently only a
+single ``<receipt_type>`` should be used: ``m.read``.
+
+Security considerations
+-----------------------
+
+As receipts are sent outside the context of the event graph, there are no
+integrity checks performed on the contents of ``m.receipt`` events.
 

specification/modules/voip_events.rst

@@ -1,20 +1,26 @@
 Voice over IP
--------------
+=============
 
 .. _module:voip:
 
-Matrix can also be used to set up VoIP calls. This is part of the core
+This module outlines how two users in a room can set up a Voice over IP (VoIP)
-specification, although is at a relatively early stage. Voice (and video) over
+call to each other. Voice and video calls are built upon the WebRTC 1.0 standard.
-Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like
+Call signalling is achieved by sending `message events`_ to the room. As a result,
-any other event. This means that clients must only send call events to rooms
+this means that clients MUST only send call events to rooms with exactly two
-with exactly two participants as currently the WebRTC standard is based around
+participants as currently the WebRTC standard is based around two-party
-two-party communication.
+communication.
+
+.. _message events: `sect:events`_
+
+Events
+------
 
 {{voip_events}}
 
-Message Exchange
+Client behaviour
-~~~~~~~~~~~~~~~~
+----------------
-A call is set up with messages exchanged as follows:
+
+A call is set up with message events exchanged as follows:
 
 ::
 
@@ -41,28 +47,55 @@ Or a rejected call:
 
 Calls are negotiated according to the WebRTC specification.
 
-
 Glare
 ~~~~~
-This specification aims to address the problem of two users calling each other
+
-at roughly the same time and their invites crossing on the wire. It is a far
+"Glare" is a problem which occurs when two users call each other at roughly the
-better experience for the users if their calls are connected if it is clear
+same time. This results in the call failing to set up as there already is an
-that their intention is to set up a call with one another. In Matrix, calls are
+incoming/outgoing call. A glare resolution algorithm can be used to determine
-to rooms rather than users (even if those rooms may only contain one other user)
+which call to hangup and which call to answer. If both clients implement the
-so we consider calls which are to the same room. The rules for dealing with such
+same algorithm then they will both select the same call and the call will be
-a situation are as follows:
+successfully connected.
-
+
- - If an invite to a room is received whilst the client is preparing to send an
+
-   invite to the same room, the client should cancel its outgoing call and
+As calls are "placed" to rooms rather than users, the glare resolution algorithm
-   instead automatically accept the incoming call on behalf of the user.
+outlined below is only considered for calls which are to the same room. The
- - If an invite to a room is received after the client has sent an invite to
+algorithm is as follows:
-   the same room and is waiting for a response, the client should perform a
+
-   lexicographical comparison of the call IDs of the two calls and use the
+ - If an ``m.call.invite`` to a room is received whilst the client is
-   lesser of the two calls, aborting the greater. If the incoming call is the
+   **preparing to send** an ``m.call.invite`` to the same room:
-   lesser, the client should accept this call on behalf of the user.
+
+    * the client should cancel its outgoing call and instead
+      automatically accept the incoming call on behalf of the user.
+
+ - If an ``m.call.invite`` to a room is received **after the client has sent**
+   an ``m.call.invite`` to the same room and is waiting for a response:
+
+    * the client should perform a lexicographical comparison of the call IDs of
+      the two calls and use the *lesser* of the two calls, aborting the
+      greater. If the incoming call is the lesser, the client should accept
+      this call on behalf of the user.
+
 
 The call setup should appear seamless to the user as if they had simply placed
-a call and the other party had accepted. Thusly, any media stream that had been
+a call and the other party had accepted. This means any media stream that had been
 setup for use on a call should be transferred and used for the call that
 replaces it.
 
+Server behaviour
+----------------
+
+The homeserver MAY provide a TURN server which clients can use to contact the
+remote party. The following HTTP API endpoints will be used by clients in order
+to get information about the TURN server.
+
+{{voip_http_api}}
+
+
+Security considerations
+-----------------------
+
+Calls should only be placed to rooms with one other user in them. If they are
+placed to group chat rooms it is possible that another user will intercept and
+answer the call.
+

templating/matrix_templates/units.py

@@ -19,6 +19,7 @@ import yaml
 V1_CLIENT_API = "../api/client-server/v1"
 V1_EVENT_EXAMPLES = "../event-schemas/examples/v1"
 V1_EVENT_SCHEMA = "../event-schemas/schema/v1"
+V2_CLIENT_API = "../api/client-server/v2_alpha"
 CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema"
 CHANGELOG = "../CHANGELOG.rst"
 TARGETS = "../specification/targets.yaml"
@@ -49,8 +50,17 @@ def get_json_schema_object_fields(obj, enforce_title=False):
     }
     tables = [fields]
 
-    props = obj.get("properties", obj.get("patternProperties"))
     parents = obj.get("allOf")
+    props = obj.get("properties")
+    if not props:
+        props = obj.get("patternProperties")
+        if props:
+            # try to replace horrible regex key names with pretty x-pattern ones
+            for key_name in props.keys():
+                pretty_key = props[key_name].get("x-pattern")
+                if pretty_key:
+                    props[pretty_key] = props[key_name]
+                    del props[key_name]
     if not props and not parents:
         raise Exception(
             "Object %s has no properties or parents." % obj
@@ -70,10 +80,17 @@ def get_json_schema_object_fields(obj, enforce_title=False):
         if props[key_name]["type"] == "object":
             if props[key_name].get("additionalProperties"):
                 # not "really" an object, just a KV store
-                value_type = (
+                prop_val = props[key_name]["additionalProperties"]["type"]
-                    "{string: %s}" %
+                if prop_val == "object":
-                    props[key_name]["additionalProperties"]["type"]
+                    nested_object = get_json_schema_object_fields(
-                )
+                        props[key_name]["additionalProperties"],
+                        enforce_title=True
+                    )
+                    value_type = "{string: %s}" % nested_object[0]["title"]
+                    if not nested_object[0].get("no-table"):
+                        tables += nested_object
+                else:
+                    value_type = "{string: %s}" % prop_val
             else:
                 nested_object = get_json_schema_object_fields(
                     props[key_name], 
@@ -337,18 +354,27 @@ class MatrixUnits(Units):
         }
 
     def load_swagger_apis(self):
-        path = V1_CLIENT_API
+        paths = [
+            V1_CLIENT_API, V2_CLIENT_API
+        ]
         apis = {}
-        for filename in os.listdir(path):
+        for path in paths:
-            if not filename.endswith(".yaml"):
+            is_v2 = (path == V2_CLIENT_API)
+            if not os.path.exists(V2_CLIENT_API):
+                self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API)
                 continue
-            self.log("Reading swagger API: %s" % filename)
+            for filename in os.listdir(path):
-            with open(os.path.join(path, filename), "r") as f:
+                if not filename.endswith(".yaml"):
-                # strip .yaml
+                    continue
-                group_name = filename[:-5].replace("-", "_")
+                self.log("Reading swagger API: %s" % filename)
-                api = yaml.load(f.read())
+                with open(os.path.join(path, filename), "r") as f:
-                api["__meta"] = self._load_swagger_meta(api, group_name)
+                    # strip .yaml
-                apis[group_name] = api
+                    group_name = filename[:-5].replace("-", "_")
+                    if is_v2:
+                        group_name = "v2_" + group_name
+                    api = yaml.load(f.read())
+                    api["__meta"] = self._load_swagger_meta(api, group_name)
+                    apis[group_name] = api
         return apis
 
     def load_common_event_fields(self):