Merge remote-tracking branch 'origin/master' into erikj/search_actual

api/client-server/v1/create_room.yaml

@@ -0,0 +1,148 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Room Creation API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/createRoom":
+    post:
+      summary: Create a new room
+      description: |-
+        Create a new room with various configuration options.
+      security:
+        - accessToken: []
+      parameters:
+        - in: body
+          name: body
+          description: The desired room configuration.
+          schema:
+            type: object
+            example: |-
+              {
+                "preset": "public_chat",
+                "room_alias_name": "thepub",
+                "name": "The Grand Duke Pub",
+                "topic": "All about happy hour",
+                "creation_content": {
+                    "m.federate": false
+                }
+              }
+            properties:
+              visibility:
+                type: string
+                enum: ["public", "private"]
+                description: |-
+                  A ``public`` visibility indicates that the room will be shown
+                  in the published room list. A ``private`` visibility will hide
+                  the room from the published room list. Rooms default to
+                  ``private`` visibility if this key is not included. NB: This
+                  should not be confused with ``join_rules`` which also uses the
+                  word ``public``.
+              room_alias_name:
+                type: string
+                description: |-
+                  The desired room alias **local part**. If this is included, a
+                  room alias will be created and mapped to the newly created
+                  room. The alias will belong on the *same* home server which
+                  created the room. For example, if this was set to "foo" and
+                  sent to the homeserver "example.com" the complete room alias
+                  would be ``#foo:example.com``.
+              name:
+                type: string
+                description: |-
+                  If this is included, an ``m.room.name`` event will be sent
+                  into the room to indicate the name of the room. See Room
+                  Events for more information on ``m.room.name``.
+              topic:
+                type: string
+                description: |-
+                  If this is included, an ``m.room.topic`` event will be sent
+                  into the room to indicate the topic for the room. See Room
+                  Events for more information on ``m.room.topic``.
+              invite:
+                type: array
+                description: |-
+                  A list of user IDs to invite to the room. This will tell the
+                  server to invite everyone in the list to the newly created room.
+                items:
+                  type: string
+              creation_content:
+                title: CreationContent
+                type: object
+                description: |-
+                  Extra keys to be added to the content of the ``m.room.create``.
+                  The server will clober the following keys: ``creator``. Future
+                  versions of the specification may allow the server to clobber
+                  other keys.
+              initial_state:
+                type: array
+                description: |-
+                  A list of state events to set in the new room. This allows
+                  the user to override the default state events set in the new
+                  room. The expected format of the state events are an object
+                  with type, state_key and content keys set.
+                  Takes precedence over events set by ``presets``, but gets
+                  overriden by ``name`` and ``topic`` keys.
+                items:
+                  type: object
+                  title: StateEvent
+                  properties:
+                    type:
+                      type: string
+                    state_key:
+                      type: string
+                    content:
+                      type: string
+              preset:
+                type: string
+                enum: ["private_chat", "public_chat", "trusted_private_chat"]
+                description: |-
+                  Convenience parameter for setting various default state events
+                  based on a preset. Must be either:
+
+                  ``private_chat`` =>
+                    ``join_rules`` is set to ``invite``.
+                    ``history_visibility`` is set to ``shared``.
+
+                  ``trusted_private_chat`` =>
+                      ``join_rules`` is set to ``invite``.
+                      ``history_visibility`` is set to ``shared``.
+                      All invitees are given the same power level as the room creator.
+
+                  ``public_chat``: =>
+                      ``join_rules`` is set to ``public``.
+                      ``history_visibility`` is set to ``shared``.
+
+      responses:
+        200:
+          description: Information about the newly created room.
+          schema:
+            type: object
+            description: Information about the newly created room.
+            properties:
+              room_id:
+                type: string
+                description: |-
+                  The created room's ID.
+          examples:
+            application/json: |-
+              {
+                "room_id": "!sefiuhWgwghwWgh:example.com"
+              }
+        400:
+          description: >
+            The request body is malformed or the room alias specified is already taken.

api/client-server/v1/login.yaml

@@ -33,17 +33,21 @@ paths:
             type: object
             example: |-
               {
-                "username": "cheeky_monkey",
+                "type": "m.login.pasword",
+                "user": "cheeky_monkey",
                 "password": "ilovebananas"
               }
             properties:
-              username:
+              type:
+                type: string
+                description: The login type being used. Currently only "m.login.password" is supported.
+              user:
                 type: string
                 description: The fully qualified user ID or just local part of the user ID, to log in.
               password:
                 type: string
                 description: The user's password.
-            required: ["username", "password"]
+            required: ["type", "user", "password"]
       responses:
         200:
           description: The user has been authenticated.
@@ -78,6 +82,15 @@ paths:
               home_server:
                 type: string
                 description: The hostname of the Home Server on which the account has been registered.
+        400:
+          description: |-
+            Part of the request was invalid. For example, the login type may not be recognised.
+          examples:
+            application/json: |-
+              {
+                  "errcode": "M_UNKNOWN",
+                  "error": "Bad login type."
+              }
         403:
           description: |-
             The login attempt failed. For example, the password may have been incorrect.

api/client-server/v1/message_pagination.yaml

@@ -0,0 +1,131 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Rooms API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/rooms/{roomId}/messages":
+    get:
+      summary: Get a list of events for this room
+      description: |-
+        This API returns a list of message and state events for a room. It uses
+        pagination query parameters to paginate history in the room.
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room to get events from.
+          required: true
+          x-example: "!636q39766251:example.com"
+        - in: query
+          type: string
+          name: from
+          description: |-
+            The token to start returning events from. This token can be obtained
+            from the initial sync API.
+          required: true
+          x-example: "s345_678_333"
+        - in: query
+          type: string
+          enum: ["b", "f"]
+          name: dir
+          description: |-
+            The direction to return events from.
+          required: true
+          x-example: "b"
+        - in: query
+          type: integer
+          name: limit
+          description: |-
+            The maximum number of events to return. Default: 10.
+          x-example: "3"
+      responses:
+        200:
+          description: A list of messages with a new token to request more.
+          schema:
+            type: object
+            description: A list of messages with a new token to request more.
+            properties:
+              start:
+                type: string
+                description: |-
+                  The token to start paginating from. If ``dir=b`` this will be
+                  the token supplied in ``from``.
+              end:
+                type: string
+                description: |-
+                  The token the pagination ends at. If ``dir=b`` this token should
+                  be used again to request even earlier events.
+              chunk:
+                type: array
+                description: |-
+                  A list of room events.
+                items:
+                  type: object
+                  title: RoomEvent
+          examples:
+            application/json: |-
+              {
+                "start": "t47429-4392820_219380_26003_2265",
+                "end": "t47409-4357353_219380_26003_2265",
+                "chunk": [
+                  {
+                    "origin_server_ts": 1444812213737,
+                    "user_id": "@alice:example.com",
+                    "event_id": "$1444812213350496Caaaa:example.com",
+                    "content": {
+                      "body": "hello world",
+                      "msgtype":"m.text"
+                    },
+                    "room_id":"!Xq3620DUiqCaoxq:example.com",
+                    "type":"m.room.message",
+                    "age": 1042
+                  },
+                  {
+                    "origin_server_ts": 1444812194656 ,
+                    "user_id": "@bob:example.com",
+                    "event_id": "$1444812213350496Cbbbb:example.com",
+                    "content": {
+                      "body": "the world is big",
+                      "msgtype":"m.text"
+                    },
+                    "room_id":"!Xq3620DUiqCaoxq:example.com",
+                    "type":"m.room.message",
+                    "age": 20123
+                  },
+                  {
+                    "origin_server_ts": 1444812163990,
+                    "user_id": "@bob:example.com",
+                    "event_id": "$1444812213350496Ccccc:example.com",
+                    "content": {
+                      "name": "New room name"
+                    },
+                    "prev_content": {
+                      "name": "Old room name"
+                    },
+                    "state_key": "",
+                    "room_id":"!Xq3620DUiqCaoxq:example.com",
+                    "type":"m.room.name",
+                    "age": 50789
+                  }
+                ]
+              }
+        403:
+          description: >
+            You aren't a member of the room.

api/client-server/v1/sync.yaml

@@ -30,7 +30,9 @@ paths:
         - in: query
           type: string
           name: from
-          description: The token to stream from.
+          description: |-
+            The token to stream from. This token is either from a previous
+            request to this API or from the initial sync API.
           required: false
           x-example: "s3456_9_0"
         - in: query
@@ -39,16 +41,6 @@ paths:
           description: The maximum time in milliseconds to wait for an event.
           required: false
           x-example: "35000"
-        - in: query
-          type: string
-          name: archived
-          description: |-
-            Whether to include rooms that the user has left. If absent then
-            only rooms that the user has been invited to or has joined are
-            included. If set to "true" then rooms that the user has left are
-            included as well.
-          required: false
-          x-example: "true"
       responses:
         200:
           description: "The events received, which may be none."
@@ -78,19 +70,19 @@ paths:
               start:
                 type: string
                 description: |-
-                  A token which correlates to the first value in ``chunk``. Used
-                  for pagination.
+                  A token which correlates to the first value in ``chunk``. This
+                  is usually the same token supplied to ``from=``.
               end:
                 type: string
                 description: |-
-                  A token which correlates to the last value in ``chunk``. Used
-                  for pagination.
+                  A token which correlates to the last value in ``chunk``. This
+                  token should be used in the next request to ``/events``.
               chunk:
                 type: array
                 description: "An array of events."
                 items:
                   type: object
-                  title: RoomEvent
+                  title: Event
                   allOf:
                    - "$ref": "core-event-schema/room_event.json"
         400:
@@ -110,6 +102,16 @@ paths:
           description: The maximum number of messages to return for each room.
           required: false
           x-example: "2"
+        - in: query
+          type: boolean
+          name: archived
+          description: |-
+            Whether to include rooms that the user has left. If ``false`` then
+            only rooms that the user has been invited to or has joined are
+            included. If set to ``true`` then rooms that the user has left are
+            included as well. By default this is ``false``.
+          required: false
+          x-example: "true"
       responses:
         200:
           description: The user's current state.

api/client-server/v2_alpha/registration.yaml

@@ -0,0 +1,101 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v2 Registration API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v2_alpha
+consumes:
+  - application/json
+produces:
+  - application/json
+paths:
+  "/register":
+    post:
+      summary: Register for an account on this homeserver.
+      description: |-
+        Register for an account on this homeserver.
+      parameters:
+        - in: body
+          name: body
+          schema:
+            type: object
+            example: |-
+              {
+                "username": "cheeky_monkey",
+                "password": "ilovebananas",
+                "bind_email": false
+              }
+            properties:
+              bind_email:
+                type: boolean
+                description: |-
+                  If true, the server binds the email used for authentication to
+                  the Matrix ID with the ID Server.
+              username:
+                type: string
+                description: |-
+                  The local part of the desired Matrix ID. If omitted,
+                  the homeserver MUST generate a Matrix ID local part.
+              password:
+                type: string
+                description: The desired password for the account.
+            required: ["password"]
+      responses:
+        200:
+          description: The account has been registered.
+          examples:
+            application/json: |-
+              {
+                "user_id": "@cheeky_monkey:matrix.org",
+                "access_token": "abc123",
+                "home_server": "matrix.org",
+                "refresh_token": "def456"
+              }
+          schema:
+            type: object
+            properties:
+              user_id:
+                type: string
+                description: The fully-qualified Matrix ID that has been registered.
+              access_token:
+                type: string
+                description: |-
+                  An access token for the account.
+                  This access token can then be used to authorize other requests.
+                  The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``.
+                  There is no specific error message to indicate that a request has failed because
+                  an access token has expired; instead, if a client has reason to believe its
+                  access token is valid, and it receives an auth error, they should attempt to
+                  refresh for a new token on failure, and retry the request with the new token.
+              refresh_token:
+                type: string
+                # TODO: Work out how to linkify /tokenrefresh
+                description: |-
+                  (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint.
+              home_server:
+                type: string
+                description: The hostname of the Home Server on which the account has been registered.
+        400:
+          description: |-
+            Part of the request was invalid. This may include one of the following error codes:
+
+              * ``M_USER_IN_USE`` : The desired user ID is already taken.
+              * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace
+                claimed by an application service.
+
+            These errors may be returned at any stage of the registration process,
+            including after authentication if the requested user ID was registered
+            whilst the client was performing authentication.
+          examples:
+            application/json: |-
+              {
+                  "errcode": "M_USER_IN_USE",
+                  "error": "Desired user ID is already taken."
+              }
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"

event-schemas/check.sh

@@ -1,5 +1,11 @@
 #!/bin/bash -e
 # Runs z-schema over all of the schema files (looking for matching examples)
+
+if ! which z-schema; then
+  echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema"
+  exit 1
+fi
+
 find schema/v1/m.* | while read line
 do
     split_path=(${line///// })

event-schemas/examples/v1/m.room.avatar

@@ -0,0 +1,18 @@
+{
+  "age": 242352,
+  "content": {
+    "info": {
+        "h": 398,
+        "w": 394,
+        "mimetype": "image/jpeg",
+        "size": 31037
+    },
+    "url": "mxc://localhost/JWEIFJgwEIhweiWJE"
+  },
+  "origin_server_ts": 1431961217939,
+  "event_id": "$WLGTSEFSEF:localhost",
+  "type": "m.room.avatar",
+  "state_key": "",
+  "room_id": "!Cuyf34gef24t:localhost",
+  "user_id": "@example:localhost"
+}

event-schemas/schema/v1/core-event-schema/state_event.json

@@ -8,7 +8,7 @@
   "properties": {
     "state_key": {
       "type": "string",
-      "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event."
+      "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'."
     },
     "prev_content": {
       "type": "object",

event-schemas/schema/v1/m.room.avatar

@@ -0,0 +1,64 @@
+{
+    "title": "RoomAvatar",
+    "description": "A picture that is associated with the room. This can be displayed alongside the room information.",
+    "type": "object",
+    "allOf": [{
+        "$ref": "core-event-schema/state_event.json"
+    }],
+    "properties": {
+        "content": {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The URL to the image."
+                },
+                "thumbnail_url": {
+                    "type": "string",
+                    "description": "The URL to the thumbnail of the image."
+                },
+                "thumbnail_info": {
+                    "type": "object",
+                    "title": "ImageInfo",
+                    "description": "Metadata about the image referred to in ``thumbnail_url``.",
+                    "allOf": [{
+                        "$ref": "core-event-schema/msgtype_infos/image_info.json"
+                    }]
+                },
+                "info": {
+                    "type": "object",
+                    "title": "ImageInfo",
+                    "description": "Metadata about the image referred to in ``url``.",
+                    "properties": {
+                        "size": {
+                            "type": "integer",
+                            "description": "Size of the image in bytes."
+                        },
+                        "w": {
+                            "type": "integer",
+                            "description": "The width of the image in pixels."
+                        },
+                        "h": {
+                            "type": "integer",
+                            "description": "The height of the image in pixels."
+                        },
+                        "mimetype": {
+                            "type": "string",
+                            "description": "The mimetype of the image, e.g. ``image/jpeg``."
+                        }
+                    }
+                }
+            },
+            "required": ["url"]
+        },
+        "state_key": {
+            "type": "string",
+            "description": "A zero-length string.",
+            "pattern": "^$"
+        },
+        "type": {
+            "type": "string",
+            "enum": ["m.room.avatar"]
+        }
+    }
+}

event-schemas/schema/v1/m.room.message

@@ -1,7 +1,7 @@
 {
     "type": "object",
     "title": "Message",
-    "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.",
+    "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message. This allows clients to display *something* even if it is just plain text.",
     "allOf": [{
         "$ref": "core-event-schema/room_event.json"
     }],

event-schemas/schema/v1/m.room.message.feedback

@@ -1,7 +1,7 @@
 {
     "type": "object",
     "title": "MessageFeedback",
-    "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.",
+    "description": "**NB: Usage of this event is discouraged in favour of the** `receipts module`_. **Most clients will not recognise this event.** Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.",
     "allOf": [{
         "$ref": "core-event-schema/room_event.json"
     }],

event-schemas/schema/v1/m.room.name

@@ -11,7 +11,7 @@
             "properties": {
                 "name": {
                     "type": "string",
-                    "description": "The name of the room."
+                    "description": "The name of the room. This MUST NOT exceed 255 bytes."
                 }
             },
             "required": ["name"]

scripts/gendoc.py

@@ -128,11 +128,8 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
                 )
             else:
                 rst = f.read()
-            if rst[-2:] != "\n\n":
-                raise Exception(
-                    ("File %s should end with TWO new-line characters to ensure " +
-                    "file concatenation works correctly.") % (file_info,)
-                )
+
+            rst += "\n\n"
             return rst
     # dicts look like {0: filepath, 1: filepath} where the key is the title level
     elif isinstance(file_info, dict):

scripts/speculator/README

@@ -8,3 +8,7 @@ It serves the following HTTP endpoints:
 
 To run it, you must install the `go` tool, and run:
  `go run main.go`
+
+To build the binary (which is necessary for deployment to the matrix.org
+servers), you must again install `go`, and then run:
+ `go build`

scripts/speculator/main.go

@@ -364,6 +364,7 @@ func main() {
 		"illicitonion":  true,
 		"Kegsay":        true,
 		"NegativeMjark": true,
+		"richvdh":       true,
 	}
 	rand.Seed(time.Now().Unix())
 	masterCloneDir, err := gitClone(matrixDocCloneURL, false)

specification/appendices.rst

@@ -24,7 +24,7 @@ Threat: Unrecoverable Consistency Violations
 ++++++++++++++++++++++++++++++++++++++++++++
 
 An attacker could send messages which created an unrecoverable "split-brain"
-state in the cluster such that the victim's servers could no longer dervive a
+state in the cluster such that the victim's servers could no longer derive a
 consistent view of the chatroom state.
 
 Threat: Bad History
@@ -63,7 +63,7 @@ Spoofing
 An attacker could try to send a message claiming to be from the victim without
 the victim having sent the message in order to:
 
-* Impersonate the victim while performing illict activity.
+* Impersonate the victim while performing illicit activity.
 * Obtain privileges of the victim.
 
 Threat: Altering Message Contents
@@ -81,7 +81,7 @@ with a phony "origin" field.
 Spamming
 ~~~~~~~~
 
-The attacker could try to send a high volume of solicicted or unsolicted
+The attacker could try to send a high volume of solicited or unsolicited
 messages to the victim in order to:
 
 * Find victims for scams.

specification/application_service_api.rst

@@ -54,6 +54,11 @@ An example HS configuration required to pass traffic to the AS is:
       aliases: []  # Namespaces of room aliases which should be delegated to the AS
       rooms: [] # Namespaces of room ids which should be delegated to the AS
 
+.. WARNING::
+  If the homeserver in question has multiple application services, each
+  ``as_token`` MUST be unique per application service as this token is used to
+  identify the application service. The homeserver MUST enforce this.
+
 - An application service can state whether they should be the only ones who 
   can manage a specified namespace. This is referred to as an "exclusive" 
   namespace. An exclusive namespace prevents humans and other application 
@@ -94,7 +99,7 @@ API called when:
  - HS receives an event for an unknown user ID in the AS's namespace, e.g. an
    invite event to a room.
 Notes:
- - When the AS receives this request, if the user exists, it must create the user via
+ - When the AS receives this request, if the user exists, it must `create the user`_ via
    the CS API.
  - It can also set arbitrary information about the user (e.g. display name, join rooms, etc)
    using the CS API.
@@ -226,6 +231,9 @@ Ordering notes:
     ]
   }
 
+
+.. _create the user: `sect:asapi-permissions`_
+
 Client-Server v2 API Extensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -244,11 +252,8 @@ additional permissions granting the AS permission to masquerade as a matrix user
 
 Inputs:
  - Application service token (``access_token``)
+ - User ID in the AS namespace to act as.
 
- Either:
-   - User ID in the AS namespace to act as.
- Or:
-   - OAuth2 token of real user (which may end up being an access token) 
 Notes:
  - This will apply on all aspects of the CS API, except for Account Management.
  - The ``as_token`` is inserted into ``access_token`` which is usually where the
@@ -263,12 +268,6 @@ Notes:
    access_token: The application service token
    user_id: The desired user ID to act as.
    
- /path?access_token=$token&user_token=$token
-
- Query Parameters:
-   access_token: The application service token
-   user_token: The token granted to the AS by the real user
-
 Timestamp massaging
 +++++++++++++++++++
 The application service may want to inject events at a certain time (reflecting
@@ -291,6 +290,9 @@ Notes:
 
 Server admin style permissions
 ++++++++++++++++++++++++++++++
+
+.. _sect:asapi-permissions:
+
 The home server needs to give the application service *full control* over its
 namespace, both for users and for room aliases. This means that the AS should
 be able to create/edit/delete any room alias in its namespace, as well as
@@ -317,7 +319,7 @@ including the AS token on a ``/register`` request, along with a login type of
 
 Application services which attempt to create users or aliases *outside* of
 their defined namespaces will receive an error code ``M_EXCLUSIVE``. Similarly,
-normal users who attempt to create users or alises *inside* an application
+normal users who attempt to create users or aliases *inside* an application
 service-defined namespace will receive the same ``M_EXCLUSIVE`` error code,
 but only if the application service has defined the namespace as ``exclusive``.
 
@@ -370,9 +372,10 @@ an API is exposed.
 Room Aliases
 ++++++++++++
 We may want to expose some 3P network rooms so Matrix users can join them directly,
-e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. mailto,
-tel. Rooms which are publicly accessible (e.g. IRC rooms) can be exposed as an alias by
-the application service. Private rooms (e.g. sending an email to someone) should not
+e.g. IRC rooms. We don't want to expose every 3P network room though, e.g.
+``mailto``, ``tel``. Rooms which are publicly accessible (e.g. IRC rooms) can be
+exposed as an alias by the application service. Private rooms
+(e.g. sending an email to someone) should not
 be exposed in this way, but should instead operate using normal invite/join semantics.
 Therefore, the ID conventions discussed below are only valid for public rooms which 
 expose room aliases.
@@ -392,9 +395,9 @@ SHOULD be mapped in the same way as "user" URIs.
   
 Event fields
 ++++++++++++
-We recommend that any gatewayed events should include an ``external_url`` field
-in their content to provide a way for Matrix clients to link into the 'native'
-client from which the event originated.  For instance, this could contain the
-message-ID for emails/nntp posts, or a link to a blog comment when gatewaying
-blog comment traffic in & out of matrix
+We recommend that any events that originated from a remote network should
+include an ``external_url`` field in their content to provide a way for Matrix
+clients to link into the 'native' client from which the event originated.
+For instance, this could contain the message-ID for emails/nntp posts, or a link
+to a blog comment when bridging blog comment traffic in & out of Matrix.
 

specification/client_server_api.rst

@@ -1,9 +1,6 @@
 Client-Server API
 =================
 
-Overview
---------
-
 The client-server API provides a simple lightweight API to let clients send
 messages, control rooms and synchronise conversation history. It is designed to
 support both lightweight clients which store no state and lazy-load data from
@@ -31,7 +28,10 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or
 ``M_UNKNOWN_TOKEN`` respectively.
 
 User-Interactive Authentication API
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _sect:auth-api:
+
 This section refers to API Version 2.
 
 Some API endpoints such as ``login`` or ``register`` require authentication that
@@ -159,7 +159,7 @@ absence of that login stage type in the 'completed' array indicating whether
 that stage is complete.
 
 Example
-~~~~~~~
++++++++
 At a high level, the requests made for an API call completing an auth flow with
 three stages will resemble the following diagram::
   
@@ -201,7 +201,7 @@ This specification defines the following login types:
  - ``m.login.dummy``
 
 Password-based
-~~~~~~~~~~~~~~
+++++++++++++++
 :Type:
   ``m.login.password``
 :Description:
@@ -215,8 +215,14 @@ To respond to this type, reply with an auth dict as follows::
     "password": "<password>"
   }
 
+.. WARNING::
+  Clients SHOULD enforce that the password provided is suitably complex. The
+  password SHOULD include a lower-case letter, an upper-case letter, a number
+  and a symbol and be at a minimum 8 characters in length. Servers MAY reject
+  weak passwords with an error code ``M_WEAK_PASSWORD``.
+
 Google ReCaptcha
-~~~~~~~~~~~~~~~~
+++++++++++++++++
 :Type:
   ``m.login.recaptcha``
 :Description:
@@ -230,7 +236,7 @@ To respond to this type, reply with an auth dict as follows::
   }
 
 Token-based
-~~~~~~~~~~~
++++++++++++
 :Type:
   ``m.login.token``
 :Description:
@@ -261,7 +267,7 @@ newly provisioned access_token).
 The ``token`` must be a macaroon.
 
 OAuth2-based
-~~~~~~~~~~~~
+++++++++++++
 :Type:
   ``m.login.oauth2``
 :Description:
@@ -285,7 +291,7 @@ the OAuth flow has completed, the client retries the request with the session
 only, as above.
 
 Email-based (identity server)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
++++++++++++++++++++++++++++++
 :Type:
   ``m.login.email.identity``
 :Description:
@@ -310,7 +316,7 @@ To respond to this type, reply with an auth dict as follows::
   }
 
 Dummy Auth
-~~~~~~~~~~
+++++++++++
 :Type:
   ``m.login.dummy``
 :Description:
@@ -327,7 +333,7 @@ if provided::
 
 
 Fallback
-~~~~~~~~
+++++++++
 Clients cannot be expected to be able to know how to process every single login
 type. If a client does not know how to handle a given login type, it can direct
 the user to a web browser with the URL of a fallback page which will allow the
@@ -343,16 +349,169 @@ This MUST return an HTML page which can perform this authentication stage. This
 page must attempt to call the JavaScript function ``window.onAuthDone`` when
 the authentication has been completed.
 
+Registration
+~~~~~~~~~~~~
+This section refers to API Version 2. These API calls currently use the prefix
+``/_matrix/client/v2_alpha``.
+
+Registering for a user account is done using the request::
+
+  POST $V2PREFIX/register
+
+This API endpoint uses the `User-Interactive Authentication API`_.
+This API endpoint does not require an access token.
+
+.. _User-Interactive Authentication API: `sect:auth-api`_
+
+The body of the POST request is a JSON object containing:
+
+username
+  Optional. This is the local part of the desired Matrix ID. If omitted, the
+  Home Server must generate a Matrix ID local part.
+password
+  Required. The desired password for the account.
+bind_email
+  Optional. If ``true``, the server binds the email used for authentication to
+  the Matrix ID with the ID Server.
+
+On success, this returns a JSON object with keys:
+
+user_id
+  The fully-qualified Matrix ID that has been registered.
+access_token
+  An access token for the new account.
+home_server
+  The hostname of the Home Server on which the account has been registered.
+refresh_token
+  A token that may be exchanged for a new ``access_token`` using the
+  ``/tokenrefresh`` API endpoint.
+
+This endpoint may also return the following error codes:
+
+M_USER_IN_USE
+  If the Matrix ID is already in use
+M_EXCLUSIVE
+  If the requested Matrix ID is in the exclusive namespace of an application
+  service.
+
+Home Servers MUST perform the relevant checks and return these codes before
+performing `User-Interactive Authentication`_, although they may also return
+them after authentication is completed if, for example, the requested user ID
+was registered whilst the client was performing authentication.
+
+.. _User-Interactive Authentication: `sect:auth-api`_
+
+Old V1 API docs: |register|_
+
+{{login_http_api}}
+
+Changing Password
++++++++++++++++++
+This section refers to API Version 2. These API calls currently use the prefix
+``/_matrix/client/v2_alpha``.
+
+Request::
+
+  POST $V2PREFIX/account/password
+
+This API endpoint uses the User-Interactive Authentication API. An access token
+should be submitted to this endpoint if the client has an active session. The
+Home Server may change the flows available depending on whether a valid access
+token is provided.
+
+The body of the POST request is a JSON object containing:
+
+new_password
+  The new password for the account.
+
+On success, an empty JSON object is returned.
+
+The error code M_NOT_FOUND is returned if the user authenticated with a third
+party identifier but the Home Server could not find a matching account in its
+database.
+
+Adding Account Administrative Contact Information
++++++++++++++++++++++++++++++++++++++++++++++++++
+This section refers to API Version 2. These API calls currently use the prefix
+``/_matrix/client/v2_alpha``.
+
+Request::
+
+  POST $V2PREFIX/account/3pid
+
+Used to add contact information to the user's account.
+
+The body of the POST request is a JSON object containing:
+
+threePidCreds
+  An object containing contact information.
+bind
+  Optional. A boolean indicating whether the Home Server should also bind this
+  third party identifier to the account's matrix ID with the Identity Server. If
+  supplied and true, the Home Server must bind the 3pid accordingly.
+
+The contact information object comprises:
+
+id_server
+  The colon-separated hostname and port of the Identity Server used to
+  authenticate the third party identifier. If the port is the default, it and the
+  colon should be omitted.
+sid
+  The session ID given by the Identity Server
+client_secret
+  The client secret used in the session with the Identity Server.
+
+On success, the empty JSON object is returned.
+
+May also return error codes:
+
+M_THREEPID_AUTH_FAILED
+  If the credentials provided could not be verified with the ID Server.
+
+Fetching Currently Associated Contact Information
++++++++++++++++++++++++++++++++++++++++++++++++++
+This section refers to API Version 2. These API calls currently use the prefix
+``/_matrix/client/v2_alpha``.
+
+Request::
+
+  GET $V2PREFIX/account/3pid
+
+This returns a list of third party identifiers that the Home Server has
+associated with the user's account. This is *not* the same as the list of third
+party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers
+in this list may be used by the Home Server as, for example, identifiers that it
+will accept to reset the user's account password.
+
+Returns a JSON object with the key ``threepids`` whose contents is an array of
+objects with the following keys:
+
+medium
+  The medium of the 3pid (eg, ``email``)
+address
+  The textual address of the 3pid, eg. the email address
+
 Pagination
 ----------
 
-Querying large datasets in Matrix always uses the same pagination API pattern to
+.. NOTE::
+  The paths referred to in this section are not actual endpoints. They only
+  serve as examples to explain how pagination functions.
+
+Pagination is the process of dividing a dataset into multiple discrete pages.
+Matrix makes use of pagination to allow clients to view extremely large datasets.
+These datasets are not limited to events in a room (for example clients may want
+to paginate a list of rooms in addition to events within those rooms). Regardless
+of *what* is being paginated, there is a common underlying API which is used to
 to give clients a consistent way of selecting subsets of a potentially changing
-dataset. Requests pass in ``from``, ``to`` and ``limit`` parameters which describe
-where to read from the stream. ``from`` and ``to`` are opaque textual 'stream
-tokens' which describe positions in the dataset. The response returns new
-``start`` and ``end`` stream token values which can then be passed to subsequent
-requests to continue pagination.
+dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters
+which describe where to read from the stream. ``from`` and ``to`` are opaque
+textual 'stream tokens' which describe the current position in the dataset.
+The ``dir`` parameter is an enum representing the direction of events to return:
+either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and
+``end`` stream token values which can then be passed to subsequent requests to
+continue pagination. Not all endpoints will make use of all the parameters
+outlined here: see the specific endpoint in question for more information.
 
 Pagination Request Query Parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -367,24 +526,26 @@ Query parameters:
   limit:
     integer - An integer representing the maximum number of items to 
     return.
+  dir:
+    f|b - The direction to return events in. Typically this is ``b`` to paginate
+    backwards in time.
 
 'START' and 'END' are placeholder values used in these examples to describe the
 start and end of the dataset respectively.
 
-Unless specified, the default pagination parameters are from=START, to=END, 
-without a limit set. This allows you to hit an API like
-/events without any query parameters to get everything.
+Unless specified, the default pagination parameters are ``from=START``,
+``to=END``, without a limit set.
 
-For example, the event stream has events E1 -> E15. The client wants the last 5 
+For example, if an endpoint had events E1 -> E15. The client wants the last 5 
 events and doesn't know any previous events::
 
     S                                                    E
     |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-|
     |                               |                    |
-    |                          _____|                    |
-    |__________________       |       ___________________|
-                       |      |      |
-     GET /events?to=START&limit=5&from=END
+    |                          _____|  <--backwards--    |
+    |__________________       |         |        ________|
+                       |      |         |        |
+     GET /somepath?to=START&limit=5&dir=b&from=END
      Returns:
        E15,E14,E13,E12,E11
 
@@ -401,7 +562,7 @@ now show page 3 (rooms R11 -> 15)::
                         Currently            |
                         viewing              |
                                              |
-                             GET /rooms/list?from=9&to=END&limit=5
+                             GET /roomslist?from=9&to=END&limit=5
                              Returns: R11,R12,R13,R14,R15
                          
 Note that tokens are treated in an *exclusive*, not inclusive, manner. The end 
@@ -429,9 +590,6 @@ Events
 
 .. _sect:events:
 
-Overview
-~~~~~~~~
-
 The model of conversation history exposed by the client-server API can be
 considered as a list of events. The server 'linearises' the
 eventually-consistent event graph of events into an 'event stream' at any given
@@ -459,7 +617,7 @@ You can visualise the range of events being returned as::
                               |                             |
                         start: '1-2-3'                end: 'a-b-c'
                              
-Now, to receive future events in real-time on the eventstream, you simply GET
+Now, to receive future events in real-time on the event stream, you simply GET
 $PREFIX/events with a ``from`` parameter of 'a-b-c': in other words passing in the
 ``end`` token returned by initial sync. The request blocks until new events are
 available or until your specified timeout elapses, and then returns a
@@ -489,34 +647,36 @@ To continue paginating backwards, one calls the /messages API again, supplying
 the new ``start`` value as the ``from`` parameter.
 
 
-Receiving live updates on a client
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Syncing
+~~~~~~~
 
-Clients receive new events by long-polling the home server via the
-$PREFIX/events API, specifying a timeout in milliseconds in the timeout
-parameter. This will hold open the HTTP connection for a short period of time
-waiting for new events, returning early if an event occurs. This is called the
-`Event Stream`_. All events which are visible to the client will appear in the
-event stream. When the request returns, an ``end`` token is included in the
+Clients receive new events by "long-polling" the home server via the events API.
+This involves specifying a timeout in the request which will hold
+open the HTTP connection for a short period of time waiting for new events,
+returning early if an event occurs. Only the events API supports long-polling.
+All events which are visible to the client will appear in the
+events API. When the request returns, an ``end`` token is included in the
 response. This token can be used in the next request to continue where the
-last request left off.
-
-All events must be de-duplicated based on their event ID.
+last request left off. Multiple events can be returned per long-poll.
 
-.. TODO
-  is deduplication actually a hard requirement in CS v2?
+.. Warning::
+  Events are ordered in this API according to the arrival time of the event on
+  the homeserver. This can conflict with other APIs which order events based on
+  their partial ordering in the event graph. This can result in duplicate events
+  being received (once per distinct API called). Clients SHOULD de-duplicate
+  events based on the event ID when this happens.
 
 .. TODO-spec
-  Do we ever return multiple events in a single request?
-  Don't we get lots of request setup RTT latency if we only do one event per request?
   Do we ever support streaming requests? Why not websockets?
 
 When the client first logs in, they will need to initially synchronise with
-their home server. This is achieved via the |initialSync|_ API. This API also
-returns an ``end`` token which can be used with the event stream.  See the 'Room Sync' section below.
+their home server. This is achieved via the initial sync API described below.
+This API also returns an ``end`` token which can be used with the event stream.
 
-Events in a room
-~~~~~~~~~~~~~~~~
+{{sync_http_api}}
+
+Types of room events
+~~~~~~~~~~~~~~~~~~~~
 
 Room events are split into two categories:
 
@@ -528,7 +688,7 @@ Room events are split into two categories:
 :Message events:
   These are events which describe transient "once-off" activity in a room:
   typically communication such as sending an instant message or setting up a
-  VoIP call. These used to be called 'non-state' events.
+  VoIP call.
 
 This specification outlines several events, all with the event type prefix
 ``m.``. However, applications may wish to add their own type of event, and this
@@ -538,7 +698,7 @@ convention, e.g. ``com.example.myapp.event``.  This ensures event types are
 suitably namespaced for each application and reduces the risk of clashes.
 
 State events
-~~~~~~~~~~~~
+++++++++++++
 
 State events can be sent by ``PUT`` ing to
 |/rooms/<room_id>/state/<event_type>/<state_key>|_.  These events will be
@@ -581,7 +741,7 @@ In some cases, there may be no need for a ``state_key``, so it can be omitted::
 See `Room Events`_ for the ``m.`` event specification.
 
 Message events
-~~~~~~~~~~~~~~
+++++++++++++++
 
 Message events can be sent by sending a request to
 |/rooms/<room_id>/send/<event_type>|_.  These requests *can* use transaction
@@ -597,69 +757,6 @@ example::
 
 See `Room Events`_ for the ``m.`` event specification.
 
-Syncing rooms
-~~~~~~~~~~~~~
-
-.. NOTE::
-  This section is a work in progress.
-
-When a client logs in, they may have a list of rooms which they have already
-joined. These rooms may also have a list of events associated with them. The
-purpose of 'syncing' is to present the current room and event information in a
-convenient, compact manner. The events returned are not limited to room events;
-presence events will also be returned. A single syncing API is provided:
-
- - |initialSync|_ : A global sync which will present room and event information
-   for all rooms the user has joined.
-
-.. TODO-spec room-scoped initial sync
- - |/rooms/<room_id>/initialSync|_ : A sync scoped to a single room. Presents
-   room and event information for this room only.
- - Room-scoped initial sync is Very Tricky because typically people would
-   want to sync the room then listen for any new content from that point
-   onwards. The event stream cannot do this for a single room currently.
-   As a result, commenting room-scoped initial sync at this time.
-
-The |initialSync|_ API contains the following keys:
-
-``presence``
-  Description:
-    Contains a list of presence information for users the client is interested
-    in.
-  Format:
-    A JSON array of ``m.presence`` events.
-
-``end``
-  Description:
-    Contains an event stream token which can be used with the `Event Stream`_.
-  Format:
-    A string containing the event stream token.
-
-``rooms``
-  Description:
-    Contains a list of room information for all rooms the client has joined,
-    and limited room information on rooms the client has been invited to.
-  Format:
-    A JSON array containing Room Information JSON objects.
-
-Room Information:
-  Description:
-    Contains all state events for the room, along with a limited amount of
-    the most recent events, configured via the ``limit`` query
-    parameter. Also contains additional keys with room metadata, such as the
-    ``room_id`` and the client's ``membership`` to the room.
-  Format:
-    A JSON object with the following keys:
-      ``room_id``
-        A string containing the ID of the room being described.
-      ``membership``
-        A string representing the client's membership status in this room.
-      ``messages``
-        An event stream JSON object containing a ``chunk`` of recent
-        events (both state events and non-state events), along with an ``end`` token.
-      ``state``
-        A JSON array containing all the current state events for this room.
-
 Getting events for a room
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -667,6 +764,9 @@ There are several APIs provided to ``GET`` events for a room:
 
 {{rooms_http_api}}
 
+
+{{message_pagination_http_api}}
+
 Redactions
 ~~~~~~~~~~
 Since events are extensible it is possible for malicious users and/or servers
@@ -717,140 +817,20 @@ Rooms
 
 Creation
 ~~~~~~~~
-To create a room, a client has to use the |createRoom|_ API. There are various
-options which can be set when creating a room:
-
-``visibility``
-  Type:
-    String
-  Optional:
-    Yes
-  Value:
-    Either ``public`` or ``private``.
-  Description:
-    A ``public`` visibility indicates that the room will be shown in the public
-    room list. A ``private`` visibility will hide the room from the public room
-    list. Rooms default to ``private`` visibility if this key is not included.
-
-``room_alias_name``
-  Type:
-    String
-  Optional:
-    Yes
-  Value:
-    The room alias localpart.
-  Description:
-    If this is included, a room alias will be created and mapped to the newly
-    created room.  The alias will belong on the same home server which created
-    the room, e.g.  ``!qadnasoi:domain.com >>> #room_alias_name:domain.com``
-
-``name``
-  Type:
-    String
-  Optional:
-    Yes
-  Value:
-    The ``name`` value for the ``m.room.name`` state event.
-  Description:
-    If this is included, an ``m.room.name`` event will be sent into the room to
-    indicate the name of the room. See `Room Events`_ for more information on
-    ``m.room.name``.
-
-``topic``
-  Type:
-    String
-  Optional:
-    Yes
-  Value:
-    The ``topic`` value for the ``m.room.topic`` state event.
-  Description:
-    If this is included, an ``m.room.topic`` event will be sent into the room
-    to indicate the topic for the room. See `Room Events`_ for more information
-    on ``m.room.topic``.
-
-``invite``
-  Type:
-    List
-  Optional:
-    Yes
-  Value:
-    A list of user ids to invite.
-  Description:
-    This will tell the server to invite everyone in the list to the newly
-    created room.
-
-``creation_content``
-  Type:
-    Object
-  Optional:
-    Yes
-  Value:
-    Extra keys to be added to the content of the ``m.room.create``. The server
-    will clober the following keys: ``creator``. Future versions of this
-    spec may allow the server to clobber other keys if required.
-  Description:
-    Allows clients to add keys to the content of ``m.room.create``.
-
-``preset``
-  Type:
-    String
-  Optional:
-    Yes
-  Value:
-    ``private_chat``, ``trusted_private_chat`` or ``public_chat``
-  Description:
-    Convenience parameter for setting various default state events based on a
-    preset.
-
-    Three presets are defined:
-
-    - ``private_chat``: Sets the ``join_rules`` to ``invite`` and
-      ``history_visibility`` to ``shared``
-    - ``trusted_private_chat``: Set the ``join_rules`` to ``invite``,
-      ``history_visibility`` to ``shared`` and gives all invitees the same
-      power level as the creator.
-    - ``public_chat``: Sets the ``join_rules`` to ``public`` and
-      ``history_visibility`` to ``shared``
-
-``initial_state``
-  Type:
-    List
-  Optional:
-    Yes
-  Value:
-    A list of state events to set in the new room.
-  Description:
-    Allows the user to override the default state events set in the new room.
-
-    The expected format of the state events are an object with ``type``,
-    ``state_key`` and ``content`` keys set.
-
-    Takes precedence over events set by ``presets``, but gets overriden by
-    ``name`` and ``topic`` keys.
-
-Example::
-
-  {
-    "preset": "public_chat",
-    "room_alias_name": "thepub",
-    "name": "The Grand Duke Pub",
-    "topic": "All about happy hour",
-    "creation_content": {
-        "m.federate": false
-    }
-  }
-
-The home server will create a ``m.room.create`` event when the room is created,
-which serves as the root of the PDU graph for this room. This event also has a
+The home server will create an ``m.room.create`` event when a room is created,
+which serves as the root of the event graph for this room. This event also has a
 ``creator`` key which contains the user ID of the room creator. It will also
 generate several other events in order to manage permissions in this room. This
 includes:
 
  - ``m.room.power_levels`` : Sets the power levels of users and required power
-    levels.
+   levels for various actions within the room such as sending events.
  - ``m.room.join_rules`` : Whether the room is "invite-only" or not.
 
-See `Room Events`_ for more information on these events.
+See `Room Events`_ for more information on these events. To create a room, a
+client has to use the the following API.
+
+{{create_room_http_api}}
 
 Room aliases
 ~~~~~~~~~~~~
@@ -915,7 +895,7 @@ certain operations such as kicking, banning and sending state events. See
 `m.room.power_levels`_ for more information.
 
 Joining rooms
--------------
+~~~~~~~~~~~~~
 Users need to be a member of a room in order to send and receive events in that
 room. There are several states in which a user may be, in relation to a room:
 
@@ -994,53 +974,17 @@ member's state, by making a request to
     "membership": "ban"
   }
 
+Account operations
+------------------
 
 Registration
-------------
-This section refers to API Version 2. These API calls currently use the prefix
-``/_matrix/client/v2_alpha``.
-
-Registering for a user account is done using the request::
-
-  POST $V2PREFIX/register
-
-This API endpoint uses the User-Interactive Authentication API.
-This API endpoint does not require an access token.
-
-The body of the POST request is a JSON object containing:
-
-username
-  Optional. This is the local part of the desired Matrix ID. If omitted, the
-  Home Server must generate a Matrix ID local part.
-password
-  Required. The desired password for the account.
-bind_email
-  Optional. If ``true``, the server binds the email used for authentication to
-  the Matrix ID with the ID Server.
-
-On success, this returns a JSON object with keys:
-
-user_id
-  The fully-qualified Matrix ID that has been registered.
-access_token
-  An access token for the new account.
-home_server
-  The hostname of the Home Server on which the account has been registered.
-
-This endpoint may also return the following error codes:
-
-M_USER_IN_USE
-  If the Matrix ID is already in use
-M_EXCLUSIVE
-  If the requested Matrix ID is in the exclusive namespace of an application
-  service.
+~~~~~~~~~~~~
+This API endpoint uses the `User-Interactive Authentication API`_.
 
-Home Servers MUST perform the relevant checks and return these codes before
-performing User-Interactive Authentication, although they may also return
-them after authentication is completed if, for example, the requested user ID
-was registered whilst the client was performing authentication.
+{{v2_registration_http_api}}
 
-Old V1 API docs: |register|_
+Login
+~~~~~
 
 {{login_http_api}}
 

specification/events.rst

@@ -1,5 +1,5 @@
-Events
-======
+Event Structure
+===============
 
 All communication in Matrix is expressed in the form of data objects called
 Events. These are the fundamental building blocks common to the client-server,
@@ -12,6 +12,22 @@ server-server and application-service APIs, and are described below.
 {{common_state_event_fields}}
 
 
+Size limits
+-----------
+
+The total size of any event MUST NOT exceed 65 KB. There are additional
+restrictions on sizes per key:
+
+ - ``user_id`` MUST NOT exceed 255 bytes (including domain).
+ - ``room_id`` MUST NOT exceed 255 bytes.
+ - ``state_key`` MUST NOT exceed 255 bytes.
+ - ``type`` MUST NOT exceed 255 bytes.
+ - ``event_id`` MUST NOT exceed 255 bytes.
+
+Some event types have additional size restrictions which are specified in
+the description of the event. Additional keys have no limit other than that
+implied by the total 65 KB limit on events.
+
 Room Events
 -----------
 .. NOTE::
@@ -26,8 +42,6 @@ prefixed with ``m.``
 
 {{m_room_create_event}}
 
-{{m_room_history_visibility_event}}
-
 {{m_room_join_rules_event}}
 
 {{m_room_member_event}}

specification/feature_profiles.rst

@@ -1,7 +1,7 @@
 Feature Profiles
 ================
 
-.. sect:feature-profiles:
+.. _sect:feature-profiles:
 
 Matrix supports many different kinds of clients: from embedded IoT devices to
 desktop clients. Not all clients can provide the same feature sets as other

specification/intro.rst

@@ -99,23 +99,23 @@ Architecture
 ------------
 
 Matrix defines APIs for synchronising extensible JSON objects known as
-``events`` between compatible clients, servers and services. Clients are
+"events" between compatible clients, servers and services. Clients are
 typically messaging/VoIP applications or IoT devices/hubs and communicate by
-synchronising communication history with their ``homeserver`` using the
-``Client-Server API``. Each homeserver stores the communication history and
+synchronising communication history with their "homeserver" using the
+"Client-Server API". Each homeserver stores the communication history and
 account information for all of its clients, and shares data with the wider
 Matrix ecosystem by synchronising communication history with other homeservers
 and their clients.
 
 Clients typically communicate with each other by emitting events in the
-context of a virtual ``room``. Room data is replicated across *all of the
+context of a virtual "room". Room data is replicated across *all of the
 homeservers* whose users are participating in a given room. As such, *no
 single homeserver has control or ownership over a given room*. Homeservers
 model communication history as a partially ordered graph of events known as
-the room's ``event graph``, which is synchronised with eventual consistency
-between the participating servers using the ``Server-Server API``. This process
+the room's "event graph", which is synchronised with eventual consistency
+between the participating servers using the "Server-Server API". This process
 of synchronising shared conversation history between homeservers run by
-different parties is called ``Federation``. Matrix optimises for the the
+different parties is called "Federation". Matrix optimises for the the
 Availability and Partitioned properties of CAP theorem at
 the expense of Consistency.
 
@@ -151,13 +151,13 @@ Users
 ~~~~~
 
 Each client is associated with a user account, which is identified in Matrix
-using a unique "User ID". This ID is namespaced to the home server which
+using a unique "User ID". This ID is namespaced to the homeserver which
 allocated the account and has the form::
 
   @localpart:domain
 
 The ``localpart`` of a user ID may be a user name, or an opaque ID identifying
-this user. They are case-insensitive.
+this user. The ``domain`` of a user ID is the domain of the homeserver.
 
 .. TODO-spec
     - Need to specify precise grammar for Matrix IDs
@@ -183,9 +183,9 @@ Event Graphs
 .. _sect:event-graph:
 
 Events exchanged in the context of a room are stored in a directed acyclic graph
-(DAG) called an ``event graph``. The partial ordering of this graph gives the
+(DAG) called an "event graph". The partial ordering of this graph gives the
 chronological ordering of events within the room. Each event in the graph has a
-list of zero or more ``parent`` events, which refer to any preceding events
+list of zero or more "parent" events, which refer to any preceding events
 which have no chronological successor from the perspective of the homeserver
 which created the event.
 
@@ -292,11 +292,12 @@ Each room can also have multiple "Room Aliases", which look like::
 
 A room alias "points" to a room ID and is the human-readable label by which
 rooms are publicised and discovered.  The room ID the alias is pointing to can
-be obtained by visiting the domain specified. They are case-insensitive. Note
-that the mapping from a room alias to a room ID is not fixed, and may change
-over time to point to a different room ID. For this reason, Clients SHOULD
-resolve the room alias to a room ID once and then use that ID on subsequent
-requests.
+be obtained by visiting the domain specified. Note that the mapping from a room
+alias to a room ID is not fixed, and may change over time to point to a
+different room ID. For this reason, Clients SHOULD resolve the room alias to a
+room ID once and then use that ID on subsequent requests. Room aliases MUST NOT
+exceed 255 bytes (including the domain).
+
 
 When resolving a room alias the server will also respond with a list of servers
 that are in the room that can be used to join via.
@@ -339,12 +340,9 @@ Profiles
 ~~~~~~~~
 
 Users may publish arbitrary key/value data associated with their account - such
-as a human readable ``display name``, a profile photo URL, contact information
+as a human readable display name, a profile photo URL, contact information
 (email address, phone numbers, website URLs etc).
 
-In Client-Server API v2, profile data is typed using namespaced keys for
-interoperability, much like events - e.g. ``m.profile.display_name``.
-
 .. TODO
   Actually specify the different types of data - e.g. what format are display
   names allowed to be?
@@ -431,13 +429,12 @@ Some requests have unique error codes:
 :``M_BAD_PAGINATION``:
   Encountered when specifying bad pagination query parameters.
 
-:``M_LOGIN_EMAIL_URL_NOT_YET``:
-  Encountered when polling for an email link which has not been clicked yet.
+.. _sect:txn_ids:
 
-The C-S API typically uses ``HTTP POST`` to submit requests. This means these
-requests are not idempotent. The C-S API also allows ``HTTP PUT`` to make
-requests idempotent. In order to use a ``PUT``, paths should be suffixed with
-``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which
+The Client-Server API typically uses ``HTTP POST`` to submit requests. This
+means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to
+make requests idempotent. In order to use a ``PUT``, paths should be suffixed
+with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which
 identifies the request, and is scoped to a given Client (identified by that
 client's ``access_token``). Crucially, it **only** serves to identify new
 requests from retransmits. After the request has finished, the ``{txnId}``

specification/modules/content_repo.rst

@@ -52,7 +52,17 @@ The HTTP GET endpoint does not require any authentication. Knowing the URL of
 the content is sufficient to retrieve the content, even if the entity isn't in
 the room.
 
-Homeservers have additional concerns:
+MXC URIs are vulnerable to directory traversal attacks such as
+``mxc://127.0.0.1/../../../some_service/etc/passwd``. This would cause the target
+homeserver to try to access and return this file. As such, homeservers MUST
+sanitise MXC URIs by allowing only alphanumeric (``A-Za-z0-9``), ``_``
+and  ``-`` characters in the ``server-name`` and ``media-id`` values. This set
+of whitelisted characters allows URL-safe base64 encodings specified in RFC 4648.
+Applying this character whitelist is preferable to blacklisting ``.`` and ``/``
+as there are techniques around blacklisted characters (percent-encoded characters,
+UTF-8 encoded traversals, etc).
+
+Homeservers have additional content-specific concerns:
 
  - Clients may try to upload very large files. Homeservers should not store files
    that are too large and should not serve them to clients.

specification/modules/history_visibility.rst

@@ -1,21 +1,54 @@
 Room History Visibility
------------------------
+=======================
 
 .. _module:history-visibility:
 
-Whether a member of a room can see the events that happened in a room from
-before they joined the room is controlled by the ``history_visibility`` key
-of the ``m.room.history_visibility`` state event. The valid values for
-``history_visibility`` are:
+This module adds support for controlling the visibility of previous events in a
+room.
+
+In all cases, a user needs to join a room to view events in that room. Once they
+have joined a room, they will gain access to a subset of events in the room. How
+this subset is chosen is controlled by the ``m.room.history_visibility`` event
+outlined below. After a user has left a room, they may seen any events which they
+were allowed to see before they left the room, but no events received after they
+left.
+
+The three options for the ``m.room.history_visibility`` event are:
+
+- ``shared`` - Previous events are always accessible to newly joined members. All
+  events in the room are accessible, even those sent when the member was not a part
+  of the room.
+- ``invited`` - Previous events are accessible to newly joined members from the point
+  they were invited onwards. Events stop being accessible when the member's state
+  changes to something other than ``invite`` or ``join``.
+- ``joined`` - Previous events are accessible to newly joined members from the point
+  they joined the room onwards. Events stop being accessible when the member's state
+  changes to something other than ``join``.
+
+.. WARNING::
+  These options are applied at the point an event is *sent*. Checks are
+  performed with the state of the ``m.room.history_visibility`` event when the
+  event in question is added to the DAG. This means clients cannot
+  retrospectively choose to show or hide history to new users if the setting at
+  that time was more restrictive.
+
+Events
+------
 
-- ``shared``
-- ``invited``
-- ``joined``
+{{m_room_history_visibility_event}}
 
-By default if no ``history_visibility`` is set it is assumed to be ``shared``.
+Client behaviour
+----------------
 
-The rules governing whether a user is allowed to see an event depend solely on
-the state of the room at that event:
+Clients that implement this module MUST present to the user the possible options
+for setting history visibility when creating a room. 
+
+Server behaviour
+----------------
+
+By default if no ``history_visibility`` is set the visibility is assumed to be
+``shared``. The rules governing whether a user is allowed to see an event depend
+solely on the state of the room *at that event*:
 
 1. If the user was joined, allow.
 2. If the user was invited and the ``history_visibility`` was set to
@@ -24,3 +57,10 @@ the state of the room at that event:
    was set to ``shared``, allow.
 4. Otherwise, deny.
 
+Security considerations
+-----------------------
+
+The default value for ``history_visibility`` is ``shared`` for
+backwards-compatibility reasons. Clients need to be aware that by not setting
+this event they are exposing all of their room history to anyone in the room.
+

specification/modules/instant_messaging.rst

@@ -3,6 +3,10 @@ Instant Messaging
 
 .. _module:im:
 
+This module adds support for sending human-readable messages to a room. It also
+adds support for associating human-readable information with the room itself
+such as a room name and topic.
+
 Events
 ------
 
@@ -10,20 +14,140 @@ Events
 
 {{m_room_message_feedback_event}}
 
+Usage of this event is discouraged for several reasons:
+ - The number of feedback events will grow very quickly with the number of users
+   in the room. This event provides no way to "batch" feedback, unlike the
+   `receipts module`_.
+ - Pairing feedback to messages gets complicated when paginating as feedback
+   arrives before the message it is acknowledging.
+ - There are no guarantees that the client has seen the event ID being
+   acknowledged.
+
+
+.. _`receipts module`: `module:receipts`_
+
 {{m_room_name_event}}
 
 {{m_room_topic_event}}
 
-m.room.message msgtypes
------------------------
-
-.. TODO-spec
-   How a client should handle unknown message types.
+{{m_room_avatar_event}}
 
+m.room.message msgtypes
+~~~~~~~~~~~~~~~~~~~~~~~
 
 Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
 of message being sent. Each type has their own required and optional keys, as
-outlined below.
+outlined below. If a client cannot display the given ``msgtype`` then it SHOULD
+display the fallback plain text ``body`` key instead.
 
 {{msgtype_events}}
 
+
+Client behaviour
+----------------
+
+Clients SHOULD verify the structure of incoming events to ensure that the
+expected keys exist and that they are of the right type. Clients can discard
+malformed events or display a placeholder message to the user. Redacted
+``m.room.message`` events MUST be removed from the client. This can either be
+replaced with placeholder text (e.g. "[REDACTED]") or the redacted message can
+be removed entirely from the messages view.
+
+Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be
+uploaded using the `content repository module`_ where available. The
+resulting ``mxc://`` URI can then be used in the ``url`` key.
+
+.. _`content repository module`: `module:content`_
+
+Recommendations when sending messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use
+``PUT`` requests with `transaction IDs`_ to make requests idempotent. This
+ensures that messages are sent exactly once even under poor network conditions.
+Clients SHOULD retry requests using an exponential-backoff algorithm for a
+certain amount of time T. It is recommended that T is no longer than 5 minutes.
+After this time, the client should stop retrying and mark the message as "unsent".
+Users should be able to manually resend unsent messages.
+
+Users may type several messages at once and send them all in quick succession.
+Clients SHOULD preserve the order in which they were sent by the user. This
+means that clients should wait for the response to the previous request before
+sending the next request. This can lead to head-of-line blocking. In order to
+reduce the impact of head-of-line blocking, clients should use a queue per room
+rather than a global queue, as ordering is only relevant within a single room
+rather than between rooms.
+
+.. _`transaction IDs`: `sect:txn_ids`_
+
+Local echo
+~~~~~~~~~~
+
+Messages SHOULD appear immediately in the message view when a user presses the
+"send" button. This should occur even if the message is still sending. This is
+referred to as "local echo". Clients SHOULD implement "local echo" of messages.
+Clients MAY display messages in a different format to indicate that the server
+has not processed the message. This format should be removed when the server
+responds.
+
+Clients need to be able to match the message they are sending with the same
+message which they receive from the event stream. The echo of the same message
+from the event stream is referred to as "remote echo". Both echoes need to be
+identified as the same message in order to prevent duplicate messages being
+displayed. Ideally this pairing would occur transparently to the user: the UI
+would not flicker as it transitions from local to remote. Flickering cannot be
+fully avoided in the current client-server API. Two scenarios need to be
+considered:
+
+- The client sends a message and the remote echo arrives on the event stream
+  *after* the request to send the message completes.
+- The client sends a message and the remote echo arrives on the event stream
+  *before* the request to send the message completes.
+
+In the first scenario, the client will receive an event ID when the request to
+send the message completes. This ID can be used to identify the duplicate event
+when it arrives on the event stream. However, in the second scenario, the event
+arrives before the client has obtained an event ID. This makes it impossible to
+identify it as a duplicate event. This results in the client displaying the
+message twice for a fraction of a second before the the original request to send
+the message completes. Once it completes, the client can take remedial actions
+to remove the duplicate event by looking for duplicate event IDs. A future version
+of the client-server API will resolve this by attaching the transaction ID of the
+sending request to the event itself.
+
+
+Displaying membership information with messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show the display name and avatar URL of the room member who
+sent a message. This can be achieved by inspecting the ``m.room.member`` state
+event for that user ID.
+
+When a user paginates the message history, clients may wish to show the
+**historical** display name and avatar URL for a room member. This is possible
+because older ``m.room.member`` events are returned when paginating. This can
+be implemented efficiently by keeping two sets of room state: old and current.
+As new events arrive and/or the user paginates back in time, these two sets of
+state diverge from each other. New events update the current state and paginated
+events update the old state. When paginated events are processed sequentially,
+the old state represents the state of the room *at the time the event was sent*.
+This can then be used to set the historical display name and avatar URL.
+
+Server behaviour
+----------------
+
+Homeservers SHOULD reject ``m.room.message`` events which don't have a
+``msgtype`` key, or which don't have a textual ``body`` key, with an HTTP status
+code of 400.
+
+Security considerations
+-----------------------
+
+Messages sent using this module are not encrypted. Messages can be encrypted
+using the `E2E module`_.
+
+Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross-Site
+Scripting (XSS) attacks. This includes room names and topics.
+
+.. _`E2E module`: `module:e2e`_
+

specification/modules/push.rst

@@ -353,14 +353,14 @@ Examples
 To create a rule that suppresses notifications for the room with ID
 ``!dj234r78wl45Gh4D:matrix.org``::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \
   '{
      "actions" : ["dont_notify"]
    }'
 
 To suppress notifications for the user ``@spambot:matrix.org``::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \
   '{
      "actions" : ["dont_notify"]
    }'
@@ -368,7 +368,7 @@ To suppress notifications for the user ``@spambot:matrix.org``::
 To always notify for messages that contain the work 'cake' and set a specific
 sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \
   '{
      "pattern": "cake",
      "actions" : ["notify", {"set_sound":"cakealarm.wav"}]
@@ -377,7 +377,7 @@ sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
 To add a rule suppressing notifications for messages starting with 'cake' but
 ending with 'lie', superseding the previous rule::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \
   '{
      "pattern": "cake*lie",
      "actions" : ["notify"]
@@ -387,7 +387,7 @@ To add a custom sound for notifications messages containing the word 'beer' in
 any rooms with 10 members or fewer (with greater importance than the room,
 sender and content rules)::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \
   '{
      "conditions": [
        {"kind": "event_match", "key": "content.body", "pattern": "beer" },

specification/server_server_api.rst

@@ -630,7 +630,7 @@ because HTTP services like Matrix are often deployed behind load balancers that
 handle the TLS and these load balancers make it difficult to check TLS client
 certificates.
 
-A home server may provide a TLS client certficate and the receiving home server
+A home server may provide a TLS client certificate and the receiving home server
 may check that the client certificate matches the certificate of the origin
 home server.
 

specification/targets.yaml

@@ -1,17 +1,17 @@
 targets:
   main:  # arbitrary name to identify this build target
     files:  # the sort order of files to cat
-      - 0-intro.rst
-      - 1-client_server_api.rst
-      - { 1: 0-events.rst }
-      - { 1: 0-event_signing.rst }
-      - 2-modules.rst
-      - { 1: 0-feature_profiles.rst }
+      - intro.rst
+      - client_server_api.rst
+      - { 1: events.rst }
+      - { 1: event_signing.rst }
+      - modules.rst
+      - { 1: feature_profiles.rst }
       - { 1: "group:modules" }  # reference a group of files
-      - 3-application_service_api.rst
-      - 4-server_server_api.rst
-      - 5-identity_servers.rst
-      - 6-appendices.rst
+      - application_service_api.rst
+      - server_server_api.rst
+      - identity_servers.rst
+      - appendices.rst
 groups:  # reusable blobs of files when prefixed with 'group:'
   modules:
     - modules/instant_messaging.rst