Merge branch 'master' into speculator-htmldiff

.gitignore

@@ -1,5 +1,6 @@
 scripts/gen
 scripts/continuserv/continuserv
+scripts/speculator/speculator
 templating/out
 *.pyc
 supporting-docs/_site

api/check_examples.py

@@ -0,0 +1,118 @@
+#! /usr/bin/env python
+
+import sys
+import json
+import os
+
+
+def import_error(module, package, debian, error):
+    sys.stderr.write((
+        "Error importing %(module)s: %(error)r\n"
+        "To install %(module)s run:\n"
+        "  pip install %(package)s\n"
+        "or on Debian run:\n"
+        "  sudo apt-get install python-%(debian)s\n"
+    ) % locals())
+    if __name__ == '__main__':
+        sys.exit(1)
+
+try:
+    import jsonschema
+except ImportError as e:
+    import_error("jsonschema", "jsonschema", "jsonschema", e)
+    raise
+
+try:
+    import yaml
+except ImportError as e:
+    import_error("yaml", "PyYAML", "yaml", e)
+    raise
+
+
+def check_parameter(filepath, request, parameter):
+    schema = parameter.get("schema")
+    example = None
+    try:
+        example_json = schema.get('example')
+        if example_json:
+            example = json.loads(example_json)
+    except Exception as e:
+        raise ValueError("Error parsing JSON example request for %r" % (
+            request
+        ), e)
+    fileurl = "file://" + os.path.abspath(filepath)
+    if example and schema:
+        try:
+            print ("Checking request schema for: %r %r" % (
+                filepath, request
+            ))
+            # Setting the 'id' tells jsonschema where the file is so that it
+            # can correctly resolve relative $ref references in the schema
+            schema['id'] = fileurl
+            jsonschema.validate(example, schema)
+        except Exception as e:
+            raise ValueError("Error validating JSON schema for %r %r" % (
+                request, code
+            ), e)
+
+
+def check_response(filepath, request, code, response):
+    example = None
+    try:
+        example_json = response.get('examples', {}).get('application/json')
+        if example_json:
+            example = json.loads(example_json)
+    except Exception as e:
+        raise ValueError("Error parsing JSON example response for %r %r" % (
+            request, code
+        ), e)
+    schema = response.get('schema')
+    fileurl = "file://" + os.path.abspath(filepath)
+    if example and schema:
+        try:
+            print ("Checking response schema for: %r %r %r" % (
+                filepath, request, code
+            ))
+            # Setting the 'id' tells jsonschema where the file is so that it
+            # can correctly resolve relative $ref references in the schema
+            schema['id'] = fileurl
+            jsonschema.validate(example, schema)
+        except Exception as e:
+            raise ValueError("Error validating JSON schema for %r %r" % (
+                request, code
+            ), e)
+
+
+def check_swagger_file(filepath):
+    with open(filepath) as f:
+        swagger = yaml.load(f)
+
+    for path, path_api in swagger.get('paths', {}).items():
+
+        for method, request_api in path_api.items():
+            request = "%s %s" % (method.upper(), path)
+            for parameter in request_api.get('parameters', ()):
+                if parameter['in'] == 'body':
+                    check_parameter(filepath, request, parameter)
+
+            try:
+                responses = request_api['responses']
+            except KeyError:
+                raise ValueError("No responses for %r" % (request,))
+            for code, response in responses.items():
+                check_response(filepath, request, code, response)
+
+
+if __name__ == '__main__':
+    paths = sys.argv[1:]
+    if not paths:
+        paths = []
+        for (root, dirs, files) in os.walk(os.curdir):
+            for filename in files:
+                if filename.endswith(".yaml"):
+                    paths.append(os.path.join(root, filename))
+    for path in paths:
+        try:
+            check_swagger_file(path)
+        except Exception as e:
+            raise ValueError("Error checking file %r" % (path,), e)

api/client-server/v1/core-event-schema

@@ -0,0 +1 @@
+v1-event-schema/core-event-schema

api/client-server/v1/definitions/event.yaml

@@ -1,7 +0,0 @@
-type: object
-description: A Matrix Event
-properties:
-  event_id:
-    type: string
-    description: An event ID.
-required: ["event_id"]

api/client-server/v1/definitions/room_event.yaml

@@ -1,9 +0,0 @@
-type: object
-description: A Matrix Room Event
-properties:
-  event_id:
-    type: string
-    description: An event ID.
-  room_id:
-    type: string
-required: ["event_id", "room_id"]

api/client-server/v1/definitions/state_event.yaml

@@ -1,11 +0,0 @@
-type: object
-description: A Matrix State Event
-properties:
-  event_id:
-    type: string
-    description: An event ID.
-  room_id:
-    type: string
-  state_key:
-    type: string
-required: ["event_id", "room_id", "state_key"]

api/client-server/v1/login.yaml

@@ -0,0 +1,147 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Registration and Login API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/login":
+    post:
+      summary: Authenticates the user.
+      description: |-
+        Authenticates the user by password, and issues an access token they can
+        use to authorize themself in subsequent requests.
+      security:
+        - accessToken: []
+      parameters:
+        - in: body
+          name: body
+          schema:
+            type: object
+            example: |-
+              {
+                "username": "cheeky_monkey",
+                "password": "ilovebananas"
+              }
+            properties:
+              username:
+                type: string
+                description: The fully qualified user ID or just local part of the user ID, to log in.
+              password:
+                type: string
+                description: The user's password.
+            required: ["username", "password"]
+      responses:
+        200:
+          description: The user has been authenticated.
+          examples:
+            application/json: |-
+              {
+                "user_id": "@cheeky_monkey:matrix.org",
+                "access_token": "abc123",
+                "home_server": "matrix.org"
+              }
+          schema:
+            type: object
+            properties:
+              user_id:
+                type: string
+                description: The fully-qualified Matrix ID that has been registered.
+              access_token:
+                type: string
+                description: |-
+                  An access token for the account.
+                  This access token can then be used to authorize other requests.
+                  The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``.
+                  There is no specific error message to indicate that a request has failed because
+                  an access token has expired; instead, if a client has reason to believe its
+                  access token is valid, and it receives an auth error, they should attempt to
+                  refresh for a new token on failure, and retry the request with the new token.
+              refresh_token:
+                type: string
+                # TODO: Work out how to linkify /tokenrefresh
+                description: |-
+                  (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint.
+              home_server:
+                type: string
+                description: The hostname of the Home Server on which the account has been registered.
+        403:
+          description: |-
+            The login attempt failed. For example, the password may have been incorrect.
+          examples:
+            application/json: |-
+              {"errcode": "M_FORBIDDEN"}
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"
+  "/tokenrefresh":
+    post:
+      summary: Exchanges a refresh token for an access token.
+      description: |-
+        Exchanges a refresh token for a new access token.
+        This is intended to be used if the access token has expired.
+      security:
+        - accessToken: []
+      parameters:
+        - in: body
+          name: body
+          schema:
+            type: object
+            example: |-
+              {
+                "refresh_token": "a1b2c3"
+              }
+            properties:
+              refresh_token:
+                type: string
+                description: The refresh token which was issued by the server.
+            required: ["refresh_token"]
+      responses:
+        200:
+          description: |-
+            The refresh token was accepted, and a new access token has been issued.
+            The passed refresh token is no longer valid and cannot be used.
+            A new refresh token will have been returned unless some policy does
+            not allow the user to continue to renew their session.
+          examples:
+            application/json: |-
+              {
+                "access_token": "bearwithme123",
+                "refresh_token": "exchangewithme987"
+              }
+          schema:
+            type: object
+            properties:
+              access_token:
+                type: string
+                description: |-
+                  An access token for the account.
+                  This access token can then be used to authorize other requests.
+                  The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``.
+              refresh_token:
+                type: string
+                description: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the TODO Linkify /tokenrefresh API endpoint.
+        403:
+          description: |-
+            The exchange attempt failed. For example, the refresh token may have already been used.
+          examples:
+            application/json: |-
+              {"errcode": "M_FORBIDDEN"}
+        429:
+          description: This request was rate-limited.
+          schema:
+            "$ref": "definitions/error.yaml"

api/client-server/v1/membership.yaml

@@ -69,7 +69,6 @@ paths:
   "/rooms/{roomId}/invite":
     post:
       summary: Invite a user to participate in a particular room.
-      # It's a crying shame that I don't know how to force line breaks.
       description: |-
         This API invites a user to participate in a particular room.
         They do not start participating in the room until they actually join the

api/client-server/v1/presence.yaml

@@ -101,7 +101,7 @@ paths:
                   The length of time in milliseconds since an action was performed
                   by this user.
               status_msg:
-                type: string
+                type: [string, "null"]
                 description: The state message for this user if one was set.
         404:
           description: |-
@@ -185,7 +185,7 @@ paths:
                     "last_active_ago": 395,
                     "presence": "offline",
                     "user_id": "@alice:matrix.org"
-                  }
+                  },
                   "type": "m.presence"
                 },
                 {
@@ -195,7 +195,7 @@ paths:
                     "last_active_ago": 16874,
                     "presence": "online",
                     "user_id": "@marisa:matrix.org"
-                  }
+                  },
                   "type": "m.presence"
                 }
               ]
@@ -205,4 +205,4 @@ paths:
               type: object
               title: PresenceEvent
               allOf:
-                - "$ref": "definitions/event.yaml"
+                - "$ref": "core-event-schema/event.json"

api/client-server/v1/rooms.yaml

@@ -0,0 +1,448 @@
+swagger: '2.0'
+info:
+  title: "Matrix Client-Server v1 Rooms API"
+  version: "1.0.0"
+host: localhost:8008
+schemes:
+  - https
+  - http
+basePath: /_matrix/client/api/v1
+consumes:
+  - application/json
+produces:
+  - application/json
+securityDefinitions:
+  accessToken:
+    type: apiKey
+    description: The user_id or application service access_token
+    name: access_token
+    in: query
+paths:
+  "/rooms/{roomId}/state/{eventType}/{stateKey}":
+    get:
+      summary: Get the state identified by the type and key.
+      description: |-
+        Looks up the contents of a state event in a room. If the user is
+        joined to the room then the state is taken from the current
+        state of the room. If the user has left the room then the state is
+        taken from the state of the room when they left.
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room to look up the state in.
+          required: true
+          x-example: "!636q39766251:example.com"
+        - in: path
+          type: string
+          name: eventType
+          description: The type of state to look up.
+          required: true
+          x-example: "m.room.name"
+        - in: path
+          type: string
+          name: stateKey
+          description: The key of the state to look up. Defaults to the empty string.
+          required: true
+          x-example: ""
+      responses:
+        200:
+          description: The content of the state event.
+          examples:
+            application/json: |-
+              {"name": "Example room name"}
+          schema:
+            type: object
+        404:
+          description: The room has no state with the given type or key.
+        403:
+          description: >
+            You aren't a member of the room and weren't previously a
+            member of the room.
+
+  "/rooms/{roomId}/state":
+    get:
+      summary: Get all state events in the current state of a room.
+      description: |-
+        Get the state events for the current state of a room.
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room to look up the state for.
+          required: true
+          x-example: "!636q39766251:example.com"
+      responses:
+        200:
+          description: The current state of the room
+          examples:
+            application/json: |-
+              [
+                {
+                    "age": 7148266897,
+                    "content": {
+                        "join_rule": "public"
+                    },
+                    "event_id": "$14259997323TLwtb:example.com",
+                    "origin_server_ts": 1425999732392,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.join_rules",
+                    "user_id": "@alice:example.com"
+                },
+                {
+                    "age": 6547561012,
+                    "content": {
+                        "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto",
+                        "displayname": null,
+                        "membership": "join"
+                    },
+                    "event_id": "$1426600438280zExKY:example.com",
+                    "membership": "join",
+                    "origin_server_ts": 1426600438277,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@alice:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@alice:example.com"
+                },
+                {
+                    "age": 7148267200,
+                    "content": {
+                        "creator": "@alice:example.com"
+                    },
+                    "event_id": "$14259997320KhbwJ:example.com",
+                    "origin_server_ts": 1425999732089,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.create",
+                    "user_id": "@alice:example.com"
+                },
+                {
+                    "age": 1622568720,
+                    "content": {
+                        "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto",
+                        "displayname": "Bob",
+                        "membership": "join"
+                    },
+                    "event_id": "$1431525430134MxlLX:example.com",
+                    "origin_server_ts": 1431525430569,
+                    "replaces_state": "$142652023736BSXcM:example.com",
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@bob:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@bob:example.com"
+                },
+                {
+                    "age": 7148267004,
+                    "content": {
+                        "ban": 50,
+                        "events": {
+                            "m.room.name": 100,
+                            "m.room.power_levels": 100
+                         },
+                         "events_default": 0,
+                         "kick": 50,
+                         "redact": 50,
+                         "state_default": 50,
+                         "users": {
+                             "@alice:example.com": 100
+                         },
+                         "users_default": 0
+                    },
+                    "event_id": "$14259997322mqfaq:example.com",
+                    "origin_server_ts": 1425999732285,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.power_levels",
+                    "user_id": "@alice:example.com"
+                }
+              ]
+          schema:
+            type: array
+            title: RoomState
+            description: |-
+              If the user is a member of the room this will be the
+              current state of the room as a list of events. If the user
+              has left the room then this will be the state of the room
+              when they left as a list of events.
+            items:
+              title: StateEvent
+              type: object
+              allOf:
+                - "$ref": "core-event-schema/state_event.json"
+        403:
+          description: >
+            You aren't a member of the room and weren't previously a
+            member of the room.
+
+  "/rooms/{roomId}/initialSync":
+    get:
+      summary: Snapshot the current state of a room and its most recent messages.
+      description: |-
+        Get a copy of the current state and the most recent messages in a room.
+      security:
+        - accessToken: []
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room to get the data.
+          required: true
+          x-example: "!636q39766251:example.com"
+      responses:
+        200:
+          description: The current state of the room
+          examples:
+            application/json: |-
+              {
+                "membership": "join",
+                "messages": {
+                  "chunk": [
+                    {
+                      "age": 343513403,
+                      "content": {
+                        "body": "foo",
+                        "msgtype": "m.text"
+                      },
+                      "event_id": "$14328044851tzTJS:example.com",
+                      "origin_server_ts": 1432804485886,
+                      "room_id": "!636q39766251:example.com",
+                      "type": "m.room.message",
+                      "user_id": "@alice:example.com"
+                    },
+                    {
+                      "age": 343511809,
+                      "content": {
+                        "body": "bar",
+                        "msgtype": "m.text"
+                      },
+                      "event_id": "$14328044872spjFg:example.com",
+                      "origin_server_ts": 1432804487480,
+                      "room_id": "!636q39766251:example.com",
+                      "type": "m.room.message",
+                      "user_id": "@bob:example.com"
+                    }
+                  ],
+                  "end": "s3456_9_0",
+                  "start": "t44-3453_9_0"
+                },
+                "room_id": "!636q39766251:example.com",
+                "state": [
+                  {
+                    "age": 7148266897,
+                    "content": {
+                      "join_rule": "public"
+                    },
+                    "event_id": "$14259997323TLwtb:example.com",
+                    "origin_server_ts": 1425999732392,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.join_rules",
+                    "user_id": "@alice:example.com"
+                  },
+                  {
+                    "age": 6547561012,
+                    "content": {
+                      "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto",
+                      "displayname": null,
+                      "membership": "join"
+                    },
+                    "event_id": "$1426600438280zExKY:example.com",
+                    "membership": "join",
+                    "origin_server_ts": 1426600438277,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@alice:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@alice:example.com"
+                  },
+                  {
+                    "age": 7148267200,
+                    "content": {
+                        "creator": "@alice:example.com"
+                    },
+                    "event_id": "$14259997320KhbwJ:example.com",
+                    "origin_server_ts": 1425999732089,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.create",
+                    "user_id": "@alice:example.com"
+                  },
+                  {
+                    "age": 1622568720,
+                    "content": {
+                        "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto",
+                        "displayname": "Bob",
+                        "membership": "join"
+                    },
+                    "event_id": "$1431525430134MxlLX:example.com",
+                    "origin_server_ts": 1431525430569,
+                    "replaces_state": "$142652023736BSXcM:example.com",
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@bob:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@bob:example.com"
+                  },
+                  {
+                    "age": 7148267004,
+                    "content": {
+                      "ban": 50,
+                      "events": {
+                        "m.room.name": 100,
+                        "m.room.power_levels": 100
+                      },
+                      "events_default": 0,
+                      "kick": 50,
+                      "redact": 50,
+                      "state_default": 50,
+                      "users": {
+                        "@alice:example.com": 100
+                      },
+                      "users_default": 0
+                    },
+                    "event_id": "$14259997322mqfaq:example.com",
+                    "origin_server_ts": 1425999732285,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "",
+                    "type": "m.room.power_levels",
+                    "user_id": "@alice:example.com"
+                  }
+                ],
+                "visibility": "private"
+              }
+          schema:
+            title: RoomInfo
+            type: object
+            properties:
+              room_id:
+                type: string
+                description: "The ID of this room."
+              membership:
+                type: string
+                description: "The user's membership state in this room."
+                enum: ["invite", "join", "leave", "ban"]
+              messages:
+                type: object
+                title: PaginationChunk
+                description: "The pagination chunk for this room."
+                properties:
+                  start:
+                    type: string
+                    description: |-
+                      A token which correlates to the first value in ``chunk``.
+                      Used for pagination.
+                  end:
+                    type: string
+                    description: |-
+                      A token which correlates to the last value in ``chunk``.
+                      Used for pagination.
+                  chunk:
+                    type: array
+                    description: |-
+                      If the user is a member of the room this will be a
+                      list of the most recent messages for this room. If
+                      the user has left the room this will be the
+                      messages that preceeded them leaving. This array
+                      will consist of at most ``limit`` elements.
+                    items:
+                      type: object
+                      title: RoomEvent
+                      allOf:
+                        - "$ref": "core-event-schema/room_event.json"
+                required: ["start", "end", "chunk"]
+              state:
+                type: array
+                description: |-
+                  If the user is a member of the room this will be the
+                  current state of the room as a list of events. If the
+                  user has left the room this will be the state of the
+                  room when they left it.
+                items:
+                  title: StateEvent
+                  type: object
+                  allOf:
+                    - "$ref": "core-event-schema/state_event.json"
+              visibility:
+                type: string
+                enum: ["private", "public"]
+                description: |-
+                  Whether this room is visible to the ``/publicRooms`` API
+                  or not."
+            required: ["room_id", "membership"]
+        403:
+          description: >
+            You aren't a member of the room and weren't previously a
+            member of the room.
+
+  "/rooms/{roomId}/members":
+    get:
+      summary: Get the m.room.member events for the room.
+      description:
+        Get the list of members for this room.
+      parameters:
+        - in: path
+          type: string
+          name: roomId
+          description: The room to get the member events for.
+          required: true
+          x-example: "!636q39766251:example.com"
+      responses:
+        200:
+          description: |-
+            A list of members of the room. If you are joined to the room then
+            this will be the current members of the room. If you have left te
+            room then this will be the members of the room when you left.
+          examples:
+            application/json: |-
+              {
+                "chunk": [
+                  {
+                    "age": 6547561012,
+                    "content": {
+                      "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto",
+                      "displayname": null,
+                      "membership": "join"
+                    },
+                    "event_id": "$1426600438280zExKY:example.com",
+                    "membership": "join",
+                    "origin_server_ts": 1426600438277,
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@alice:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@alice:example.com"
+                  },
+                  {
+                    "age": 1622568720,
+                    "content": {
+                        "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto",
+                        "displayname": "Bob",
+                        "membership": "join"
+                    },
+                    "event_id": "$1431525430134MxlLX:example.com",
+                    "origin_server_ts": 1431525430569,
+                    "replaces_state": "$142652023736BSXcM:example.com",
+                    "room_id": "!636q39766251:example.com",
+                    "state_key": "@bob:example.com",
+                    "type": "m.room.member",
+                    "user_id": "@bob:example.com"
+                  }
+                ]
+              }
+          schema:
+            type: object
+            properties:
+              chunk:
+                type: array
+                items:
+                  title: MemberEvent
+                  type: object
+                  allOf:
+                    - "$ref": "v1-event-schema/m.room.member"
+        403:
+          description: >
+            You aren't a member of the room and weren't previously a
+            member of the room.
+

api/client-server/v1/sync.yaml

@@ -82,7 +82,7 @@ paths:
                   type: object
                   title: RoomEvent
                   allOf:
-                   - "$ref": "definitions/room_event.yaml"
+                   - "$ref": "core-event-schema/room_event.json"
         400:
           description: "Bad pagination ``from`` parameter."
   "/initialSync":
@@ -253,7 +253,7 @@ paths:
                   type: object
                   title: Event
                   allOf:
-                   - "$ref": "definitions/event.yaml"
+                   - "$ref": "core-event-schema/event.json"
               rooms:
                 type: array
                 items:
@@ -294,7 +294,7 @@ paths:
                             type: object
                             title: RoomEvent
                             allOf:
-                              - "$ref": "definitions/room_event.yaml"
+                              - "$ref": "core-event-schema/room_event.json"
                       required: ["start", "end", "chunk"]
                     state:
                       type: array
@@ -307,7 +307,7 @@ paths:
                         title: StateEvent
                         type: object
                         allOf:
-                          - "$ref": "definitions/state_event.yaml"
+                          - "$ref": "core-event-schema/state_event.json"
                     visibility:
                       type: string
                       enum: ["private", "public"]
@@ -343,13 +343,13 @@ paths:
                   "body": "Hello world!",
                   "msgtype": "m.text"
                 },
-                "room_id:" "!wfgy43Sg4a:matrix.org",
+                "room_id:": "!wfgy43Sg4a:matrix.org",
                 "user_id": "@bob:matrix.org",
                 "event_id": "$asfDuShaf7Gafaw:matrix.org",
                 "type": "m.room.message"
               }
           schema:
             allOf:
-              - "$ref": "definitions/event.yaml"
+              - "$ref": "core-event-schema/event.json"
         404:
           description: The event was not found or you do not have permission to read this event.

api/client-server/v1/v1-event-schema

@@ -0,0 +1 @@
+../../../event-schemas/schema/v1

api/validator.js

@@ -44,7 +44,8 @@ if (isDir) {
             process.exit(1);
         }
         files.forEach(function(f) {
-            if (f.indexOf(".yaml") > 0) {
+            var suffix = ".yaml";
+            if (f.indexOf(suffix, f.length - suffix.length) > 0) {
                 parser.parse(path.join(opts.schema, f), function(err, api, metadata) {
                     if (!err) {
                         console.log("%s is valid.", f);

drafts/address-book-repo.rst

@@ -3,10 +3,10 @@ Address book repository
 
 .. NOTE::
   This section is a work in progress.
-  Do we even need it?  Clients can use out-of-band addressbook servers for now;
-  this should definitely not be core.
 
 .. TODO-spec
+  Do we even need it?  Clients can use out-of-band addressbook servers for now;
+  this should definitely not be core.
   - format: POST(?) wodges of json, some possible processing, then return wodges of json on GET.
   - processing may remove dupes, merge contacts, pepper with extra info (e.g. matrix-ability of
     contacts), etc.

drafts/macaroons_caveats.rst

@@ -0,0 +1,34 @@
+Macaroon Caveats
+================
+
+`Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them.
+
+.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf
+
+Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand.
+
+Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed.
+
+All caveats must take the form:
+
+`key` `operator` `value`
+where `key` is a non-empty string drawn from the character set [A-Za-z0-9_]
+`operator` is a non-empty string which does not contain whitespace
+`value` is a non-empty string
+And these are joined by single space characters.
+
+Specified caveats:
+
++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+
+| Caveat name | Description                                      | Legal Values                                                                                   |
++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+
+| gen         | Generation of the macaroon caveat spec.          | 1                                                                                              |
+| user_id     | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =.                                                       |
+| type        | The purpose of this macaroon.                    | access - used to authorize any action except token refresh                                     |
+|                                                                   refresh - only used to authorize a token refresh                                              |
+| time        | Time before/after which this macaroon is valid.  | A POSIX timestamp in milliseconds (in UTC).                                                    |
+|                                                                  Operator < means the macaroon is valid before the timestamp, as interpreted by the server.     |
+|                                                                  Operator > means the macaroon is valid after the timestamp, as interpreted by the server.      |
+|                                                                  Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.|
+|                                                                  Note that exact equality of time is largely meaningless.                                       |
++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+

event-schemas/check_examples.py

@@ -0,0 +1,76 @@
+#! /usr/bin/env python
+
+import sys
+import json
+import os
+import traceback
+
+
+def import_error(module, package, debian, error):
+    sys.stderr.write((
+        "Error importing %(module)s: %(error)r\n"
+        "To install %(module)s run:\n"
+        "  pip install %(package)s\n"
+        "or on Debian run:\n"
+        "  sudo apt-get install python-%(debian)s\n"
+    ) % locals())
+    if __name__ == '__main__':
+        sys.exit(1)
+
+try:
+    import jsonschema
+except ImportError as e:
+    import_error("jsonschema", "jsonschema", "jsonschema", e)
+    raise
+
+try:
+    import yaml
+except ImportError as e:
+    import_error("yaml", "PyYAML", "yaml", e)
+    raise
+
+
+def check_example_file(examplepath, schemapath):
+    with open(examplepath) as f:
+        example = yaml.load(f)
+
+    with open(schemapath) as f:
+        schema = yaml.load(f)
+
+    fileurl = "file://" + os.path.abspath(schemapath)
+
+    print ("Checking schema for: %r %r" % (examplepath, schemapath))
+    # Setting the 'id' tells jsonschema where the file is so that it
+    # can correctly resolve relative $ref references in the schema
+    schema['id'] = fileurl
+    try:
+        jsonschema.validate(example, schema)
+    except Exception as e:
+        raise ValueError("Error validating JSON schema for %r %r" % (
+            examplepath, schemapath
+        ), e)
+
+
+def check_example_dir(exampledir, schemadir):
+    errors = []
+    for root, dirs, files in os.walk(exampledir):
+        for filename in files:
+            if filename.startswith("."):
+                # Skip over any vim .swp files.
+                continue
+            examplepath = os.path.join(root, filename)
+            schemapath = examplepath.replace(exampledir, schemadir)
+            try:
+                check_example_file(examplepath, schemapath)
+            except Exception as e:
+                errors.append(sys.exc_info())
+    for (exc_type, exc_value, exc_trace) in errors:
+        traceback.print_exception(exc_type, exc_value, exc_trace)
+    if errors:
+        raise ValueError("Error validating examples")
+
+if __name__ == '__main__':
+    try:
+        check_example_dir("examples", "schema")
+    except:
+        sys.exit(1)

event-schemas/schema/v1/core

@@ -1,88 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "definitions": {
-    "event": {
-      "title": "Event",
-      "description": "The basic set of fields all events must have.",
-      "type": "object",
-      "properties": {
-        "event_id": {
-          "type": "string",
-          "description": "The globally unique event identifier."
-        },
-        "user_id": {
-          "type": "string",
-          "description": "Contains the fully-qualified ID of the user who *sent* this event."
-        },
-        "content": {
-          "type": "object",
-          "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body."
-        },
-        "type": {
-          "type": "string",
-          "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'"
-        }
-      },
-      "required": ["event_id", "user_id", "content", "type"]
-    },
-    "room_event": {
-        "type": "object",
-        "title": "Room Event",
-        "description": "In addition to the Event fields, Room Events MUST have the following additional field.",
-        "allOf":[{
-            "$ref": "#/definitions/event"
-        }],
-        "properties": {
-            "room_id": {
-                "type": "string",
-                "description": "The ID of the room associated with this event."
-            }
-        },
-        "required": ["room_id"]
-    },
-    "state_event": {
-        "type": "object",
-        "title": "State Event",
-        "description": "In addition to the Room Event fields, State Events have the following additional fields.",
-        "allOf":[{
-            "$ref": "#/definitions/room_event"
-        }],
-        "properties": {
-            "state_key": {
-                "type": "string",
-                "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event."
-            },
-            "prev_content": {
-                "type": "object",
-                "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing."
-            }
-        },
-        "required": ["state_key"]
-    },
-    "msgtype_infos": {
-      "image_info": {
-        "type": "object",
-        "title": "ImageInfo",
-        "description": "Metadata about an image.",
-        "properties": {
-            "size": {
-                "type": "integer",
-                "description": "Size of the image in bytes."
-            },
-            "w": {
-                "type": "integer",
-                "description": "The width of the image in pixels."
-            },
-            "h": {
-                "type": "integer",
-                "description": "The height of the image in pixels."
-            },
-            "mimetype": {
-                "type": "string",
-                "description": "The mimetype of the image, e.g. ``image/jpeg``."
-            }
-        }
-      }
-    }
-  }
-}

event-schemas/schema/v1/core-event-schema/core-event-schema

@@ -0,0 +1 @@
+.

event-schemas/schema/v1/core-event-schema/event.json

@@ -0,0 +1,15 @@
+{
+  "type": "object",
+  "title": "Event",
+  "description": "The basic set of fields all events must have.",
+  "properties": {
+    "content": {
+      "type": "object",
+      "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body."
+    },
+    "type": {
+      "type": "string",
+      "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'"
+    }
+  }
+}

event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "ImageInfo",
+  "description": "Metadata about an image.",
+  "properties": {
+    "size": {
+      "type": "integer",
+      "description": "Size of the image in bytes."
+    },
+    "w": {
+      "type": "integer",
+      "description": "The width of the image in pixels."
+    },
+    "h": {
+      "type": "integer",
+      "description": "The height of the image in pixels."
+    },
+    "mimetype": {
+      "type": "string",
+      "description": "The mimetype of the image, e.g. ``image/jpeg``."
+    }
+  }
+}

event-schemas/schema/v1/core-event-schema/room_event.json

@@ -0,0 +1,23 @@
+{
+  "type": "object",
+  "title": "Room Event",
+  "description": "In addition to the Event fields, Room Events MUST have the following additional field.",
+  "allOf":[{
+    "$ref": "core-event-schema/event.json"
+  }],
+  "properties": {
+    "room_id": {
+      "type": "string",
+      "description": "The ID of the room associated with this event."
+    },
+    "event_id": {
+      "type": "string",
+      "description": "The globally unique event identifier."
+    },
+    "user_id": {
+      "type": "string",
+      "description": "Contains the fully-qualified ID of the user who *sent* this event."
+    }
+  },
+  "required": ["room_id"]
+}

event-schemas/schema/v1/core-event-schema/state_event.json

@@ -0,0 +1,19 @@
+{
+  "type": "object",
+  "title": "State Event",
+  "description": "In addition to the Room Event fields, State Events have the following additional fields.",
+  "allOf":[{
+    "$ref": "core-event-schema/room_event.json"
+  }],
+  "properties": {
+    "state_key": {
+      "type": "string",
+      "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event."
+    },
+    "prev_content": {
+      "type": "object",
+      "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing."
+    }
+  },
+  "required": ["state_key"]
+}

event-schemas/schema/v1/m.call.answer

@@ -1,9 +1,8 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "description": "This event is sent by the callee when they wish to answer the call.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.call.candidates

@@ -1,9 +1,8 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "description": "This event is sent by callers after sending an invite and by the callee after answering. Its purpose is to give the other party additional ICE candidates to try using to communicate.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.call.hangup

@@ -1,9 +1,8 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "description": "Sent by either party to signal their termination of the call. This can be sent either once the call has has been established or before to abort the call.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.call.invite

@@ -1,9 +1,8 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "description": "This event is sent by the caller when they wish to establish a call.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.presence

@@ -1,5 +1,4 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Presence Event",
     "description": "Informs the client of a user's presence state change.",

event-schemas/schema/v1/m.receipt

@@ -1,5 +1,4 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Receipt Event",
     "description": "Informs the client of new receipts.",

event-schemas/schema/v1/m.room.aliases

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Informs the room about what room aliases it has been given.",
     "description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.canonical_alias

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Informs the room as to which alias is the canonical one.",
     "description": "This event is used to inform the room about which alias should be considered the canonical one. This could be for display purposes or as suggestion to users which alias to use to advertise the room.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.create

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "The first event in the room.",
     "description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.history_visibility

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Controls visibility of history.",
     "description": "This event controls whether a member of a room can see the events that happened in a room from before they joined.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.join_rules

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Describes how users are allowed to join the room.",
     "description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room. Currently, ``knock`` and ``private`` are reserved keywords which are not implemented.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.member

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "The current membership state of a user in the room.",
     "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms/<room id>/invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {
@@ -20,7 +19,7 @@
                     "description": "The avatar URL for this user, if any. This is added by the homeserver."
                 },
                 "displayname": {
-                    "type": "string",
+                    "type": ["string", "null"],
                     "description": "The display name for this user, if any. This is added by the homeserver."
                 }
             },

event-schemas/schema/v1/m.room.message

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Message",
     "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.message#m.audio

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "AudioMessage",
     "description": "This message represents a single audio clip.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.message#m.emote

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "EmoteMessage",
     "description": "This message is similar to ``m.text`` except that the sender is 'performing' the action contained in the ``body`` key, similar to ``/me`` in IRC. This message should be prefixed by the name of the sender. This message could also be represented in a different colour to distinguish it from regular ``m.text`` messages.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.message#m.file

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "FileMessage",
     "description": "This message represents a generic file.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {
@@ -50,7 +49,7 @@
                     "title": "ImageInfo",
                     "description": "Metadata about the image referred to in ``thumbnail_url``.",
                     "allOf": [{
-                        "$ref": "core#/definitions/msgtype_infos/image_info"
+                        "$ref": "core-event-schema/msgtype_infos/image_info.json"
                     }]
                 }
             },

event-schemas/schema/v1/m.room.message#m.image

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "ImageMessage",
     "description": "This message represents a single image and an optional thumbnail.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {
@@ -31,7 +30,7 @@
                     "title": "ImageInfo",
                     "description": "Metadata about the image referred to in ``thumbnail_url``.",
                     "allOf": [{
-                        "$ref": "core#/definitions/msgtype_infos/image_info"
+                        "$ref": "core-event-schema/msgtype_infos/image_info.json"
                     }]
                 },
                 "info": {

event-schemas/schema/v1/m.room.message#m.location

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "LocationMessage",
     "description": "This message represents a real-world location.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {
@@ -30,7 +29,7 @@
                     "type": "object",
                     "title": "ImageInfo",
                     "allOf": [{
-                        "$ref": "core#/definitions/msgtype_infos/image_info"
+                        "$ref": "core-event-schema/msgtype_infos/image_info.json"
                     }]
                 }
             },

event-schemas/schema/v1/m.room.message#m.notice

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "NoticeMessage",
     "description": "A m.notice message should be considered similar to a plain m.text message except that clients should visually distinguish it in some way. It is intended to be used by automated clients, such as bots, bridges, and other entities, rather than humans. Additionally, such automated agents which watch a room for messages and respond to them ought to ignore m.notice messages. This helps to prevent infinite-loop situations where two automated clients continuously exchange messages, as each responds to the other.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.message#m.text

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "TextMessage",
     "description": "This message is the most basic message and is used to represent text.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.message#m.video

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "VideoMessage",
     "description": "This message represents a single video clip.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {
@@ -55,7 +54,7 @@
                             "type": "object",
                             "title": "ImageInfo",
                             "allOf": [{
-                                "$ref": "core#/definitions/msgtype_infos/image_info"
+                                "$ref": "core-event-schema/msgtype_infos/image_info.json"
                             }]
                         }
                     }

event-schemas/schema/v1/m.room.message.feedback

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "MessageFeedback",
     "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.name

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "title": "RoomName",
     "description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.",
     "type": "object",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.power_levels

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Defines the power levels (privileges) of users in the room.",
     "description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.redaction

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Redaction",
     "description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data.  There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.",
     "allOf": [{
-        "$ref": "core#/definitions/room_event"
+        "$ref": "core-event-schema/room_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/m.room.topic

@@ -1,10 +1,9 @@
 {
-    "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
     "title": "Topic",
     "description": "A topic is a short message detailing what is currently being discussed in the room.  It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.",
     "allOf": [{
-        "$ref": "core#/definitions/state_event"
+        "$ref": "core-event-schema/state_event.json"
     }],
     "properties": {
         "content": {

event-schemas/schema/v1/v1-event-schema

@@ -0,0 +1 @@
+.

jenkins.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+set -ex
+
+(cd event-schemas/ && ./check_examples.py)
+(cd api && ./check_examples.py)
+(cd scripts && ./gendoc.py)
+(cd api && npm install && node validator.js -s "client-server/v1")
+(cd event-schemas/ && ./check.sh)

scripts/codehighlight.css

@@ -0,0 +1,6 @@
+pre.code .comment, code .comment { color: green }
+pre.code .keyword, code .keyword { color: darkred; font-weight: bold }
+pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold }
+pre.code .literal.number, code .literal.number { color: blue }
+pre.code .name.tag, code .name.tag { color: darkgreen }
+pre.code .literal.string, code .literal.string { color: darkblue }

scripts/continuserv/main.go

@@ -56,7 +56,7 @@ func main() {
 	go doPopulate(ch, dir)
 
 	go watchFS(ch, w)
-
+	fmt.Printf("Listening on port %d\n", *port)
 	http.HandleFunc("/", serve)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
 

scripts/gendoc.py

@@ -1,22 +1,231 @@
 #! /usr/bin/env python
 
+from argparse import ArgumentParser
 from docutils.core import publish_file
+import copy
 import fileinput
 import glob
 import os
+import re
 import shutil
 import subprocess
 import sys
+import yaml
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 stylesheets = {
-    "stylesheet_path": ["basic.css", "nature.css"]
+    "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"]
 }
 
-def glob_spec_to(out_file_name):
+VERBOSE = False
-    with open(out_file_name, "wb") as outfile:
+
-        for f in sorted(glob.glob("../specification/*.rst")):
+"""
-            with open(f, "rb") as infile:
+Read a RST file and replace titles with a different title level if required.
-                outfile.write(infile.read())
+Args:
+    filename: The name of the file being read (for debugging)
+    file_stream: The open file stream to read from.
+    title_level: The integer which determines the offset to *start* from.
+    title_styles: An array of characters detailing the right title styles to use
+                  e.g. ["=", "-", "~", "+"]
+Returns:
+    string: The file contents with titles adjusted.
+Example:
+    Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file
+    when read line-by-line encounters the titles "===", "---", "---", "===", "---".
+    This function will bump every title encountered down a sub-heading e.g.
+    "=" to "-" and "-" to "~" because title_level = 1, so the output would be
+    "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level.
+"""
+def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
+    rst_lines = []
+    title_chars = "".join(title_styles)
+    title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$")
+
+    prev_line_title_level = 0 # We expect the file to start with '=' titles
+    file_offset = None
+    prev_non_title_line = None
+    for i, line in enumerate(file_stream, 1):
+        # ignore anything which isn't a title (e.g. '===============')
+        if not title_regex.match(line):
+            rst_lines.append(line)
+            prev_non_title_line = line
+            continue
+        # The title underline must match at a minimum the length of the title
+        if len(prev_non_title_line) > len(line):
+            rst_lines.append(line)
+            prev_non_title_line = line
+            continue
+
+        line_title_style = line[0]
+        line_title_level = title_styles.index(line_title_style)
+
+        # Not all files will start with "===" and we should be flexible enough
+        # to allow that. The first title we encounter sets the "file offset"
+        # which is added to the title_level desired.
+        if file_offset is None:
+            file_offset = line_title_level
+            if file_offset != 0:
+                logv(("     WARNING: %s starts with a title style of '%s' but '%s' " +
+                    "is preferable.") % (filename, line_title_style, title_styles[0]))
+
+        # Sanity checks: Make sure that this file is obeying the title levels
+        # specified and bail if it isn't.
+        # The file is allowed to go 1 deeper or any number shallower
+        if prev_line_title_level - line_title_level < -1:
+            raise Exception(
+                ("File '%s' line '%s' has a title " +
+                "style '%s' which doesn't match one of the " +
+                "allowed title styles of %s because the " +
+                "title level before this line was '%s'") %
+                (filename, (i + 1), line_title_style, title_styles,
+                title_styles[prev_line_title_level])
+            )
+        prev_line_title_level = line_title_level
+
+        adjusted_level = (
+            title_level + line_title_level - file_offset
+        )
+
+        # Sanity check: Make sure we can bump down the title and we aren't at the
+        # lowest level already
+        if adjusted_level >= len(title_styles):
+            raise Exception(
+                ("Files '%s' line '%s' has a sub-title level too low and it " +
+                "cannot be adjusted to fit. You can add another level to the " +
+                "'title_styles' key in targets.yaml to fix this.") %
+                (filename, (i + 1))
+            )
+
+        if adjusted_level == line_title_level:
+            # no changes required
+            rst_lines.append(line)
+            continue
+
+        # Adjusting line levels
+        logv(
+            "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" %
+            (filename, line_title_style, title_styles[adjusted_level],
+                file_offset, title_level)
+        )
+        rst_lines.append(line.replace(
+            line_title_style,
+            title_styles[adjusted_level]
+        ))
+            
+    return "".join(rst_lines)
+
+
+def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
+    # string are file paths to RST blobs
+    if isinstance(file_info, basestring):
+        log("%s %s" % (">" * (1 + title_level), file_info))
+        with open(os.path.join(spec_dir, file_info), "r") as f:
+            rst = None
+            if adjust_titles:
+                rst = load_with_adjusted_titles(
+                    file_info, f, title_level, title_styles
+                )
+            else:
+                rst = f.read()
+            if rst[-2:] != "\n\n":
+                raise Exception(
+                    ("File %s should end with TWO new-line characters to ensure " +
+                    "file concatenation works correctly.") % (file_info,)
+                )
+            return rst
+    # dicts look like {0: filepath, 1: filepath} where the key is the title level
+    elif isinstance(file_info, dict):
+        levels = sorted(file_info.keys())
+        rst = []
+        for l in levels:
+            rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles))
+        return "".join(rst)
+    # lists are multiple file paths e.g. [filepath, filepath]
+    elif isinstance(file_info, list):
+        rst = []
+        for f in file_info:
+            rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles))
+        return "".join(rst)
+    raise Exception(
+        "The following 'file' entry in this target isn't a string, list or dict. " +
+        "It really really should be. Entry: %s" % (file_info,)
+    )
+
+
+def build_spec(target, out_filename):
+    with open(out_filename, "wb") as outfile:
+        for file_info in target["files"]:
+            section = get_rst(
+                file_info=file_info,
+                title_level=0,
+                title_styles=target["title_styles"],
+                spec_dir="../specification/",
+                adjust_titles=True
+            )
+            outfile.write(section)
+
+
+"""
+Replaces relative title styles with actual title styles.
+
+The templating system has no idea what the right title style is when it produces
+RST because it depends on the build target. As a result, it uses relative title
+styles defined in targets.yaml to say "down a level, up a level, same level".
+
+This function replaces these relative titles with actual title styles from the
+array in targets.yaml.
+"""
+def fix_relative_titles(target, filename, out_filename):
+    title_styles = target["title_styles"]
+    relative_title_chars = [
+        target["relative_title_styles"]["subtitle"],
+        target["relative_title_styles"]["sametitle"],
+        target["relative_title_styles"]["supertitle"]
+    ]
+    relative_title_matcher = re.compile(
+        "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$"
+    )
+    title_matcher = re.compile(
+        "^[" + re.escape("".join(title_styles)) + "]{3,}$"
+    )
+    current_title_style = None
+    with open(filename, "r") as infile:
+        with open(out_filename, "w") as outfile:
+            for line in infile.readlines():
+                if not relative_title_matcher.match(line):
+                    if title_matcher.match(line):
+                        current_title_style = line[0]
+                    outfile.write(line)
+                    continue
+                line_char = line[0]
+                replacement_char = None
+                current_title_level = title_styles.index(current_title_style)
+                if line_char == target["relative_title_styles"]["subtitle"]:
+                    if (current_title_level + 1) == len(title_styles):
+                        raise Exception(
+                            "Encountered sub-title line style but we can't go " +
+                            "any lower."
+                        )
+                    replacement_char = title_styles[current_title_level + 1]
+                elif line_char == target["relative_title_styles"]["sametitle"]:
+                    replacement_char = title_styles[current_title_level]
+                elif line_char == target["relative_title_styles"]["supertitle"]:
+                    if (current_title_level - 1) < 0:
+                        raise Exception(
+                            "Encountered super-title line style but we can't go " +
+                            "any higher."
+                        )
+                    replacement_char = title_styles[current_title_level - 1]
+                else:
+                    raise Exception(
+                        "Unknown relative line char %s" % (line_char,)
+                    )
+
+                outfile.write(
+                    line.replace(line_char, replacement_char)
+                )
+
 
 
 def rst2html(i, o):
@@ -31,25 +240,127 @@ def rst2html(i, o):
                 settings_overrides=stylesheets
             )
 
-def run_through_template(input):
+
+def run_through_template(input, set_verbose):
     tmpfile = './tmp/output'
     try:
         with open(tmpfile, 'w') as out:
-            subprocess.check_output(
+            args = [
-                [
                 'python', 'build.py',
                 "-i", "matrix_templates",
                 "-o", "../scripts/tmp",
                 "../scripts/"+input
-                ],
+            ]
+            if set_verbose:
+                args.insert(2, "-v")
+            log("EXEC: %s" % " ".join(args))
+            log(" ==== build.py output ==== ")
+            print subprocess.check_output(
+                args,
                 stderr=out,
-                cwd="../templating",
+                cwd="../templating"
             )
     except subprocess.CalledProcessError as e:
         with open(tmpfile, 'r') as f:
-            print f.read()
+            sys.stderr.write(f.read() + "\n")
         raise
 
+
+"""
+Extract and resolve groups for the given target in the given targets listing.
+Args:
+    targets_listing (str): The path to a YAML file containing a list of targets
+    target_name (str): The name of the target to extract from the listings.
+Returns:
+    dict: Containing "filees" (a list of file paths), "relative_title_styles"
+          (a dict of relative style keyword to title character) and "title_styles"
+          (a list of characters which represent the global title style to follow,
+           with the top section title first, the second section second, and so on.)
+"""
+def get_build_target(targets_listing, target_name):
+    build_target = {
+        "title_styles": [],
+        "relative_title_styles": {},
+        "files": []
+    }
+    with open(targets_listing, "r") as targ_file:
+        all_targets = yaml.load(targ_file.read())
+
+    build_target["title_styles"] = all_targets["title_styles"]
+    build_target["relative_title_styles"] = all_targets["relative_title_styles"]
+    target = all_targets["targets"].get(target_name)
+    if not target:
+        raise Exception(
+            "No target by the name '" + target_name + "' exists in '" +
+            targets_listing + "'."
+        )
+    if not isinstance(target.get("files"), list):
+        raise Exception(
+            "Found target but 'files' key is not a list."
+        )
+
+    def get_group(group_id, depth):
+        group_name = group_id[len("group:"):]
+        group = all_targets.get("groups", {}).get(group_name)
+        if not group:
+            raise Exception(
+                "Tried to find group '%s' but it doesn't exist." % group_name
+            )
+        if not isinstance(group, list):
+            raise Exception(
+                "Expected group '%s' to be a list but it isn't." % group_name
+            )
+        # deep copy so changes to depths don't contaminate multiple uses of this group
+        group = copy.deepcopy(group)
+        # swap relative depths for absolute ones
+        for i, entry in enumerate(group):
+            if isinstance(entry, dict):
+                group[i] = {
+                    (rel_depth + depth): v for (rel_depth, v) in entry.items()
+                }
+        return group
+
+    resolved_files = []
+    for file_entry in target["files"]:
+        # file_entry is a group id
+        if isinstance(file_entry, basestring) and file_entry.startswith("group:"):
+            group = get_group(file_entry, 0)
+            # The group may be resolved to a list of file entries, in which case
+            # we want to extend the array to insert each of them rather than
+            # insert the entire list as a single element (which is what append does)
+            if isinstance(group, list):
+                resolved_files.extend(group)
+            else:
+                resolved_files.append(group)
+        # file_entry is a dict which has more file entries as values
+        elif isinstance(file_entry, dict):
+            resolved_entry = {}
+            for (depth, entry) in file_entry.iteritems():
+                if not isinstance(entry, basestring):
+                    raise Exception(
+                        "Double-nested depths are not supported. Entry: %s" % (file_entry,)
+                    )
+                if entry.startswith("group:"):
+                    resolved_entry[depth] = get_group(entry, depth)
+                else:
+                    # map across without editing (e.g. normal file path)
+                    resolved_entry[depth] = entry
+            resolved_files.append(resolved_entry)
+            continue
+        # file_entry is just a plain ol' file path
+        else:
+            resolved_files.append(file_entry)
+    build_target["files"] = resolved_files
+    return build_target
+
+def log(line):
+    print "gendoc: %s" % line
+
+def logv(line):
+    if VERBOSE:
+        print "gendoc:V: %s" % line
+
+
 def prepare_env():
     try:
         os.makedirs("./gen")
@@ -60,32 +371,48 @@ def prepare_env():
     except OSError:
         pass
 
+
 def cleanup_env():
     shutil.rmtree("./tmp")
 
-def main():
+
+def main(target_name, keep_intermediates):
     prepare_env()
-    glob_spec_to("tmp/full_spec.rst")
+    log("Building spec [target=%s]" % target_name)
-    run_through_template("tmp/full_spec.rst")
+    target = get_build_target("../specification/targets.yaml", target_name)
+    build_spec(target=target, out_filename="tmp/templated_spec.rst")
+    run_through_template("tmp/templated_spec.rst", VERBOSE)
+    fix_relative_titles(
+        target=target, filename="tmp/templated_spec.rst",
+        out_filename="tmp/full_spec.rst"
+    )
     shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
-    run_through_template("tmp/howto.rst")
+    run_through_template("tmp/howto.rst", False)  # too spammy to mark -v on this
     rst2html("tmp/full_spec.rst", "gen/specification.html")
     rst2html("tmp/howto.rst", "gen/howtos.html")
-    if "--nodelete" not in sys.argv:
+    if not keep_intermediates:
         cleanup_env()
 
+
 if __name__ == '__main__':
-    if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]:
+    parser = ArgumentParser(
-        # we accept almost no args, so they don't know what they're doing!
+        "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder."
-        print "gendoc.py - Generate the Matrix specification as HTML."
+    )
-        print "Usage:"
+    parser.add_argument(
-        print "  python gendoc.py [--nodelete]"
+        "--nodelete", "-n", action="store_true",
-        print ""
+        help="Do not delete intermediate files. They will be found in tmp/"
-        print "The specification can then be found in the gen/ folder."
+    )
-        print ("If --nodelete was specified, intermediate files will be "
+    parser.add_argument(
-               "present in the tmp/ folder.")
+        "--target", "-t", default="main",
-        print ""
+        help="Specify the build target to build from specification/targets.yaml"
-        print "Requirements:"
+    )
-        print " - This script requires Jinja2 and rst2html (docutils)."
+    parser.add_argument(
-        sys.exit(0)
+        "--verbose", "-v", action="store_true",
-    main()
+        help="Turn on verbose mode."
+    )
+    args = parser.parse_args()
+    if not args.target:
+        parser.print_help()
+        sys.exit(1)
+    VERBOSE = args.verbose
+    main(args.target, args.nodelete)

specification/0-event_signing.rst

@@ -1,8 +1,8 @@
 Signing Events
-==============
+--------------
 
 Canonical JSON
---------------
+~~~~~~~~~~~~~~
 
 Matrix events are represented using JSON objects. If we want to sign JSON
 events we need to encode the JSON as a binary string. Unfortunately the same
@@ -30,7 +30,7 @@ using this representation.
          value,
          # Encode code-points outside of ASCII as UTF-8 rather than \u escapes
          ensure_ascii=False,
-         # Remove unecessary white space.
+         # Remove unnecessary white space.
          separators=(',',':'),
          # Sort the keys of dictionaries.
          sort_keys=True,
@@ -38,7 +38,7 @@ using this representation.
      ).encode("UTF-8")
 
 Grammar
-~~~~~~~
++++++++
 
 Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing
 insignificant whitespace, fractions, exponents and redundant character escapes
@@ -69,14 +69,14 @@ insignificant whitespace, fractions, exponents and redundant character escapes
            / %x75.30.30.31 (%x30-39 / %x61-66)        ; u001X
 
 Signing JSON
-------------
+~~~~~~~~~~~~
 
 We can now sign a JSON object by encoding it as a sequence of bytes, computing
 the signature for that sequence and then adding the signature to the original
 JSON object.
 
 Signing Details
-~~~~~~~~~~~~~~~
++++++++++++++++
 
 JSON is signed by encoding the JSON object without ``signatures`` or keys grouped
 as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the
@@ -133,7 +133,7 @@ and additional signatures.
       return json_object
 
 Checking for a Signature
-~~~~~~~~~~~~~~~~~~~~~~~~
+++++++++++++++++++++++++
 
 To check if an entity has signed a JSON object a server does the following
 
@@ -151,7 +151,7 @@ To check if an entity has signed a JSON object a server does the following
    the check fails. Otherwise the check succeeds.
 
 Signing Events
---------------
+~~~~~~~~~~~~~~
 
 Signing events is a more complicated process since servers can choose to redact
 non-essential parts of an event. Before signing the event it is encoded as

specification/0-events.rst

@@ -0,0 +1,38 @@
+Events
+======
+
+All communication in Matrix is expressed in the form of data objects called
+Events. These are the fundamental building blocks common to the client-server,
+server-server and application-service APIs, and are described below.
+
+{{common_event_fields}}
+
+{{common_room_event_fields}}
+
+{{common_state_event_fields}}
+
+
+Room Events
+-----------
+.. NOTE::
+  This section is a work in progress.
+
+This specification outlines several standard event types, all of which are
+prefixed with ``m.``
+
+{{m_room_aliases_event}}
+
+{{m_room_canonical_alias_event}}
+
+{{m_room_create_event}}
+
+{{m_room_history_visibility_event}}
+
+{{m_room_join_rules_event}}
+
+{{m_room_member_event}}
+
+{{m_room_power_levels_event}}
+
+{{m_room_redaction_event}}
+

specification/0-feature_profiles.rst

@@ -0,0 +1,3 @@
+Feature Profiles
+================
+

specification/0-intro.rst

@@ -79,7 +79,7 @@ The functionality that Matrix provides includes:
 - Extensible user management (inviting, joining, leaving, kicking, banning)
   mediated by a power-level based user privilege system.
 - Extensible room state management (room naming, aliasing, topics, bans)
-- Extensible user profile management (avatars, displaynames, etc)
+- Extensible user profile management (avatars, display names, etc)
 - Managing user accounts (registration, login, logout)
 - Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
   Facebook accounts to authenticate, identify and discover users on Matrix.
@@ -91,7 +91,7 @@ The functionality that Matrix provides includes:
 The end goal of Matrix is to be a ubiquitous messaging layer for synchronising
 arbitrary data between sets of people, devices and services - be that for
 instant messages, VoIP call setups, or any other objects that need to be
-reliably and persistently pushed from A to B in an interoperable and federated
+reliably and persistently pushed from A to B in an inter-operable and federated
 manner.
 
 Overview
@@ -171,20 +171,21 @@ All data exchanged over Matrix is expressed as an "event". Typically each client
 action (e.g. sending a message) correlates with exactly one event. Each event
 has a ``type`` which is used to differentiate different kinds of data. ``type``
 values MUST be uniquely globally namespaced following Java's `package naming
-conventions
+conventions`_, e.g.
-<http://docs.oracle.com/javase/specs/jls/se5.0/html/packages.html#7.7>`, e.g.
 ``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved
 for events defined in the Matrix specification - for instance ``m.room.message``
 is the event type for instant messages. Events are usually sent in the context
 of a "Room".
 
+.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions
+
 Event Graphs
 ~~~~~~~~~~~~
 
 Events exchanged in the context of a room are stored in a directed acyclic graph
 (DAG) called an ``event graph``. The partial ordering of this graph gives the
 chronological ordering of events within the room. Each event in the graph has a
-list of zero or more ``parent`` events, which refer to any preceeding events
+list of zero or more ``parent`` events, which refer to any preceding events
 which have no chronological successor from the perspective of the homeserver
 which created the event.
 
@@ -367,7 +368,8 @@ room). An example of a non-proactive client activity would be a client setting
 key called ``last_active_ago``, which gives the relative number of milliseconds
 since the message is generated/emitted that the user was last seen active.
 
-N.B. in v1 API, status/online/idle state are muxed into a single 'presence' field on the m.presence event.
+N.B. in v1 API, status/online/idle state are muxed into a single 'presence'
+field on the ``m.presence`` event.
 
 Presence Lists
 ~~~~~~~~~~~~~~
@@ -385,7 +387,7 @@ Profiles
 
 Users may publish arbitrary key/value data associated with their account - such
 as a human readable ``display name``, a profile photo URL, contact information
-(email address, phone nubers, website URLs etc).
+(email address, phone numbers, website URLs etc).
 
 In Client-Server API v2, profile data is typed using namespaced keys for
 interoperability, much like events - e.g. ``m.profile.display_name``.
@@ -442,7 +444,7 @@ The ``error`` string will be a human-readable error message, usually a sentence
 explaining what went wrong. The ``errcode`` string will be a unique string
 which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error
 codes should have their namespace first in ALL CAPS, followed by a single _ to
-ease seperating the namespace from the error code.. For example, if there was a
+ease separating the namespace from the error code. For example, if there was a
 custom namespace ``com.mydomain.here``, and a
 ``FORBIDDEN`` code, the error code should look like
 ``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the

specification/1-client_server_api.rst

@@ -8,11 +8,11 @@ The client-server API provides a simple lightweight API to let clients send
 messages, control rooms and synchronise conversation history. It is designed to
 support both lightweight clients which store no state and lazy-load data from
 the server as required - as well as heavyweight clients which maintain a full
-local peristent copy of server state.
+local persistent copy of server state.
 
 This mostly describes v1 of the Client-Server API as featured in the original September
 2014 launch of Matrix, apart from user-interactive authentication where it is
-encouraged to move to V2, therefore this is the version documented here.
+encouraged to move to v2, therefore this is the version documented here.
 Version 2 is currently in development (as of Jan-March 2015) as an incremental
 but backwards-incompatible refinement of Version 1 and will be released
 shortly.
@@ -154,7 +154,7 @@ Matrix client, for example, an email confirmation may be completed when the user
 clicks on the link in the email. In this case, the client retries the request
 with an auth dict containing only the session key. The response to this will be
 the same as if the client were attempting to complete an auth state normally,
-ie. the request will either complete or request auth, with the presence or
+i.e. the request will either complete or request auth, with the presence or
 absence of that login stage type in the 'completed' array indicating whether
 that stage is complete.
 
@@ -204,7 +204,7 @@ Password-based
 :Type:
   ``m.login.password``
 :Description:
-  The client submits a username and secret password, both sent in plaintext.
+  The client submits a username and secret password, both sent in plain-text.
 
 To respond to this type, reply with an auth dict as follows::
 
@@ -247,10 +247,10 @@ service which the home server accepts when logging in, this indirection can be
 skipped and the "uri" key can be the ``Authorization Request URI``.
 
 The client then visits the ``Authorization Request URI``, which then shows the
-OAuth2 Allow/Deny prompt. Hitting 'Allow' returns the [XXX: redirects to the?]``redirect URI`` with the
+OAuth2 Allow/Deny prompt. Hitting 'Allow' redirects to the ``redirect URI`` with
-auth code. Home servers can choose any path for the ``redirect URI``. Once the
+the auth code. Home servers can choose any path for the ``redirect URI``. Once
-OAuth flow has completed, the client retries the request with the session only,
+the OAuth flow has completed, the client retries the request with the session
-as above.
+only, as above.
 
 Email-based (identity server)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -308,7 +308,7 @@ Where ``stage type`` is the type name of the stage it is attempting and
 ``session id`` is the ID of the session given by the home server.
 
 This MUST return an HTML page which can perform this authentication stage. This
-page must attempt to call the Javascript function ``window.onAuthDone`` when
+page must attempt to call the JavaScript function ``window.onAuthDone`` when
 the authentication has been completed.
 
 Pagination
@@ -373,7 +373,7 @@ now show page 3 (rooms R11 -> 15)::
                              Returns: R11,R12,R13,R14,R15
                          
 Note that tokens are treated in an *exclusive*, not inclusive, manner. The end 
-token from the intial request was '9' which corresponded to R10. When the 2nd
+token from the initial request was '9' which corresponded to R10. When the 2nd
 request was made, R10 did not appear again, even though from=9 was specified. If
 you know the token, you already have the data.
 
@@ -425,9 +425,9 @@ You can visualise the range of events being returned as::
                               |                             |
                         start: '1-2-3'                end: 'a-b-c'
                              
-Now, to receive future events in realtime on the eventstream, you simply GET
+Now, to receive future events in real-time on the eventstream, you simply GET
 $PREFIX/events with a ``from`` parameter of 'a-b-c': in other words passing in the
-``end`` token returned by initialsync. The request blocks until new events are
+``end`` token returned by initial sync. The request blocks until new events are
 available or until your specified timeout elapses, and then returns a
 new paginatable chunk of events alongside new start and end parameters::
 
@@ -467,7 +467,7 @@ event stream. When the request returns, an ``end`` token is included in the
 response. This token can be used in the next request to continue where the
 last request left off.
 
-All events must be deduplicated based on their event ID.
+All events must be de-duplicated based on their event ID.
 
 .. TODO
   is deduplication actually a hard requirement in CS v2?
@@ -493,7 +493,7 @@ Room events are split into two categories:
 
 :Message events:
   These are events which describe transient "once-off" activity in a room:
-  typically communication such as sending an instant messaage or setting up a
+  typically communication such as sending an instant message or setting up a
   VoIP call. These used to be called 'non-state' events.
 
 This specification outlines several events, all with the event type prefix
@@ -631,49 +631,7 @@ Getting events for a room
 
 There are several APIs provided to ``GET`` events for a room:
 
-``/rooms/<room id>/state/<event type>/<state key>``
+{{rooms_http_api}}
-  Description:
-    Get the state event identified.
-  Response format:
-    A JSON object representing the state event **content**.
-  Example:
-    ``/rooms/!room:domain.com/state/m.room.name`` returns ``{ "name": "Room name" }``
-
-|/rooms/<room_id>/state|_
-  Description:
-    Get all state events for a room.
-  Response format:
-    ``[ { state event }, { state event }, ... ]``
-  Example:
-    TODO-doc
-
-|/rooms/<room_id>/members|_
-  Description:
-    Get all ``m.room.member`` state events.
-  Response format:
-    ``{ "start": "<token>", "end": "<token>", "chunk": [ { m.room.member event }, ... ] }``
-  Example:
-    TODO-doc
-
-|/rooms/<room_id>/messages|_
-  Description:
-    Get all events from the room's timeline. This API supports
-    pagination using ``from`` and ``to`` query parameters, coupled with the
-    ``start`` and ``end`` tokens from an |initialSync|_ API.
-    
-  Response format:
-    ``{ "start": "<token>", "end": "<token>" }``
-  Example:
-    TODO-doc
-
-|/rooms/<room_id>/initialSync|_
-  Description:
-    Get all relevant events for a room. This includes state events, paginated
-    non-state events and presence events.
-  Response format:
-    `` { TODO-doc } ``
-  Example:
-    TODO-doc
 
 Redactions
 ~~~~~~~~~~
@@ -932,11 +890,8 @@ directly by sending the following request to
     "membership": "leave"
   }
 
-See the `Room events`_ section for more information on ``m.room.member``.
+See the `Room events`_ section for more information on ``m.room.member``. Once a
-
+user has left a room, that room will no longer appear on the |initialSync|_ API.
-Once a user has left a room, that room will no longer appear on the
-|initialSync|_ API.
-
 If all members in a room leave, that room becomes eligible for deletion.
 
 Banning users in a room
@@ -974,7 +929,7 @@ Registering for a user account is done using the request::
   POST $V2PREFIX/register
 
 This API endpoint uses the User-Interactive Authentication API.
-This API endoint does not require an access token.
+This API endpoint does not require an access token.
 
 The body of the POST request is a JSON object containing:
 
@@ -1011,32 +966,7 @@ was registered whilst the client was performing authentication.
 
 Old V1 API docs: |register|_
 
-Login
+{{login_http_api}}
-~~~~~
-This section refers to API Version 1.
-
-API docs: |login|_
-
-Obtaining an access token for an existing user account is done using the
-request::
-
-  POST $PREFIX/login
-
-The body of the POST request is a JSON object containing:
-
-username
-  The full qualified or local part of the Matrix ID to log in with.
-password
-  The password for the account.
-
-On success, this returns a JSON object with keys:
-
-user_id
-  The fully-qualified Matrix ID that has been registered.
-access_token
-  An access token for the new account.
-home_server
-  The hostname of the Home Server on which the account has been registered.
 
 Changing Password
 ~~~~~~~~~~~~~~~~~
@@ -1087,7 +1017,7 @@ The third party identifier credentials object comprises:
 
 id_server
   The colon-separated hostname and port of the Identity Server used to
-  authenticate the third party identifer. If the port is the default, it and the
+  authenticate the third party identifier. If the port is the default, it and the
   colon should be omitted.
 sid
   The session ID given by the Identity Server
@@ -1124,12 +1054,6 @@ medium
 address
   The textual address of the 3pid, eg. the email address
 
-Presence
---------
-.. TODO-spec
-  - Define how users receive presence invites, and how they accept/decline them
-
-{{presence_http_api}}
 
 Profiles
 --------

specification/2-modules.rst

@@ -0,0 +1,3 @@
+Modules
+=======
+

specification/20_events.rst

@@ -1,157 +0,0 @@
-Events
-======
-
-All communication in Matrix is expressed in the form of data objects called
-Events. These are the fundamental building blocks common to the client-server,
-server-server and application-service APIs, and are described below.
-
-{{common_event_fields}}
-
-{{common_room_event_fields}}
-
-{{common_state_event_fields}}
-
-
-Room Events
------------
-.. NOTE::
-  This section is a work in progress.
-
-This specification outlines several standard event types, all of which are
-prefixed with ``m.``
-
-{{room_events}}
-
-m.room.message msgtypes
-~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO-spec
-   How a client should handle unknown message types.
-
-
-Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
-of message being sent. Each type has their own required and optional keys, as
-outlined below.
-
-{{msgtype_events}}
-
-Presence Events
-~~~~~~~~~~~~~~~
-
-{{presence_events}}
- 
-Each user has the concept of presence information. This encodes the
-"availability" of that user, suitable for display on other user's clients.
-This is transmitted as an ``m.presence`` event and is one of the few events
-which are sent *outside the context of a room*. The basic piece of presence
-information is represented by the ``presence`` key, which is an enum of one
-of the following:
-
-      - ``online`` : The default state when the user is connected to an event
-        stream.
-      - ``unavailable`` : The user is not reachable at this time.
-      - ``offline`` : The user is not connected to an event stream.
-      - ``free_for_chat`` : The user is generally willing to receive messages
-        moreso than default.
-      - ``hidden`` : Behaves as offline, but allows the user to see the client
-        state anyway and generally interact with client features. (Not yet
-        implemented in synapse).
-
-In addition, the server maintains a timestamp of the last time it saw a
-pro-active event from the user; either sending a message to a room, or
-changing presence state from a lower to a higher level of availability
-(thus: changing state from ``unavailable`` to ``online`` counts as a
-proactive event, whereas in the other direction it will not). This timestamp
-is presented via a key called ``last_active_ago``, which gives the relative
-number of milliseconds since the message is generated/emitted that the user
-was last seen active.
-    
-
-Events on Change of Profile Information
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Because the profile displayname and avatar information are likely to be used in
-many places of a client's display, changes to these fields cause an automatic
-propagation event to occur, informing likely-interested parties of the new
-values. This change is conveyed using two separate mechanisms:
-
- - a ``m.room.member`` event is sent to every room the user is a member of,
-   to update the ``displayname`` and ``avatar_url``.
- - a ``m.presence`` presence status update is sent, again containing the new values of the
-   ``displayname`` and ``avatar_url`` keys, in addition to the required
-   ``presence`` key containing the current presence state of the user.
-
-Both of these should be done automatically by the home server when a user
-successfully changes their displayname or avatar URL fields.
-
-Additionally, when home servers emit room membership events for their own
-users, they should include the displayname and avatar URL fields in these
-events so that clients already have these details to hand, and do not have to
-perform extra roundtrips to query it.
-
-Voice over IP
--------------
-Matrix can also be used to set up VoIP calls. This is part of the core
-specification, although is at a relatively early stage. Voice (and video) over
-Matrix is built on the WebRTC 1.0 standard.
-
-Call events are sent to a room, like any other event. This means that clients
-must only send call events to rooms with exactly two participants as currently
-the WebRTC standard is based around two-party communication.
-
-{{voip_events}}
-
-Message Exchange
-~~~~~~~~~~~~~~~~
-A call is set up with messages exchanged as follows:
-
-::
-
-   Caller                   Callee
- m.call.invite ----------->
- m.call.candidate -------->
- [more candidates events]
-                         User answers call
-                  <------ m.call.answer
-               [...]
-                  <------ m.call.hangup
-
-Or a rejected call:
-
-::
-
-   Caller                   Callee
- m.call.invite ----------->
- m.call.candidate -------->
- [more candidates events]
-                        User rejects call
-                 <------- m.call.hangup
-
-Calls are negotiated according to the WebRTC specification.
-
-
-Glare
-~~~~~
-This specification aims to address the problem of two users calling each other
-at roughly the same time and their invites crossing on the wire. It is a far
-better experience for the users if their calls are connected if it is clear
-that their intention is to set up a call with one another.
-
-In Matrix, calls are to rooms rather than users (even if those rooms may only
-contain one other user) so we consider calls which are to the same room.
-
-The rules for dealing with such a situation are as follows:
-
- - If an invite to a room is received whilst the client is preparing to send an
-   invite to the same room, the client should cancel its outgoing call and
-   instead automatically accept the incoming call on behalf of the user.
- - If an invite to a room is received after the client has sent an invite to
-   the same room and is waiting for a response, the client should perform a
-   lexicographical comparison of the call IDs of the two calls and use the
-   lesser of the two calls, aborting the greater. If the incoming call is the
-   lesser, the client should accept this call on behalf of the user.
-
-The call setup should appear seamless to the user as if they had simply placed
-a call and the other party had accepted. Thusly, any media stream that had been
-setup for use on a call should be transferred and used for the call that
-replaces it.
-

specification/3-application_service_api.rst

@@ -66,13 +66,13 @@ An example HS configuration required to pass traffic to the AS is:
   application service is merely augmenting the room itself (e.g. providing
   logging or searching facilities).
 - Namespaces are represented by POSIX extended regular expressions, 
-  e.g.:
+  e.g:
 
 .. code-block:: yaml
 
    users:
      - exclusive: true
-       regex: @irc.freenode.net/.*
+       regex: @irc.freenode.net_.*
 
 
 Home Server -> Application Service API
@@ -326,7 +326,7 @@ but only if the application service has defined the namespace as ``exclusive``.
 
 ID conventions
 ~~~~~~~~~~~~~~
-.. NOTE::
+.. TODO-spec
   - Giving HSes the freedom to namespace still feels like the Right Thing here.
   - Exposing a public API provides the consistency which was the main complaint
     against namespacing.
@@ -345,7 +345,7 @@ types, including:
 - MSISDNs (``tel``)
 - Email addresses (``mailto``)
 - IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04)
-- XMPP (xep-0032)
+- XMPP (XEP-0032)
 - SIP URIs (RFC 3261)
 
 As a result, virtual user IDs SHOULD relate to their URI counterpart. This
@@ -401,21 +401,3 @@ client from which the event originated.  For instance, this could contain the
 message-ID for emails/nntp posts, or a link to a blog comment when gatewaying
 blog comment traffic in & out of matrix
 
-Active Application Services
-----------------------------
-
-.. TODO-spec
-  API that provides hooks into the server so that you can intercept and
-  manipulate events, and/or insert virtual users & rooms into the server.
-
-Policy Servers
-==============
-.. NOTE::
-  This section is a work in progress.
-
-.. TODO-spec
-  We should mention them in the Architecture section at least: how they fit
-  into the picture.
-
-Enforcing policies
-------------------

specification/4-server_server_api.rst

@@ -2,10 +2,9 @@ Federation API
 ==============
 
 Matrix home servers use the Federation APIs (also known as server-server APIs)
-to communicate with each other.
+to communicate with each other. Home servers use these APIs to push messages to
-Home servers use these APIs to push messages to each other in real-time, to
+each other in real-time, to request historic messages from each other, and to
-request historic messages from each other, and to query profile and presence
+query profile and presence information about users on each other's servers.
-information about users on each other's servers.
 
 The APIs are implemented using HTTPS GETs and PUTs between each of the
 servers. These HTTPS requests are strongly authenticated using public key
@@ -21,7 +20,7 @@ Persisted Data Units (PDUs):
     context.
 
     Like email, it is the responsibility of the originating server of a PDU
-    to deliver that event to its recepient servers. However PDUs are signed
+    to deliver that event to its recipient servers. However PDUs are signed
     using the originating server's public key so that it is possible to
     deliver them through third-party servers.
 
@@ -84,18 +83,19 @@ directly or by querying an intermediate notary server using a
 response with their own key. A server may query multiple notary servers to
 ensure that they all report the same public keys.
 
-This approach is borrowed from the Perspectives Project
+This approach is borrowed from the `Perspectives Project`_, but modified to
-(http://perspectives-project.org/), but modified to include the NACL keys and to
+include the NACL keys and to use JSON instead of XML. It has the advantage of
-use JSON instead of XML. It has the advantage of avoiding a single trust-root
+avoiding a single trust-root since each server is free to pick which notary
-since each server is free to pick which notary servers they trust and can
+servers they trust and can corroborate the keys returned by a given notary
-corroborate the keys returned by a given notary server by querying other
+server by querying other servers.
-servers.
+
+.. _Perspectives Project: http://perspectives-project.org/
 
 Publishing Keys
-_______________
+^^^^^^^^^^^^^^^
 
 Home servers publish the allowed TLS fingerprints and signing keys in a JSON
-object at ``/_matrix/key/v2/server/${key_id}``. The response contains a list of
+object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of
 ``verify_keys`` that are valid for signing federation requests made by the
 server and for signing events. It contains a list of ``old_verify_keys``
 which are only valid for signing events. Finally the response contains a list
@@ -178,7 +178,7 @@ events sent by that server can still be checked.
     }
 
 Querying Keys Through Another Server
-____________________________________
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys
 for another server. This API can be used to GET at list of JSON objects for a
@@ -510,7 +510,7 @@ To backfill events on a given context::
 
 Retrieves a sliding-window history of previous PDUs that occurred on the given
 context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that
-preceeded it are retrieved, up to a total number given by the "limit" argument.
+preceded it are retrieved, up to a total number given by the "limit" argument.
 These are then returned in a new Transaction containing all of the PDUs.
 
 
@@ -554,9 +554,7 @@ Every HTTP request made by a homeserver is authenticated using public key
 digital signatures. The request method, target and body are signed by wrapping
 them in a JSON object and signing it using the JSON signing algorithm. The
 resulting signatures are added as an Authorization header with an auth scheme
-of X-Matrix.
+of X-Matrix. Note that the target field should include the full path starting with
-
-Note that the target field should include the full path starting with
 ``/_matrix/...``, including the ``?`` and any query parameters if present, but
 should not include the leading ``https:``, nor the destination server's
 hostname.
@@ -656,12 +654,12 @@ State Conflict Resolution
   - How does this work with deleting current state
   - How do we reject invalid federation traffic?
 
-[[TODO(paul): At this point we should probably have a long description of how
+  [[TODO(paul): At this point we should probably have a long description of how
-State management works, with descriptions of clobbering rules, power levels, etc
+  State management works, with descriptions of clobbering rules, power levels, etc
-etc... But some of that detail is rather up-in-the-air, on the whiteboard, and
+  etc... But some of that detail is rather up-in-the-air, on the whiteboard, and
-so on. This part needs refining. And writing in its own document as the details
+  so on. This part needs refining. And writing in its own document as the details
-relate to the server/system as a whole, not specifically to server-server
+  relate to the server/system as a whole, not specifically to server-server
-federation.]]
+  federation.]]
 
 Presence
 --------
@@ -677,8 +675,8 @@ Performing a presence update and poll subscription request::
       Each should be an object with the following keys:
         user_id: string containing a User ID
         presence: "offline"|"unavailable"|"online"|"free_for_chat"
-        status_msg: (optional) string of freeform text
+        status_msg: (optional) string of free-form text
-        last_active_ago: miliseconds since the last activity by the user
+        last_active_ago: milliseconds since the last activity by the user
 
     poll: (optional): list of strings giving User IDs
 
@@ -696,7 +694,7 @@ removed until explicitly requested by a later ``unpoll``.
 On receipt of a message containing a non-empty ``poll`` list, the receiving
 server should immediately send the sending server a presence update EDU of its
 own, containing in a ``push`` list the current state of every user that was in
-the orginal EDU's ``poll`` list.
+the original EDU's ``poll`` list.
 
 Sending a presence invite::
 
@@ -721,7 +719,7 @@ Rejecting a presence invite::
   Content keys - as for m.presence_invite
 
 .. TODO-doc
-  - Explain the timing-based roundtrip reduction mechanism for presence
+  - Explain the timing-based round-trip reduction mechanism for presence
     messages
   - Explain the zero-byte presence inference logic
   See also: docs/client-server/model/presence
@@ -742,8 +740,8 @@ Querying profile information::
     field: (optional) string giving a field name
 
   Returns: JSON object containing the following keys:
-    displayname: string of freeform text
+    displayname: string of free-form text
-    avatar_url: string containing an http-scheme URL
+    avatar_url: string containing an HTTP-scheme URL
 
 If the query contains the optional ``field`` key, it should give the name of a
 result field. If such is present, then the result should contain only a field
@@ -769,3 +767,4 @@ Querying directory information::
 The list of join candidates is a list of server names that are likely to hold
 the given room; these are servers that the requesting server may wish to try
 joining with. This list may or may not include the server answering the query.
+

specification/5-identity_servers.rst

@@ -0,0 +1,8 @@
+Identity Servers
+================
+.. NOTE::
+  This section is a work in progress.
+
+.. TODO-doc Dave
+  - 3PIDs and identity server, functions
+

specification/6-appendices.rst

@@ -128,11 +128,3 @@ Threat: Disclosure to Servers Within Chatroom
 An attacker could take control of a server within a chatroom to expose message
 contents or metadata for messages in that room.
 
-
-Identity Servers
-================
-.. NOTE::
-  This section is a work in progress.
-
-.. TODO-doc Dave
-  - 3PIDs and identity server, functions

specification/modules/content_repo.rst

@@ -39,10 +39,10 @@ thumbnailing method::
 
        <thumbnail>
 
-The thumbnail methods are "crop" and "scale". "scale" trys to return an
+The thumbnail methods are "crop" and "scale". "scale" tries to return an
 image where either the width or the height is smaller than the requested
 size. The client should then scale and letterbox the image if it needs to
-fit within a given rectangle. "crop" trys to return an image where the
+fit within a given rectangle. "crop" tries to return an image where the
 width and height are close to the requested size and the aspect matches
 the requested size. The client should scale the image if it needs to fit
 within a given rectangle.
@@ -53,8 +53,8 @@ the content. Homeservers may return thumbnails of a different size to that
 requested. However homeservers should provide exact matches where reasonable.
 Homeservers must never upscale images.
 
-Security
+Security considerations
---------
+-----------------------
 
 Clients may try to upload very large files. Homeservers should not store files
 that are too large and should not serve them to clients.

specification/modules/history_visibility.rst

@@ -1,5 +1,5 @@
 Room History Visibility
-=======================
+-----------------------
 
 Whether a member of a room can see the events that happened in a room from
 before they joined the room is controlled by the ``history_visibility`` key

specification/modules/instant_messaging.rst

@@ -0,0 +1,27 @@
+Instant Messaging
+=================
+
+Events
+------
+
+{{m_room_message_event}}
+
+{{m_room_message_feedback_event}}
+
+{{m_room_name_event}}
+
+{{m_room_topic_event}}
+
+m.room.message msgtypes
+-----------------------
+
+.. TODO-spec
+   How a client should handle unknown message types.
+
+
+Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
+of message being sent. Each type has their own required and optional keys, as
+outlined below.
+
+{{msgtype_events}}
+

specification/modules/presence.rst

@@ -0,0 +1,63 @@
+Presence
+========
+ 
+Each user has the concept of presence information. This encodes the
+"availability" of that user, suitable for display on other user's clients.
+This is transmitted as an ``m.presence`` event and is one of the few events
+which are sent *outside the context of a room*. The basic piece of presence
+information is represented by the ``presence`` key, which is an enum of one
+of the following:
+
+      - ``online`` : The default state when the user is connected to an event
+        stream.
+      - ``unavailable`` : The user is not reachable at this time.
+      - ``offline`` : The user is not connected to an event stream.
+      - ``free_for_chat`` : The user is generally willing to receive messages
+        moreso than default.
+      - ``hidden`` : Behaves as offline, but allows the user to see the client
+        state anyway and generally interact with client features. (Not yet
+        implemented in synapse).
+
+In addition, the server maintains a timestamp of the last time it saw a
+pro-active event from the user; either sending a message to a room, or
+changing presence state from a lower to a higher level of availability
+(thus: changing state from ``unavailable`` to ``online`` counts as a
+proactive event, whereas in the other direction it will not). This timestamp
+is presented via a key called ``last_active_ago``, which gives the relative
+number of milliseconds since the message is generated/emitted that the user
+was last seen active.
+
+Events
+------
+
+{{presence_events}}
+
+Presence HTTP API
+-----------------
+.. TODO-spec
+  - Define how users receive presence invites, and how they accept/decline them
+
+{{presence_http_api}}
+    
+
+Events on Change of Profile Information
+---------------------------------------
+Because the profile displayname and avatar information are likely to be used in
+many places of a client's display, changes to these fields cause an automatic
+propagation event to occur, informing likely-interested parties of the new
+values. This change is conveyed using two separate mechanisms:
+
+ - a ``m.room.member`` event is sent to every room the user is a member of,
+   to update the ``displayname`` and ``avatar_url``.
+ - a ``m.presence`` presence status update is sent, again containing the new values of the
+   ``displayname`` and ``avatar_url`` keys, in addition to the required
+   ``presence`` key containing the current presence state of the user.
+
+Both of these should be done automatically by the home server when a user
+successfully changes their displayname or avatar URL fields.
+
+Additionally, when home servers emit room membership events for their own
+users, they should include the displayname and avatar URL fields in these
+events so that clients already have these details to hand, and do not have to
+perform extra roundtrips to query it.
+

specification/modules/push_cs_api.rst

@@ -70,7 +70,7 @@ Room Rules
 Sender
   These rules configure notification behaviour for messages from a specific,
   named Matrix user ID. The rule_id of Sender rules is always the Matrix user
-  ID of the user whose messages theyt apply to.
+  ID of the user whose messages they'd apply to.
 Underride
   These are identical to override rules, but have a lower priority than content,
   room and sender rules.
@@ -112,7 +112,7 @@ In addition, all rules may be enabled or disabled. Disabled rules never match.
 
 If no rules match an event, the Home Server should not notify for the message
 (that is to say, the default action is "dont-notify"). Events that the user sent
-themself are never alerted for.
+themselves are never alerted for.
 
 Predefined Rules
 ----------------
@@ -128,7 +128,7 @@ with these IDs, their semantics should match those given below:
 
     {
         "rule_id": ".m.rule.contains_user_name"
-        "pattern": "[the lcoal part of the user's Matrix ID]",
+        "pattern": "[the local part of the user's Matrix ID]",
         "actions": [
             "notify",
             {
@@ -220,7 +220,7 @@ with these IDs, their semantics should match those given below:
 Push Rules: Actions:
 --------------------
 All rules have an associated list of 'actions'. An action affects if and how a
-notification is delievered for a matching event. This standard defines the
+notification is delivered for a matching event. This standard defines the
 following actions, although if Home servers wish to support more, they are free
 to do so:
 
@@ -241,11 +241,11 @@ set_tweak
 
 Actions that have no parameters are represented as a string. Otherwise, they are
 represented as a dictionary with a key equal to their name and other keys as
-their parameters, eg. { "set_tweak": "sound", "value": "default" }
+their parameters, e.g. ``{ "set_tweak": "sound", "value": "default" }``
 
 Push Rules: Actions: Tweaks
 ---------------------------
-The 'set_tweak' key action is used to add an entry to the 'tweaks' dictionary
+The ``set_tweak`` key action is used to add an entry to the 'tweaks' dictionary
 that is sent in the notification poke. The following tweaks are defined:
 
 sound
@@ -275,7 +275,7 @@ do so:
 
 event_match
   This is a glob pattern match on a field of the event. Parameters:
-   * 'key': The dot-separated field of the event to match, eg. content.body
+   * 'key': The dot-separated field of the event to match, e.g. content.body
    * 'pattern': The glob-style pattern to match against. Patterns with no
                 special glob characters should be treated as having asterisks
                 prepended and appended when testing the condition.
@@ -295,7 +295,7 @@ room_member_count
      '>=' or '<='. A prefix of '<' matches rooms where the member count is
      strictly less than the given number and so forth. If no prefix is present,
      this matches rooms where the member count is exactly equal to the given
-     number (ie. the same as '==').
+     number (i.e. the same as '==').
 
 Room, Sender, User and Content rules do not have conditions in the same way,
 but instead have predefined conditions, the behaviour of which can be configured
@@ -314,7 +314,7 @@ scope
   Either 'global' or 'device/<profile_tag>' to specify global rules or
   device rules for the given profile_tag.
 kind
-  The kind of rule, ie. 'override', 'underride', 'sender', 'room', 'content'.
+  The kind of rule, i.e. 'override', 'underride', 'sender', 'room', 'content'.
 rule_id
   The identifier for the rule.
 
@@ -330,7 +330,7 @@ after
   rule.
 
 All requests to the push rules API also require an access_token as a query
-paraemter.
+parameter.
 
 The content of the PUT request is a JSON object with a list of actions under the
 'actions' key and either conditions (under the 'conditions' key) or the

specification/modules/push_push_gw_api.rst

@@ -1,7 +1,7 @@
 HTTP Notification Protocol
 --------------------------
 
-This describes the format used by "http" pushers to send notifications of
+This describes the format used by "HTTP" pushers to send notifications of
 events.
 
 Notifications are sent as HTTP POST requests to the URL configured when the
@@ -77,10 +77,10 @@ counts
   This is a dictionary of the current number of unacknowledged communications
   for the recipient user. Counts whose value is zero are omitted.
 unread
-  The number of unread messages a user has accross all of the rooms they are a
+  The number of unread messages a user has across all of the rooms they are a
   member of.
 missed_calls
-  The number of unacknowledged missed calls a user has accross all rooms of
+  The number of unacknowledged missed calls a user has across all rooms of
   which they are a member.
 device
   This is an array of devices that the notification should be sent to.
@@ -104,13 +104,13 @@ And additional key is defined but only present on member events:
 
 user_is_target
   This is true if the user receiving the notification is the subject of a member
-  event (ie. the state_key of the member event is equal to the user's Matrix
+  event (i.e. the state_key of the member event is equal to the user's Matrix
   ID).
 
 The recipient of an HTTP notification should respond with an HTTP 2xx response
 when the notification has been processed. If the endpoint returns an HTTP error
 code, the Home Server should retry for a reasonable amount of time with a
-reasonable backoff scheme.
+reasonable back-off scheme.
 
 The endpoint should return a JSON dictionary as follows::
 

specification/modules/receipts.rst

@@ -1,14 +1,13 @@
 Receipts
-========
+--------
 
 Receipts are used to publish which events in a room the user or their devices
-have interacted with. For example, which events the user has read.
+have interacted with. For example, which events the user has read. For
-
+efficiency this is done as "up to" markers, i.e. marking a particular event
-For efficiency this is done as "up to" markers, i.e. marking a particular event
 as, say, ``read`` indicates the user has read all events *up to* that event.
 
 Client-Server API
------------------
+~~~~~~~~~~~~~~~~~
 
 Clients will receive receipts in the following format::
 
@@ -43,14 +42,11 @@ For example::
     }
 
 For efficiency, receipts are batched into one event per room. In the initialSync
-and v2 sync APIs the receipts are listed in a seperate top level ``receipts``
+and v2 sync APIs the receipts are listed in a separate top level ``receipts``
-key.
+key. Each ``user_id``, ``receipt_type`` pair must be associated with only a
-
+single ``event_id``. New receipts that come down the event streams are deltas.
-Each ``user_id``, ``receipt_type`` pair must be associated with only a single
+Deltas update existing mappings, clobbering based on ``user_id``,
-``event_id``.
+``receipt_type`` pairs.
-
-New receipts that come down the event streams are deltas. Deltas update
-existing mappings, clobbering based on ``user_id``, ``receipt_type`` pairs.
 
 
 A client can update the markers for its user by issuing a request::
@@ -62,7 +58,7 @@ other users. The server will automatically set the ``ts`` field.
 
 
 Server-Server API
------------------
+~~~~~~~~~~~~~~~~~
 
 Receipts are sent across federation as EDUs with type ``m.receipt``. The
 format of the EDUs are::
@@ -77,5 +73,5 @@ format of the EDUs are::
         ...
     }
 
-These are always sent as deltas to previously sent reciepts.
+These are always sent as deltas to previously sent receipts.
 

specification/modules/typing_notifications.rst

@@ -1,25 +1,27 @@
 Typing Notifications
-====================
+--------------------
 
 Client APIs
------------
+~~~~~~~~~~~
 
 To set "I am typing for the next N msec"::
+
   PUT .../rooms/<room_id>/typing/<user_id>
   Content:  { "typing": true, "timeout": N }
-  # timeout is in msec; I suggest no more than 20 or 30 seconds
+  # timeout is in milliseconds; suggested no more than 20 or 30 seconds
 
 This should be re-sent by the client to continue informing the server the user
-is still typing; I suggest a safety margin of 5 seconds before the expected
+is still typing; a safety margin of 5 seconds before the expected
-timeout runs out. Just keep declaring a new timeout, it will replace the old
+timeout runs out is recommended. Just keep declaring a new timeout, it will
-one.
+replace the old one.
 
 To set "I am no longer typing"::
+
   PUT ../rooms/<room_id>/typing/<user_id>
   Content: { "typing": false }
 
 Client Events
--------------
+~~~~~~~~~~~~~
 
 All room members will receive an event on the event stream::
 
@@ -37,7 +39,7 @@ users who are not currently typing, as that list gets big quickly. The client
 should mark as not typing, any user ID who is not in that list.
 
 Server APIs
------------
+~~~~~~~~~~~
 
 Servers will emit EDUs in the following form::
 
@@ -46,13 +48,14 @@ Servers will emit EDUs in the following form::
     "content": {
       "room_id": "!room-id-here:matrix.org",
       "user_id": "@user-id-here:matrix.org",
-      "typing": true/false,
+      "typing": true/false
     }
   }
 
 Server EDUs don't (currently) contain timing information; it is up to
 originating HSes to ensure they eventually send "stop" notifications.
 
-((This will eventually need addressing, as part of the wider typing/presence
+.. TODO
-timer addition work))
+  ((This will eventually need addressing, as part of the wider typing/presence
+  timer addition work))
 

specification/modules/voip_events.rst

@@ -0,0 +1,66 @@
+Voice over IP
+-------------
+Matrix can also be used to set up VoIP calls. This is part of the core
+specification, although is at a relatively early stage. Voice (and video) over
+Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like
+any other event. This means that clients must only send call events to rooms
+with exactly two participants as currently the WebRTC standard is based around
+two-party communication.
+
+{{voip_events}}
+
+Message Exchange
+~~~~~~~~~~~~~~~~
+A call is set up with messages exchanged as follows:
+
+::
+
+   Caller                    Callee
+ [Place Call]
+ m.call.invite ----------->
+ m.call.candidate -------->
+ [..candidates..] -------->
+                           [Answers call]
+          <--------------- m.call.answer
+    [Call is active and ongoing]
+          <--------------- m.call.hangup
+
+Or a rejected call:
+
+::
+
+   Caller                      Callee
+ m.call.invite ------------>
+ m.call.candidate --------->
+ [..candidates..] --------->
+                            [Rejects call]
+            <-------------- m.call.hangup
+
+Calls are negotiated according to the WebRTC specification.
+
+
+Glare
+~~~~~
+This specification aims to address the problem of two users calling each other
+at roughly the same time and their invites crossing on the wire. It is a far
+better experience for the users if their calls are connected if it is clear
+that their intention is to set up a call with one another.
+
+In Matrix, calls are to rooms rather than users (even if those rooms may only
+contain one other user) so we consider calls which are to the same room. The
+rules for dealing with such a situation are as follows:
+
+ - If an invite to a room is received whilst the client is preparing to send an
+   invite to the same room, the client should cancel its outgoing call and
+   instead automatically accept the incoming call on behalf of the user.
+ - If an invite to a room is received after the client has sent an invite to
+   the same room and is waiting for a response, the client should perform a
+   lexicographical comparison of the call IDs of the two calls and use the
+   lesser of the two calls, aborting the greater. If the incoming call is the
+   lesser, the client should accept this call on behalf of the user.
+
+The call setup should appear seamless to the user as if they had simply placed
+a call and the other party had accepted. Thusly, any media stream that had been
+setup for use on a call should be transferred and used for the call that
+replaces it.
+

specification/targets.yaml

@@ -0,0 +1,40 @@
+targets:
+  main:  # arbitrary name to identify this build target
+    files:  # the sort order of files to cat
+      - 0-intro.rst
+      - { 1: 0-feature_profiles.rst }
+      - 1-client_server_api.rst
+      - { 1: 0-events.rst }
+      - { 1: 0-event_signing.rst }
+      - 2-modules.rst
+      - { 1: "group:modules" }  # reference a group of files
+      - 3-application_service_api.rst
+      - 4-server_server_api.rst
+      - 5-identity_servers.rst
+      - 6-appendices.rst
+groups:  # reusable blobs of files when prefixed with 'group:'
+  modules:
+    - modules/instant_messaging.rst
+    - modules/voip_events.rst
+    - modules/typing_notifications.rst
+    - modules/receipts.rst
+    - modules/presence.rst
+    - modules/content_repo.rst
+    - modules/end_to_end_encryption.rst
+    - modules/history_visibility.rst
+    - modules/push_overview.rst
+      # relative depth
+    - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] }
+
+title_styles: ["=", "-", "~", "+", "^"]
+
+# The templating system doesn't know the right title style to use when generating
+# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title
+# at the same level (0)", or "make a title one above (+1)". The gendoc script
+# will inspect this file and replace these relative styles with actual title
+# styles. The templating system will also inspect this file to know which symbols
+# to inject.
+relative_title_styles:
+  subtitle: "<"
+  sametitle: "/"
+  supertitle: ">"

templating/batesian/sections.py

@@ -16,7 +16,7 @@ class Sections(object):
 
     def log(self, text):
         if self.debug:
-            print text
+            print "batesian:sections: %s" % text
 
     def get_sections(self):
         render_list = inspect.getmembers(self, predicate=inspect.ismethod)
@@ -27,12 +27,38 @@ class Sections(object):
             section_key = func_name[len("render_"):]
             self.log("Generating section '%s'" % section_key)
             section = func()
-            if not isinstance(section, basestring):
+            if isinstance(section, basestring):
+                if section_key in section_dict:
                     raise Exception(
-                    "Section function '%s' didn't return a string!" % func_name
+                        ("%s : Section %s already exists. It must have been " +
+                        "generated dynamically. Check which render_ methods " +
+                        "return a dict.") %
+                        (func_name, section_key)
                     )
                 section_dict[section_key] = section
                 self.log(
                     "  Generated. Snippet => %s" % section[:60].replace("\n","")
                 )
+            elif isinstance(section, dict):
+                self.log("  Generated multiple sections:")
+                for (k, v) in section.iteritems():
+                    if not isinstance(k, basestring) or not isinstance(v, basestring):
+                        raise Exception(
+                            ("Method %s returned multiple sections as a dict but " +
+                            "expected the dict elements to be strings but they aren't.") %
+                            (func_name, )
+                        )
+                    if k in section_dict:
+                        raise Exception(
+                            "%s tried to produce section %s which already exists." %
+                            (func_name, k)
+                        )
+                    section_dict[k] = v
+                    self.log(
+                        "  %s  =>  %s" % (k, v[:60].replace("\n",""))
+                    )
+            else:
+                raise Exception(
+                    "Section function '%s' didn't return a string/dict!" % func_name
+                )
         return section_dict

templating/batesian/units.py

@@ -22,7 +22,11 @@ class Units(object):
 
     def log(self, text):
         if self.debug:
-            print text
+            func_name = ""
+            trace = inspect.stack()
+            if len(trace) > 1 and len(trace[1]) > 2:
+                func_name = trace[1][3] + ":"
+            print "batesian:units:%s %s" % (func_name, text)
 
     def get_units(self, debug=False):
         unit_list = inspect.getmembers(self, predicate=inspect.ismethod)

templating/build.py

@@ -52,8 +52,8 @@ def create_from_template(template, sections):
 def check_unaccessed(name, store):
     unaccessed_keys = store.get_unaccessed_set()
     if len(unaccessed_keys) > 0:
-        print "Found %s unused %s keys." % (len(unaccessed_keys), name)
+        log("Found %s unused %s keys." % (len(unaccessed_keys), name))
-        print unaccessed_keys
+        log(unaccessed_keys)
 
 def main(input_module, file_stream=None, out_dir=None, verbose=False):
     if out_dir and not os.path.exists(out_dir):
@@ -121,17 +121,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
         return
 
     # check the input files and substitute in sections where required
-    print "Parsing input template: %s" % file_stream.name
+    log("Parsing input template: %s" % file_stream.name)
     temp = Template(file_stream.read())
-    print "Creating output for: %s" % file_stream.name
+    log("Creating output for: %s" % file_stream.name)
     output = create_from_template(temp, sections)
     with open(
             os.path.join(out_dir, os.path.basename(file_stream.name)), "w"
             ) as f:
         f.write(output)
-    print "Output file for: %s" % file_stream.name
+    log("Output file for: %s" % file_stream.name)
     check_unaccessed("units", units)
 
+def log(line):
+    print "batesian: %s" % line
 
 if __name__ == '__main__':
     parser = ArgumentParser(
@@ -175,7 +177,7 @@ if __name__ == '__main__':
         sys.exit(0)
 
     if not args.file:
-        print "No file supplied."
+        log("No file supplied.")
         parser.print_help()
         sys.exit(1)
 

templating/matrix_templates/sections.py

@@ -23,10 +23,13 @@ class MatrixSections(Sections):
         spec_meta = self.units.get("spec_meta")
         return spec_meta["changelog"]
 
-    def _render_events(self, filterFn, sortFn, title_kind="~"):
+    def _render_events(self, filterFn, sortFn):
         template = self.env.get_template("events.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         for event_name in sortFn(schemas):
             if not filterFn(event_name):
@@ -34,14 +37,16 @@ class MatrixSections(Sections):
             sections.append(template.render(
                 example=examples[event_name], 
                 event=schemas[event_name],
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
-    def _render_http_api_group(self, group, sortFnOrPathList=None, 
+    def _render_http_api_group(self, group, sortFnOrPathList=None):
-                               title_kind="-"):
         template = self.env.get_template("http-api.tmpl")
         http_api = self.units.get("swagger_apis")[group]["__meta"]
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         endpoints = []
         if sortFnOrPathList:
@@ -67,34 +72,40 @@ class MatrixSections(Sections):
         for endpoint in endpoints:
             sections.append(template.render(
                 endpoint=endpoint,
-                title_kind=title_kind
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
-    def render_profile_http_api(self):
-        return self._render_http_api_group(
-            "profile", 
-            sortFnOrPathList=["displayname", "avatar_url"],
-            title_kind="~"
-        )
-
-    def render_sync_http_api(self):
-        return self._render_http_api_group(
-            "sync"
-        )
 
-    def render_presence_http_api(self):
+    # Special function: Returning a dict will specify multiple sections where
-        return self._render_http_api_group(
+    # the key is the section name and the value is the value of the section
-            "presence",
+    def render_group_http_apis(self):
-            sortFnOrPathList=["status"],
+        # map all swagger_apis to the form $GROUP_http_api
-            title_kind="~"
+        swagger_groups = self.units.get("swagger_apis").keys()
+        renders = {}
+        for group in swagger_groups:
+            sortFnOrPathList = None
+            if group == "presence":
+                sortFnOrPathList = ["status"]
+            elif group == "profile":
+                sortFnOrPathList=["displayname", "avatar_url"]
+            renders[group + "_http_api"] = self._render_http_api_group(
+                group, sortFnOrPathList
             )
+        return renders
 
-    def render_membership_http_api(self):
+    # Special function: Returning a dict will specify multiple sections where
-        return self._render_http_api_group(
+    # the key is the section name and the value is the value of the section
-            "membership",
+    def render_group_events(self):
-            title_kind="~"
+        # map all event schemata to the form $EVENTTYPE_event with s/./_/g
+        # e.g. m_room_topic_event
+        schemas = self.units.get("event_schemas")
+        renders = {}
+        for event_type in schemas:
+            renders[event_type.replace(".", "_") + "_event"] = self._render_events(
+                lambda x: x == event_type, sorted
             )
+        return renders
 
     def render_room_events(self):
         def filterFn(eventType):
@@ -108,6 +119,9 @@ class MatrixSections(Sections):
         template = self.env.get_template("msgtypes.tmpl")
         examples = self.units.get("event_examples")
         schemas = self.units.get("event_schemas")
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
         sections = []
         msgtype_order = [
             "m.room.message#m.text", "m.room.message#m.emote",
@@ -123,7 +137,8 @@ class MatrixSections(Sections):
                 continue
             sections.append(template.render(
                 example=examples[event_name], 
-                event=schemas[event_name]
+                event=schemas[event_name],
+                title_kind=subtitle_title_char
             ))
         return "\n\n".join(sections)
 
@@ -144,12 +159,17 @@ class MatrixSections(Sections):
     def render_presence_events(self):
         def filterFn(eventType):
             return eventType.startswith("m.presence")
-        return self._render_events(filterFn, sorted, title_kind="+")
+        return self._render_events(filterFn, sorted)
 
     def _render_ce_type(self, type):
         template = self.env.get_template("common-event-fields.tmpl")
         ce_types = self.units.get("common_event_fields")
-        return template.render(common_event=ce_types[type])
+        subtitle_title_char = self.units.get("spec_targets")[
+            "relative_title_styles"
+        ]["subtitle"]
+        return template.render(
+            common_event=ce_types[type], title_kind=subtitle_title_char
+        )
 
     def render_common_event_fields(self):
         return self._render_ce_type("event")
@@ -159,3 +179,4 @@ class MatrixSections(Sections):
 
     def render_common_state_event_fields(self):
         return self._render_ce_type("state_event")
+

templating/matrix_templates/templates/common-event-fields.tmpl

@@ -1,5 +1,5 @@
 {{common_event.title}} Fields
-{{(7 + common_event.title | length) * '-'}}
+{{(7 + common_event.title | length) * title_kind}}
 
 {{common_event.desc | wrap(80)}}
 

templating/matrix_templates/templates/http-api.tmpl

@@ -62,7 +62,9 @@ Response{{"s" if endpoint.example.responses|length > 1 else "" }}:
 
 {{res["description"]}}
 
-Example::
+Example
+
+.. code:: json
 
   {{res["example"] | indent_block(2)}}
 

templating/matrix_templates/templates/msgtypes.tmpl

@@ -1,5 +1,5 @@
 ``{{event.msgtype}}``
-{{(4 + event.msgtype | length) * '+'}}
+{{(4 + event.msgtype | length) * title_kind}}
 {{event.desc | wrap(80)}}
 {% for table in event.content_fields -%}
 {{"``"+table.title+"``" if table.title else "" }}

templating/matrix_templates/units.py

@@ -1,4 +1,12 @@
-"""Contains all the units for the spec."""
+"""
+Contains all the units for the spec.
+
+This file loads swagger and JSON schema files and parses out the useful bits
+and returns them as Units for use in Batesian.
+
+For the actual conversion of data -> RST (including templates), see the sections
+file instead.
+"""
 from batesian.units import Units
 import inspect
 import json
@@ -8,6 +16,17 @@ import subprocess
 import urllib
 import yaml
 
+V1_CLIENT_API = "../api/client-server/v1"
+V1_EVENT_EXAMPLES = "../event-schemas/examples/v1"
+V1_EVENT_SCHEMA = "../event-schemas/schema/v1"
+CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema"
+CHANGELOG = "../CHANGELOG.rst"
+TARGETS = "../specification/targets.yaml"
+
+ROOM_EVENT = "core-event-schema/room_event.json"
+STATE_EVENT = "core-event-schema/state_event.json"
+
+
 def get_json_schema_object_fields(obj, enforce_title=False):
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
@@ -87,6 +106,8 @@ def get_json_schema_object_fields(obj, enforce_title=False):
                     desc += (
                         " Must be '%s'." % props[key_name]["enum"][0]
                     )
+            if isinstance(value_type, list):
+                value_type = " or ".join(value_type)
 
         fields["rows"].append({
             "key": key_name,
@@ -121,7 +142,7 @@ class MatrixUnits(Units):
                         "good_response": ""
                     }
                 }
-                self.log(".o.O.o. Endpoint: %s %s" % (method, path))
+                self.log(" ------- Endpoint: %s %s ------- " % (method, path))
                 for param in single_api.get("parameters", []):
                     # description
                     desc = param.get("description", "")
@@ -149,8 +170,8 @@ class MatrixUnits(Units):
                     # object with some keys; we'll add entries f.e one)
                     if "schema" not in param:
                         raise Exception(
-                            "API endpoint group=%s path=%s method=%s param=%s"+
+                            ("API endpoint group=%s path=%s method=%s param=%s"+
-                            " has no valid parameter value." % (
+                            " has no valid parameter value.") % (
                                 group_name, path, method, param
                             )
                         )
@@ -170,6 +191,9 @@ class MatrixUnits(Units):
                             "desc": json_body[key]["description"]
                         })
                 # endfor[param]
+                for row in endpoint["req_params"]:
+                    self.log("Request parameter: %s" % row)
+
                 # group params by location to ease templating
                 endpoint["req_param_by_loc"] = {
                     #   path: [...], query: [...], body: [...]
@@ -227,6 +251,7 @@ class MatrixUnits(Units):
 
                 # add response params if this API has any.
                 if good_response:
+                    self.log("Found a 200 response for this API")
                     res_type = Units.prop(good_response, "schema/type")
                     if res_type and res_type not in ["object", "array"]:
                         # response is a raw string or something like that
@@ -235,7 +260,8 @@ class MatrixUnits(Units):
                             "rows": [{
                                 "key": good_response["schema"].get("name", ""),
                                 "type": res_type,
-                                "desc": res.get("description", "")
+                                "desc": res.get("description", ""),
+                                "req_str": ""
                             }]
                         })
                     elif res_type and Units.prop(good_response, "schema/properties"):
@@ -245,6 +271,34 @@ class MatrixUnits(Units):
                         for table in res_tables:
                             if "no-table" not in table:
                                 endpoint["res_tables"].append(table)
+                    elif res_type and Units.prop(good_response, "schema/items"):
+                        # response is an array:
+                        # FIXME: Doesn't recurse at all.
+                        schema = good_response["schema"]
+                        array_type = Units.prop(schema, "items/type")
+                        if Units.prop(schema, "items/allOf"):
+                            array_type = (
+                                Units.prop(schema, "items/title")
+                            )
+                        endpoint["res_tables"].append({
+                            "title": schema.get("title", ""),
+                            "rows": [{
+                                "key": "N/A",
+                                "type": ("[%s]" % array_type),
+                                "desc": schema.get("description", ""),
+                                "req_str": ""
+                            }]
+                        })
+
+                for response_table in endpoint["res_tables"]:
+                    self.log("Response: %s" % response_table["title"])
+                    for r in response_table["rows"]:
+                        self.log("Row: %s" % r)
+                if len(endpoint["res_tables"]) == 0:
+                    self.log(
+                        "This API appears to have no response table. Are you " +
+                        "sure this API returns no parameters?"
+                    )
 
                 endpoints.append(endpoint)
 
@@ -266,7 +320,7 @@ class MatrixUnits(Units):
         }
 
     def load_swagger_apis(self):
-        path = "../api/client-server/v1"
+        path = V1_CLIENT_API
         apis = {}
         for filename in os.listdir(path):
             if not filename.endswith(".yaml"):
@@ -281,19 +335,33 @@ class MatrixUnits(Units):
         return apis
 
     def load_common_event_fields(self):
-        path = "../event-schemas/schema/v1/core"
+        path = CORE_EVENT_SCHEMA
         event_types = {}
-        with open(path, "r") as f:
+
-            core_json = json.loads(f.read())
+        for (root, dirs, files) in os.walk(path):
-            for event_type in core_json["definitions"]:
+            for filename in files:
+                if not filename.endswith(".json"):
+                    continue
+
+                event_type = filename[:-5]  # strip the ".json"
+                filepath = os.path.join(root, filename)
+                with open(filepath) as f:
+                    try:
+                        event_info = json.load(f)
+                    except Exception as e:
+                        raise ValueError(
+                            "Error reading file %r" % (filepath,), e
+                        )
+
                 if "event" not in event_type:
                     continue  # filter ImageInfo and co
-                event_info = core_json["definitions"][event_type]
+
                 table = {
                     "title": event_info["title"],
                     "desc": event_info["description"],
                     "rows": []
                 }
+
                 for prop in sorted(event_info["properties"]):
                     row = {
                         "key": prop,
@@ -301,11 +369,12 @@ class MatrixUnits(Units):
                         "desc": event_info["properties"][prop].get("description","")
                     }
                     table["rows"].append(row)
+
                 event_types[event_type] = table
         return event_types
 
     def load_event_examples(self):
-        path = "../event-schemas/examples/v1"
+        path = V1_EVENT_EXAMPLES
         examples = {}
         for filename in os.listdir(path):
             if not filename.startswith("m."):
@@ -317,7 +386,7 @@ class MatrixUnits(Units):
         return examples
 
     def load_event_schemas(self):
-        path = "../event-schemas/schema/v1"
+        path = V1_EVENT_SCHEMA
         schemata = {}
 
         for filename in os.listdir(path):
@@ -346,8 +415,8 @@ class MatrixUnits(Units):
 
                 # add typeof
                 base_defs = {
-                    "core#/definitions/room_event": "Message Event",
+                    ROOM_EVENT: "Message Event",
-                    "core#/definitions/state_event": "State Event"
+                    STATE_EVENT: "State Event"
                 }
                 if type(json_schema.get("allOf")) == list:
                     schema["typeof"] = base_defs.get(
@@ -384,7 +453,6 @@ class MatrixUnits(Units):
                         "`m.room.message msgtypes`_."
                     )
 
-
                 # Assign state key info if it has some
                 if schema["typeof"] == "State Event":
                     skey_desc = Units.prop(
@@ -398,7 +466,7 @@ class MatrixUnits(Units):
         return schemata
 
     def load_spec_meta(self):
-        path = "../CHANGELOG.rst"
+        path = CHANGELOG
         title_part = None
         version = None
         changelog_lines = []
@@ -429,7 +497,7 @@ class MatrixUnits(Units):
             if re.match("^v[0-9\.]+$", word):
                 version = word[1:]  # strip the 'v'
 
-        self.log("Version: %s Title part: %s Changelog lines: %s" % (
+        self.log("Version: %s Title part: %s Changelog line count: %s" % (
             version, title_part, len(changelog_lines)
         ))
         if not version or len(changelog_lines) == 0:
@@ -440,6 +508,12 @@ class MatrixUnits(Units):
             "changelog": "".join(changelog_lines)
         }
 
+
+    def load_spec_targets(self):
+        with open(TARGETS, "r") as f:
+            return yaml.load(f.read())
+
+
     def load_git_version(self):
         null = open(os.devnull, 'w')
         cwd = os.path.dirname(os.path.abspath(__file__))