diff --git a/.gitignore b/.gitignore index 63ef5bb3..74b1c7d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ scripts/gen +scripts/continuserv/continuserv +scripts/speculator/speculator templating/out *.pyc supporting-docs/_site supporting-docs/.sass-cache +api/node_modules diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78aebddc..6e3198ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,34 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.2.0 (2015-10-02) +============================================ + +This update fundamentally restructures the specification. The specification has +been split into more digestible "modules" which each describe a particular +function (e.g. typing). This was done in order make the specification easier to +maintain and help define which modules are mandatory for certain types +of clients. Types of clients along with the mandatory modules can be found in a +new "Feature Profiles" section. This update also begins to aggressively +standardise on using Swagger and JSON Schema to document HTTP endpoints and +Events respectively. It also introduces a number of new concepts to Matrix. + +Additions: + - New section: Feature Profiles. + - New section: Receipts. + - New section: Room history visibility. + - New event: ``m.receipt``. + - New event: ``m.room.canonical_alias`` + - New event: ``m.room.history_visibility`` + - New keys: ``/createRoom`` - allows room "presets" using ``preset`` and + ``initial_state`` keys. + - New endpoint: ``/tokenrefresh`` - Related to refreshing access tokens. + +Modifications: + - Convert most of the older HTTP APIs to Swagger documentation. + - Convert most of the older event formats to JSON Schema. + - Move selected client-server sections to be "Modules". + Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. diff --git a/api/check_examples.py b/api/check_examples.py new file mode 100755 index 00000000..f08b2dc1 --- /dev/null +++ b/api/check_examples.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python + +import sys +import json +import os + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + + +def check_parameter(filepath, request, parameter): + schema = parameter.get("schema") + example = None + try: + example_json = schema.get('example') + if example_json and not schema.get("format") == "byte": + example = json.loads(example_json) + except Exception as e: + raise ValueError("Error parsing JSON example request for %r" % ( + request + ), e) + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking request schema for: %r %r" % ( + filepath, request + )) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + + +def check_response(filepath, request, code, response): + example = None + try: + example_json = response.get('examples', {}).get('application/json') + if example_json: + example = json.loads(example_json) + except Exception as e: + raise ValueError("Error parsing JSON example response for %r %r" % ( + request, code + ), e) + schema = response.get('schema') + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking response schema for: %r %r %r" % ( + filepath, request, code + )) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + + +def check_swagger_file(filepath): + with open(filepath) as f: + swagger = yaml.load(f) + + for path, path_api in swagger.get('paths', {}).items(): + + for method, request_api in path_api.items(): + request = "%s %s" % (method.upper(), path) + for parameter in request_api.get('parameters', ()): + if parameter['in'] == 'body': + check_parameter(filepath, request, parameter) + + try: + responses = request_api['responses'] + except KeyError: + raise ValueError("No responses for %r" % (request,)) + for code, response in responses.items(): + check_response(filepath, request, code, response) + + +if __name__ == '__main__': + paths = sys.argv[1:] + if not paths: + paths = [] + for (root, dirs, files) in os.walk(os.curdir): + for filename in files: + if filename.endswith(".yaml"): + paths.append(os.path.join(root, filename)) + for path in paths: + try: + check_swagger_file(path) + except Exception as e: + raise ValueError("Error checking file %r" % (path,), e) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index fe3d1dc3..8e6e8d1a 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -15,16 +15,22 @@ paths: summary: Upload some content to the content repository. produces: ["application/json"] parameters: + - in: header + name: Content-Type + type: string + description: The content type of the file being uploaded + x-example: "Content-Type: audio/mpeg" - in: body - name: content + name: "" description: The content to be uploaded. required: true schema: type: string + example: "" format: byte responses: 200: - description: Information about the uploaded content. + description: The MXC URI for the uploaded content. schema: type: object required: ["content_uri"] @@ -32,6 +38,11 @@ paths: content_uri: type: string description: "The MXC URI to the uploaded content." + examples: + "application/json": |- + { + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + } "/download/{serverName}/{mediaId}": get: summary: "Download content from the content repository." @@ -40,18 +51,27 @@ paths: - in: path type: string name: serverName + x-example: matrix.org required: true description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) responses: 200: - description: "The content downloaded." + description: "The content that was previously uploaded." + headers: + Content-Type: + description: "The content type of the file that was previously uploaded." + type: "string" + Content-Disposition: + description: "The name of the file that was previously uploaded, if set." + type: "string" schema: type: file "/thumbnail/{serverName}/{mediaId}": @@ -63,30 +83,44 @@ paths: type: string name: serverName required: true + x-example: matrix.org description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) - in: query type: integer + x-example: 64 name: width - description: The desired width of the thumbnail. + description: |- + The *desired* width of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: integer + x-example: 64 name: height - description: The desired height of the thumbnail. + description: |- + The *desired* height of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: string enum: ["crop", "scale"] name: method + x-example: "scale" description: The desired resizing method. responses: 200: description: "A thumbnail of the requested content." + headers: + Content-Type: + description: "The content type of the thumbnail." + type: "string" + enum: ["image/jpeg", "image/png"] schema: type: file diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema new file mode 120000 index 00000000..045aecb0 --- /dev/null +++ b/api/client-server/v1/core-event-schema @@ -0,0 +1 @@ +v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/definitions/event.yaml b/api/client-server/v1/definitions/event.yaml deleted file mode 100644 index cfb1b924..00000000 --- a/api/client-server/v1/definitions/event.yaml +++ /dev/null @@ -1,7 +0,0 @@ -type: object -description: A Matrix Event -properties: - event_id: - type: string - description: An event ID. -required: ["event_id"] \ No newline at end of file diff --git a/api/client-server/v1/definitions/room_event.yaml b/api/client-server/v1/definitions/room_event.yaml deleted file mode 100644 index 5a9e3a09..00000000 --- a/api/client-server/v1/definitions/room_event.yaml +++ /dev/null @@ -1,9 +0,0 @@ -type: object -description: A Matrix Room Event -properties: - event_id: - type: string - description: An event ID. - room_id: - type: string -required: ["event_id", "room_id"] \ No newline at end of file diff --git a/api/client-server/v1/definitions/state_event.yaml b/api/client-server/v1/definitions/state_event.yaml deleted file mode 100644 index f8e6f4d4..00000000 --- a/api/client-server/v1/definitions/state_event.yaml +++ /dev/null @@ -1,11 +0,0 @@ -type: object -description: A Matrix State Event -properties: - event_id: - type: string - description: An event ID. - room_id: - type: string - state_key: - type: string -required: ["event_id", "room_id", "state_key"] \ No newline at end of file diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml new file mode 100644 index 00000000..3d415c29 --- /dev/null +++ b/api/client-server/v1/login.yaml @@ -0,0 +1,147 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Registration and Login API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/login": + post: + summary: Authenticates the user. + description: |- + Authenticates the user by password, and issues an access token they can + use to authorize themself in subsequent requests. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "username": "cheeky_monkey", + "password": "ilovebananas" + } + properties: + username: + type: string + description: The fully qualified user ID or just local part of the user ID, to log in. + password: + type: string + description: The user's password. + required: ["username", "password"] + responses: + 200: + description: The user has been authenticated. + examples: + application/json: |- + { + "user_id": "@cheeky_monkey:matrix.org", + "access_token": "abc123", + "home_server": "matrix.org" + } + schema: + type: object + properties: + user_id: + type: string + description: The fully-qualified Matrix ID that has been registered. + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + There is no specific error message to indicate that a request has failed because + an access token has expired; instead, if a client has reason to believe its + access token is valid, and it receives an auth error, they should attempt to + refresh for a new token on failure, and retry the request with the new token. + refresh_token: + type: string + # TODO: Work out how to linkify /tokenrefresh + description: |- + (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. + home_server: + type: string + description: The hostname of the Home Server on which the account has been registered. + 403: + description: |- + The login attempt failed. For example, the password may have been incorrect. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/tokenrefresh": + post: + summary: Exchanges a refresh token for an access token. + description: |- + Exchanges a refresh token for a new access token. + This is intended to be used if the access token has expired. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "refresh_token": "a1b2c3" + } + properties: + refresh_token: + type: string + description: The refresh token which was issued by the server. + required: ["refresh_token"] + responses: + 200: + description: |- + The refresh token was accepted, and a new access token has been issued. + The passed refresh token is no longer valid and cannot be used. + A new refresh token will have been returned unless some policy does + not allow the user to continue to renew their session. + examples: + application/json: |- + { + "access_token": "bearwithme123", + "refresh_token": "exchangewithme987" + } + schema: + type: object + properties: + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + refresh_token: + type: string + description: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the TODO Linkify /tokenrefresh API endpoint. + 403: + description: |- + The exchange attempt failed. For example, the refresh token may have already been used. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml new file mode 100644 index 00000000..41f2febc --- /dev/null +++ b/api/client-server/v1/membership.yaml @@ -0,0 +1,128 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} + + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + description: |- + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + This serves two purposes; firstly, to notify the user that the room + exists (and that their presence is requested). Secondly, some rooms can + only be joined if a user is invited to join it; sending the invite gives + that user permission to join the room. + + Only users currently in a particular room can invite other users to + join that room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: user_id + required: true + schema: + type: object + example: |- + { + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the invitee. + required: ["user_id"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/presence.yaml b/api/client-server/v1/presence.yaml index 16665341..5684398b 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/v1/presence.yaml @@ -101,7 +101,7 @@ paths: The length of time in milliseconds since an action was performed by this user. status_msg: - type: string + type: [string, "null"] description: The state message for this user if one was set. 404: description: |- @@ -185,7 +185,7 @@ paths: "last_active_ago": 395, "presence": "offline", "user_id": "@alice:matrix.org" - } + }, "type": "m.presence" }, { @@ -195,7 +195,7 @@ paths: "last_active_ago": 16874, "presence": "online", "user_id": "@marisa:matrix.org" - } + }, "type": "m.presence" } ] @@ -205,5 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "definitions/event.yaml" - \ No newline at end of file + - "$ref": "core-event-schema/event.json" diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml new file mode 100644 index 00000000..9300d7d1 --- /dev/null +++ b/api/client-server/v1/rooms.yaml @@ -0,0 +1,448 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Rooms API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/state/{eventType}/{stateKey}": + get: + summary: Get the state identified by the type and key. + description: |- + Looks up the contents of a state event in a room. If the user is + joined to the room then the state is taken from the current + state of the room. If the user has left the room then the state is + taken from the state of the room when they left. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to look up the state in. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of state to look up. + required: true + x-example: "m.room.name" + - in: path + type: string + name: stateKey + description: The key of the state to look up. Defaults to the empty string. + required: true + x-example: "" + responses: + 200: + description: The content of the state event. + examples: + application/json: |- + {"name": "Example room name"} + schema: + type: object + 404: + description: The room has no state with the given type or key. + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/state": + get: + summary: Get all state events in the current state of a room. + description: |- + Get the state events for the current state of a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to look up the state for. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: The current state of the room + examples: + application/json: |- + [ + { + "age": 7148266897, + "content": { + "join_rule": "public" + }, + "event_id": "$14259997323TLwtb:example.com", + "origin_server_ts": 1425999732392, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.join_rules", + "user_id": "@alice:example.com" + }, + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 7148267200, + "content": { + "creator": "@alice:example.com" + }, + "event_id": "$14259997320KhbwJ:example.com", + "origin_server_ts": 1425999732089, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.create", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + }, + { + "age": 7148267004, + "content": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "kick": 50, + "redact": 50, + "state_default": 50, + "users": { + "@alice:example.com": 100 + }, + "users_default": 0 + }, + "event_id": "$14259997322mqfaq:example.com", + "origin_server_ts": 1425999732285, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.power_levels", + "user_id": "@alice:example.com" + } + ] + schema: + type: array + title: RoomState + description: |- + If the user is a member of the room this will be the + current state of the room as a list of events. If the user + has left the room then this will be the state of the room + when they left as a list of events. + items: + title: StateEvent + type: object + allOf: + - "$ref": "core-event-schema/state_event.json" + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/initialSync": + get: + summary: Snapshot the current state of a room and its most recent messages. + description: |- + Get a copy of the current state and the most recent messages in a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to get the data. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: The current state of the room + examples: + application/json: |- + { + "membership": "join", + "messages": { + "chunk": [ + { + "age": 343513403, + "content": { + "body": "foo", + "msgtype": "m.text" + }, + "event_id": "$14328044851tzTJS:example.com", + "origin_server_ts": 1432804485886, + "room_id": "!636q39766251:example.com", + "type": "m.room.message", + "user_id": "@alice:example.com" + }, + { + "age": 343511809, + "content": { + "body": "bar", + "msgtype": "m.text" + }, + "event_id": "$14328044872spjFg:example.com", + "origin_server_ts": 1432804487480, + "room_id": "!636q39766251:example.com", + "type": "m.room.message", + "user_id": "@bob:example.com" + } + ], + "end": "s3456_9_0", + "start": "t44-3453_9_0" + }, + "room_id": "!636q39766251:example.com", + "state": [ + { + "age": 7148266897, + "content": { + "join_rule": "public" + }, + "event_id": "$14259997323TLwtb:example.com", + "origin_server_ts": 1425999732392, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.join_rules", + "user_id": "@alice:example.com" + }, + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 7148267200, + "content": { + "creator": "@alice:example.com" + }, + "event_id": "$14259997320KhbwJ:example.com", + "origin_server_ts": 1425999732089, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.create", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + }, + { + "age": 7148267004, + "content": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "kick": 50, + "redact": 50, + "state_default": 50, + "users": { + "@alice:example.com": 100 + }, + "users_default": 0 + }, + "event_id": "$14259997322mqfaq:example.com", + "origin_server_ts": 1425999732285, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.power_levels", + "user_id": "@alice:example.com" + } + ], + "visibility": "private" + } + schema: + title: RoomInfo + type: object + properties: + room_id: + type: string + description: "The ID of this room." + membership: + type: string + description: "The user's membership state in this room." + enum: ["invite", "join", "leave", "ban"] + messages: + type: object + title: PaginationChunk + description: "The pagination chunk for this room." + properties: + start: + type: string + description: |- + A token which correlates to the first value in ``chunk``. + Used for pagination. + end: + type: string + description: |- + A token which correlates to the last value in ``chunk``. + Used for pagination. + chunk: + type: array + description: |- + If the user is a member of the room this will be a + list of the most recent messages for this room. If + the user has left the room this will be the + messages that preceeded them leaving. This array + will consist of at most ``limit`` elements. + items: + type: object + title: RoomEvent + allOf: + - "$ref": "core-event-schema/room_event.json" + required: ["start", "end", "chunk"] + state: + type: array + description: |- + If the user is a member of the room this will be the + current state of the room as a list of events. If the + user has left the room this will be the state of the + room when they left it. + items: + title: StateEvent + type: object + allOf: + - "$ref": "core-event-schema/state_event.json" + visibility: + type: string + enum: ["private", "public"] + description: |- + Whether this room is visible to the ``/publicRooms`` API + or not." + required: ["room_id", "membership"] + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/members": + get: + summary: Get the m.room.member events for the room. + description: + Get the list of members for this room. + parameters: + - in: path + type: string + name: roomId + description: The room to get the member events for. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: |- + A list of members of the room. If you are joined to the room then + this will be the current members of the room. If you have left te + room then this will be the members of the room when you left. + examples: + application/json: |- + { + "chunk": [ + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + } + ] + } + schema: + type: object + properties: + chunk: + type: array + items: + title: MemberEvent + type: object + allOf: + - "$ref": "v1-event-schema/m.room.member" + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index a08146b4..8fe30056 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -82,7 +82,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "definitions/room_event.yaml" + - "$ref": "core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -253,7 +253,7 @@ paths: type: object title: Event allOf: - - "$ref": "definitions/event.yaml" + - "$ref": "core-event-schema/event.json" rooms: type: array items: @@ -267,6 +267,12 @@ paths: type: string description: "The user's membership state in this room." enum: ["invite", "join", "leave", "ban"] + invite: + type: object + title: "InviteEvent" + description: "The invite event if ``membership`` is ``invite``" + allOf: + - "$ref": "v1-event-schema/m.room.member" messages: type: object title: PaginationChunk @@ -285,24 +291,29 @@ paths: chunk: type: array description: |- - A list of the most recent messages for this room. This - array will consist of at most ``limit`` elements. + If the user is a member of the room this will be a + list of the most recent messages for this room. If + the user has left the room this will be the + messages that preceeded them leaving. This array + will consist of at most ``limit`` elements. items: type: object title: RoomEvent allOf: - - "$ref": "definitions/room_event.yaml" + - "$ref": "core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array description: |- - A list of state events representing the current state - of the room. + If the user is a member of the room this will be the + current state of the room as a list of events. If the + user has left the room this will be the state of the + room when they left it. items: title: StateEvent type: object allOf: - - "$ref": "definitions/state_event.yaml" + - "$ref": "core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -338,13 +349,13 @@ paths: "body": "Hello world!", "msgtype": "m.text" }, - "room_id:" "!wfgy43Sg4a:matrix.org", + "room_id:": "!wfgy43Sg4a:matrix.org", "user_id": "@bob:matrix.org", "event_id": "$asfDuShaf7Gafaw:matrix.org", "type": "m.room.message" } schema: allOf: - - "$ref": "definitions/event.yaml" + - "$ref": "core-event-schema/event.json" 404: - description: The event was not found or you do not have permission to read this event. \ No newline at end of file + description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/typing.yaml b/api/client-server/v1/typing.yaml new file mode 100644 index 00000000..737c6928 --- /dev/null +++ b/api/client-server/v1/typing.yaml @@ -0,0 +1,77 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Typing API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/typing/{userId}": + put: + summary: Informs the server that the user has started or stopped typing. + description: |- + This tells the server that the user is typing for the next N + milliseconds where N is the value specified in the ``timeout`` key. + Alternatively, if ``typing`` is ``false``, it tells the server that the + user has stopped typing. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + description: The user who has started to type. + required: true + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + description: The room in which the user is typing. + required: true + x-example: "!wefh3sfukhs:example.com" + - in: body + name: typingState + description: The current typing state. + required: true + schema: + type: object + example: |- + { + "typing": true, + "timeout": 30000 + } + properties: + typing: + type: boolean + description: |- + Whether the user is typing or not. If ``false``, the ``timeout`` + key can be omitted. + timeout: + type: integer + description: The length of time in milliseconds to mark this user as typing. + required: ["typing"] + responses: + 200: + description: The new typing state was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/api/client-server/v1/v1-event-schema b/api/client-server/v1/v1-event-schema new file mode 120000 index 00000000..7a0d0326 --- /dev/null +++ b/api/client-server/v1/v1-event-schema @@ -0,0 +1 @@ +../../../event-schemas/schema/v1 \ No newline at end of file diff --git a/api/client-server/v1/voip.yaml b/api/client-server/v1/voip.yaml new file mode 100644 index 00000000..5fdf1ca7 --- /dev/null +++ b/api/client-server/v1/voip.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Voice over IP API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/turnServer": + get: + summary: Obtain TURN server credentials. + description: |- + This API provides credentials for the client to use when initiating + calls. + security: + - accessToken: [] + responses: + 200: + description: The TURN server credentials. + examples: + application/json: |- + { + "username":"1443779631:@user:example.com", + "password":"JlKfBy1QwLrO20385QyAtEyIv0=", + "uris":[ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp" + ], + "ttl":86400 + } + schema: + type: object + properties: + username: + type: string + description: |- + The username to use. + password: + type: string + description: |- + The password to use. + uris: + type: array + items: + type: string + description: A list of TURN URIs + ttl: + type: integer + description: The time-to-live in seconds + required: ["username", "password", "uris", "ttl"] + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions new file mode 120000 index 00000000..945c9b46 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/definitions @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json new file mode 100644 index 00000000..75762d75 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of events", + "items": { + "type": "object" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json new file mode 100644 index 00000000..1cdcb1f4 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -0,0 +1,42 @@ +{ + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": + "The maximum number of events to return." + }, + "types": { + "type": "array", + "description": + "A list of event types to include. If this list is absent then all event types are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_types": { + "type": "array", + "description": + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "senders": { + "type": "array", + "description": + "A list of senders IDs to include. If this list is absent then all senders are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_senders": { + "type": "array", + "description": + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json new file mode 100644 index 00000000..fcf148f3 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of event ids", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json new file mode 100644 index 00000000..86375781 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -0,0 +1,22 @@ +{ + "type": "object", + "allOf": [{"$ref": "definitions/event_filter.json"}], + "properties": { + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json new file mode 100644 index 00000000..0cd6a798 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "room": { + "type": "object", + "properties": { + "state": { + "description": + "The state events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "timeline": { + "description": + "The message and state update events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "ephemeral": { + "description": + "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + } + } + }, + "presence": { + "description": + "The presence updates to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "event_format": { + "description": + "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", + "type": "string", + "enum": ["client", "federation"] + }, + "event_fields": { + "type": "array", + "description": + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A literal '.' character in a field name may be escaped using a '\\'. A server may include more fields than were requested.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json new file mode 100644 index 00000000..ddf8d341 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "properties": { + "limited": { + "type": "boolean", + "description": "Whether there are more events on the server" + }, + "prev_batch": { + "type": "string", + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + } + } +} diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml new file mode 100644 index 00000000..37a0a3aa --- /dev/null +++ b/api/client-server/v2_alpha/filter.yaml @@ -0,0 +1,139 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 filter API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/filter": + post: + summary: Upload a new filter. + description: |- + Uploads a new filter definition to the homeserver. + Returns a filter ID that may be used in /sync requests to + retrict which events are returned to the client. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: + The id of the user uploading the filter. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: body + name: filter + required: true + description: The filter to upload. + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + responses: + 200: + description: The filter was created. + examples: + application/json: |- + { + "filter_id": "66696p746572" + } + schema: + type: object + properties: + filter_id: + type: string + description: |- + The ID of the filter that was created. + "/user/{userId}/filter/{filterId}": + get: + summary: Download a filter + parameters: + - in: path + name: userId + type: string + description: |- + The user ID to download a filter for. + x-example: "@alice:example.com" + required: true + - in: path + name: filterId + type: string + description: |- + The filter ID to download. + x-example: "66696p746572" + required: true + responses: + 200: + description: |- + "The filter defintion" + examples: + application/json: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/v2_alpha/receipts.yaml new file mode 100644 index 00000000..4ef435b0 --- /dev/null +++ b/api/client-server/v2_alpha/receipts.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Receipts API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/receipt/{receiptType}/{eventId}": + post: + summary: Send a receipt for the given event ID. + description: |- + This API updates the marker for the given receipt type to the event ID + specified. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room in which to send the event. + required: true + x-example: "!wefuh21ffskfuh345:example.com" + - in: path + type: string + name: receiptType + description: The type of receipt to send. + required: true + x-example: "m.read" + enum: ["m.read"] + - in: path + type: string + name: eventId + description: The event ID to acknowledge up to. + required: true + x-example: "$1924376522eioj:example.com" + - in: body + description: |- + Extra receipt information to attach to ``content`` if any. The + server will automatically set the ``ts`` field. + schema: + type: object + example: |- + {} + responses: + 200: + description: The receipt was sent. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" \ No newline at end of file diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml new file mode 100644 index 00000000..7d60d639 --- /dev/null +++ b/api/client-server/v2_alpha/sync.yaml @@ -0,0 +1,257 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 sync API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/sync": + get: + summary: Synchronise the client's state and receive new messages. + description: |- + Synchronise the client's state with the latest state on the server. + Client's use this API when they first log in to get an initial snapshot + of the state on the server, and then continue to call this API to get + incremental deltas to the state, and to receive new messages. + security: + - accessToken: [] + parameters: + - in: query + name: filter + type: string + description: |- + The ID of a filter created using the filter API. + x-example: "66696p746572" + - in: query + name: since + type: string + description: |- + A point in time to continue a sync from. + x-example: "s72594_4483_1934" + - in: query + name: set_presence + type: string + enum: ["offline"] + description: |- + Controls whether the client is automatically marked as online by + polling this API. If this parameter is omitted then the client is + automatically marked as online when it uses this API. Otherwise if + the parameter is set to "offline" then the client is not marked as + being online when it uses this API. + x-example: "offline" + - in: query + name: timeout + type: integer + description: |- + The maximum time to poll in milliseconds before returning this + request. + x-example: 30000 + responses: + 200: + description: + The initial snapshot or delta for the client to use to update their + state. + schema: + type: object + properties: + next_batch: + type: string + description: |- + The batch token to supply in the ``since`` param of the next + ``/sync`` request. + rooms: + type: object + description: |- + Updates to rooms. + properties: + joined: + type: object + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't + recorded in the timeline or state of the room. + e.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + invited: + type: object + description: |- + The rooms that the user has been invited to. + additionalProperties: + type: object + properties: + invite_state: + description: |- + The state of a room that the user has been invited + to. These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys + present. These events do not replace any state that + the client already has for the room, for example if + the client has archived the room. Instead the + client should keep two separate copies of the + state: the one from the ``invite_state`` and one + from the archived ``state``. If the client joins + the room then the current state will be given as a + delta against the archived ``state`` not the + ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" + archived: + type: object + description: |- + The rooms that the user has left or been banned from. The + entries in the room_map will lack an ``ephemeral`` key. + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room up to the point when + the user left. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room up to the point when the user left. + allOf: + - $ref: "definitions/timeline_batch.json" + presence: + description: |- + The updates to the presence status of other users. + allOf: + - $ref: "definitions/event_batch.json" + examples: + application/json: |- + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + } + ] + }, + "rooms": { + "joined": { + "!726s6s6q:example.com": { + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797 + } + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] + } + } + }, + "invited": { + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "invite"} + } + ] + } + } + }, + "archived": {} + } + } diff --git a/api/package.json b/api/package.json index 15193493..84b9dd7b 100644 --- a/api/package.json +++ b/api/package.json @@ -10,6 +10,6 @@ "license": "ISC", "dependencies": { "nopt": "^3.0.2", - "swagger-parser": "^2.4.1" + "swagger-parser": "^3.2.1" } } diff --git a/api/validator.js b/api/validator.js index 6eef652f..0d76c09d 100644 --- a/api/validator.js +++ b/api/validator.js @@ -26,11 +26,10 @@ if (!opts.schema) { } -var errFn = function(err, api, metadata) { +var errFn = function(err, api) { if (!err) { return; } - console.log(metadata); console.error(err); process.exit(1); }; @@ -44,12 +43,14 @@ if (isDir) { process.exit(1); } files.forEach(function(f) { - if (f.indexOf(".yaml") > 0) { - parser.parse(path.join(opts.schema, f), function(err, api, metadata) { + var suffix = ".yaml"; + if (f.indexOf(suffix, f.length - suffix.length) > 0) { + parser.validate(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); } else { + console.error("%s is not valid.", f); errFn(err, api, metadata); } }); @@ -58,12 +59,12 @@ if (isDir) { }); } else{ - parser.parse(opts.schema, function(err, api, metadata) { + parser.validate(opts.schema, function(err, api) { if (!err) { console.log("%s is valid", opts.schema); } else { - errFn(err, api, metadata); + errFn(err, api); } }); }; diff --git a/specification/49_other_non_core_apis.rst b/drafts/address-book-repo.rst similarity index 100% rename from specification/49_other_non_core_apis.rst rename to drafts/address-book-repo.rst index 8855ce9e..d6315a96 100644 --- a/specification/49_other_non_core_apis.rst +++ b/drafts/address-book-repo.rst @@ -3,10 +3,10 @@ Address book repository .. NOTE:: This section is a work in progress. - Do we even need it? Clients can use out-of-band addressbook servers for now; - this should definitely not be core. .. TODO-spec + Do we even need it? Clients can use out-of-band addressbook servers for now; + this should definitely not be core. - format: POST(?) wodges of json, some possible processing, then return wodges of json on GET. - processing may remove dupes, merge contacts, pepper with extra info (e.g. matrix-ability of contacts), etc. diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst new file mode 100644 index 00000000..93622c3d --- /dev/null +++ b/drafts/macaroons_caveats.rst @@ -0,0 +1,34 @@ +Macaroon Caveats +================ + +`Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. + +.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf + +Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. + +Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. + +All caveats must take the form: + +`key` `operator` `value` +where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] +`operator` is a non-empty string which does not contain whitespace +`value` is a non-empty string +And these are joined by single space characters. + +Specified caveats: + ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ +| Caveat name | Description | Legal Values | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ +| gen | Generation of the macaroon caveat spec. | 1 | +| user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | +| refresh - only used to authorize a token refresh | +| time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | +| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| Note that exact equality of time is largely meaningless. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py new file mode 100755 index 00000000..e54d3a1c --- /dev/null +++ b/event-schemas/check_examples.py @@ -0,0 +1,76 @@ +#! /usr/bin/env python + +import sys +import json +import os +import traceback + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + + +def check_example_file(examplepath, schemapath): + with open(examplepath) as f: + example = yaml.load(f) + + with open(schemapath) as f: + schema = yaml.load(f) + + fileurl = "file://" + os.path.abspath(schemapath) + + print ("Checking schema for: %r %r" % (examplepath, schemapath)) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + try: + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + examplepath, schemapath + ), e) + + +def check_example_dir(exampledir, schemadir): + errors = [] + for root, dirs, files in os.walk(exampledir): + for filename in files: + if filename.startswith("."): + # Skip over any vim .swp files. + continue + examplepath = os.path.join(root, filename) + schemapath = examplepath.replace(exampledir, schemadir) + try: + check_example_file(examplepath, schemapath) + except Exception as e: + errors.append(sys.exc_info()) + for (exc_type, exc_value, exc_trace) in errors: + traceback.print_exception(exc_type, exc_value, exc_trace) + if errors: + raise ValueError("Error validating examples") + +if __name__ == '__main__': + try: + check_example_dir("examples", "schema") + except: + sys.exit(1) diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/v1/m.receipt new file mode 100644 index 00000000..bd0b726c --- /dev/null +++ b/event-schemas/examples/v1/m.receipt @@ -0,0 +1,13 @@ +{ + "type": "m.receipt", + "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", + "content": { + "$1435641916114394fHBLK:matrix.org": { + "m.read": { + "@rikj:jki.re": { + "ts": 1436451550453 + } + } + } + } +} diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/v1/m.typing new file mode 100644 index 00000000..bd53f6fb --- /dev/null +++ b/event-schemas/examples/v1/m.typing @@ -0,0 +1,7 @@ +{ + "type": "m.typing", + "room_id": "!z0mnsuiwhifuhwwfw:matrix.org", + "content": { + "user_ids": ["@alice:matrix.org", "@bob:example.com"] + } +} \ No newline at end of file diff --git a/event-schemas/schema/v1/core b/event-schemas/schema/v1/core deleted file mode 100644 index ed374a69..00000000 --- a/event-schemas/schema/v1/core +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "definitions": { - "event": { - "title": "Event", - "description": "The basic set of fields all events must have.", - "type": "object", - "properties": { - "event_id": { - "type": "string", - "description": "The globally unique event identifier." - }, - "user_id": { - "type": "string", - "description": "Contains the fully-qualified ID of the user who *sent* this event." - }, - "content": { - "type": "object", - "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." - }, - "type": { - "type": "string", - "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'" - } - }, - "required": ["event_id", "user_id", "content", "type"] - }, - "room_event": { - "type": "object", - "title": "Room Event", - "description": "In addition to the Event fields, Room Events MUST have the following additional field.", - "allOf":[{ - "$ref": "#/definitions/event" - }], - "properties": { - "room_id": { - "type": "string", - "description": "The ID of the room associated with this event." - } - }, - "required": ["room_id"] - }, - "state_event": { - "type": "object", - "title": "State Event", - "description": "In addition to the Room Event fields, State Events have the following additional fields.", - "allOf":[{ - "$ref": "#/definitions/room_event" - }], - "properties": { - "state_key": { - "type": "string", - "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event." - }, - "prev_content": { - "type": "object", - "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." - } - }, - "required": ["state_key"] - }, - "msgtype_infos": { - "image_info": { - "type": "object", - "title": "ImageInfo", - "description": "Metadata about an image.", - "properties": { - "size": { - "type": "integer", - "description": "Size of the image in bytes." - }, - "w": { - "type": "integer", - "description": "The width of the image in pixels." - }, - "h": { - "type": "integer", - "description": "The height of the image in pixels." - }, - "mimetype": { - "type": "string", - "description": "The mimetype of the image, e.g. ``image/jpeg``." - } - } - } - } - } -} diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema new file mode 120000 index 00000000..945c9b46 --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/core-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/v1/core-event-schema/event.json new file mode 100644 index 00000000..e73aec80 --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/event.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "title": "Event", + "description": "The basic set of fields all events must have.", + "properties": { + "content": { + "type": "object", + "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." + }, + "type": { + "type": "string", + "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'" + } + } +} diff --git a/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json b/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json new file mode 100644 index 00000000..ee75745e --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "ImageInfo", + "description": "Metadata about an image.", + "properties": { + "size": { + "type": "integer", + "description": "Size of the image in bytes." + }, + "w": { + "type": "integer", + "description": "The width of the image in pixels." + }, + "h": { + "type": "integer", + "description": "The height of the image in pixels." + }, + "mimetype": { + "type": "string", + "description": "The mimetype of the image, e.g. ``image/jpeg``." + } + } +} diff --git a/event-schemas/schema/v1/core-event-schema/room_event.json b/event-schemas/schema/v1/core-event-schema/room_event.json new file mode 100644 index 00000000..d5413f8a --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/room_event.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "Room Event", + "description": "In addition to the Event fields, Room Events MUST have the following additional field.", + "allOf":[{ + "$ref": "core-event-schema/event.json" + }], + "properties": { + "room_id": { + "type": "string", + "description": "The ID of the room associated with this event." + }, + "event_id": { + "type": "string", + "description": "The globally unique event identifier." + }, + "user_id": { + "type": "string", + "description": "Contains the fully-qualified ID of the user who *sent* this event." + } + }, + "required": ["room_id"] +} diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json new file mode 100644 index 00000000..f70118f4 --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -0,0 +1,19 @@ +{ + "type": "object", + "title": "State Event", + "description": "In addition to the Room Event fields, State Events have the following additional fields.", + "allOf":[{ + "$ref": "core-event-schema/room_event.json" + }], + "properties": { + "state_key": { + "type": "string", + "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event." + }, + "prev_content": { + "type": "object", + "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." + } + }, + "required": ["state_key"] +} diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/v1/m.call.answer index 598554f0..f11a61ba 100644 --- a/event-schemas/schema/v1/m.call.answer +++ b/event-schemas/schema/v1/m.call.answer @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by the callee when they wish to answer the call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/v1/m.call.candidates index 6475cf51..52e06705 100644 --- a/event-schemas/schema/v1/m.call.candidates +++ b/event-schemas/schema/v1/m.call.candidates @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by callers after sending an invite and by the callee after answering. Its purpose is to give the other party additional ICE candidates to try using to communicate.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/v1/m.call.hangup index 3704ec75..02667635 100644 --- a/event-schemas/schema/v1/m.call.hangup +++ b/event-schemas/schema/v1/m.call.hangup @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "Sent by either party to signal their termination of the call. This can be sent either once the call has has been established or before to abort the call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/v1/m.call.invite index 2be01466..585403d0 100644 --- a/event-schemas/schema/v1/m.call.invite +++ b/event-schemas/schema/v1/m.call.invite @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by the caller when they wish to establish a call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.presence b/event-schemas/schema/v1/m.presence index bb69ce40..79852ac6 100644 --- a/event-schemas/schema/v1/m.presence +++ b/event-schemas/schema/v1/m.presence @@ -1,5 +1,4 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Presence Event", "description": "Informs the client of a user's presence state change.", diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt new file mode 100644 index 00000000..d0f79ac4 --- /dev/null +++ b/event-schemas/schema/v1/m.receipt @@ -0,0 +1,48 @@ +{ + "type": "object", + "title": "Receipt Event", + "description": "Informs the client of new receipts.", + "properties": { + "content": { + "type": "object", + "patternProperties": { + "^\\$": { + "type": "object", + "x-pattern": "$EVENT_ID", + "title": "Receipts", + "description": "The mapping of event ID to a collection of receipts for this event ID. The event ID is the ID of the event being acknowledged and *not* an ID for the receipt itself.", + "properties": { + "m.read": { + "type": "object", + "title": "Users", + "description": "A collection of users who have sent ``m.read`` receipts for this event.", + "patternProperties": { + "^@": { + "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", + "properties": { + "ts": { + "type": "number", + "description": "The timestamp the receipt was sent at." + } + } + } + } + } + } + } + }, + "additionalProperties": false + }, + "type": { + "type": "string", + "enum": ["m.receipt"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["room_id", "type", "content"] +} diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/v1/m.room.aliases index 6585f13d..43cc9dbc 100644 --- a/event-schemas/schema/v1/m.room.aliases +++ b/event-schemas/schema/v1/m.room.aliases @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Informs the room about what room aliases it has been given.", "description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias index 9f044e58..25cd00c0 100644 --- a/event-schemas/schema/v1/m.room.canonical_alias +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Informs the room as to which alias is the canonical one.", "description": "This event is used to inform the room about which alias should be considered the canonical one. This could be for display purposes or as suggestion to users which alias to use to advertise the room.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index 8f74e69b..348b6d86 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "The first event in the room.", "description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { @@ -13,6 +12,10 @@ "creator": { "type": "string", "description": "The ``user_id`` of the room creator. This is set by the homeserver." + }, + "m.federate": { + "type": "boolean", + "description": "Whether users on other servers can join this room. Defaults to ``true`` if key does not exist." } }, "required": ["creator"] diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/v1/m.room.history_visibility index 07bd39ff..bc33f4b3 100644 --- a/event-schemas/schema/v1/m.room.history_visibility +++ b/event-schemas/schema/v1/m.room.history_visibility @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Controls visibility of history.", "description": "This event controls whether a member of a room can see the events that happened in a room from before they joined.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/v1/m.room.join_rules index 2ba20b89..70c36ccb 100644 --- a/event-schemas/schema/v1/m.room.join_rules +++ b/event-schemas/schema/v1/m.room.join_rules @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Describes how users are allowed to join the room.", "description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room. Currently, ``knock`` and ``private`` are reserved keywords which are not implemented.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index ab616176..c0fb103c 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "The current membership state of a user in the room.", "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { @@ -20,7 +19,7 @@ "description": "The avatar URL for this user, if any. This is added by the homeserver." }, "displayname": { - "type": "string", + "type": ["string", "null"], "description": "The display name for this user, if any. This is added by the homeserver." } }, @@ -33,6 +32,26 @@ "type": { "type": "string", "enum": ["m.room.member"] + }, + "invite_room_state": { + "type": "array", + "description": "A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``", + "items": { + "type": "object", + "title": "StateEvent", + "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.", + "properties": { + "type": { + "type": "string" + }, + "state_key": { + "type": "string" + }, + "content": { + "type": "object" + } + } + } } } } diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/v1/m.room.message index a849f07f..27b7e925 100644 --- a/event-schemas/schema/v1/m.room.message +++ b/event-schemas/schema/v1/m.room.message @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Message", "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/v1/m.room.message#m.audio index 9910cb0c..0a6625f5 100644 --- a/event-schemas/schema/v1/m.room.message#m.audio +++ b/event-schemas/schema/v1/m.room.message#m.audio @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "AudioMessage", "description": "This message represents a single audio clip.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/v1/m.room.message#m.emote index 886abafc..690600f4 100644 --- a/event-schemas/schema/v1/m.room.message#m.emote +++ b/event-schemas/schema/v1/m.room.message#m.emote @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "EmoteMessage", "description": "This message is similar to ``m.text`` except that the sender is 'performing' the action contained in the ``body`` key, similar to ``/me`` in IRC. This message should be prefixed by the name of the sender. This message could also be represented in a different colour to distinguish it from regular ``m.text`` messages.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/v1/m.room.message#m.file index cf6d6552..c97480ef 100644 --- a/event-schemas/schema/v1/m.room.message#m.file +++ b/event-schemas/schema/v1/m.room.message#m.file @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "FileMessage", "description": "This message represents a generic file.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -50,7 +49,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/v1/m.room.message#m.image index 8c010b0d..af78096d 100644 --- a/event-schemas/schema/v1/m.room.message#m.image +++ b/event-schemas/schema/v1/m.room.message#m.image @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "ImageMessage", "description": "This message represents a single image and an optional thumbnail.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -31,7 +30,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] }, "info": { diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/v1/m.room.message#m.location index cd15f31e..ef4b5e81 100644 --- a/event-schemas/schema/v1/m.room.message#m.location +++ b/event-schemas/schema/v1/m.room.message#m.location @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "LocationMessage", "description": "This message represents a real-world location.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -30,7 +29,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/v1/m.room.message#m.notice index 0f8044f5..195c56a3 100644 --- a/event-schemas/schema/v1/m.room.message#m.notice +++ b/event-schemas/schema/v1/m.room.message#m.notice @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "NoticeMessage", "description": "A m.notice message should be considered similar to a plain m.text message except that clients should visually distinguish it in some way. It is intended to be used by automated clients, such as bots, bridges, and other entities, rather than humans. Additionally, such automated agents which watch a room for messages and respond to them ought to ignore m.notice messages. This helps to prevent infinite-loop situations where two automated clients continuously exchange messages, as each responds to the other.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/v1/m.room.message#m.text index 054d4a96..de59cf0e 100644 --- a/event-schemas/schema/v1/m.room.message#m.text +++ b/event-schemas/schema/v1/m.room.message#m.text @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "TextMessage", "description": "This message is the most basic message and is used to represent text.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/v1/m.room.message#m.video index 22266506..ad68e72a 100644 --- a/event-schemas/schema/v1/m.room.message#m.video +++ b/event-schemas/schema/v1/m.room.message#m.video @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "VideoMessage", "description": "This message represents a single video clip.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -55,7 +54,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } } diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/v1/m.room.message.feedback index 4cfd44d1..1bbfc1ba 100644 --- a/event-schemas/schema/v1/m.room.message.feedback +++ b/event-schemas/schema/v1/m.room.message.feedback @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "MessageFeedback", "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/v1/m.room.name index 5077774c..b43f02cc 100644 --- a/event-schemas/schema/v1/m.room.name +++ b/event-schemas/schema/v1/m.room.name @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "title": "RoomName", "description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.", "type": "object", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/v1/m.room.power_levels index 9e0cb81a..2ef3bfa3 100644 --- a/event-schemas/schema/v1/m.room.power_levels +++ b/event-schemas/schema/v1/m.room.power_levels @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Defines the power levels (privileges) of users in the room.", "description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/v1/m.room.redaction index 2896c0da..5c3ef391 100644 --- a/event-schemas/schema/v1/m.room.redaction +++ b/event-schemas/schema/v1/m.room.redaction @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Redaction", "description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data. There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/v1/m.room.topic index 266df8fa..d13f0c23 100644 --- a/event-schemas/schema/v1/m.room.topic +++ b/event-schemas/schema/v1/m.room.topic @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Topic", "description": "A topic is a short message detailing what is currently being discussed in the room. It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/v1/m.typing new file mode 100644 index 00000000..b712f6ec --- /dev/null +++ b/event-schemas/schema/v1/m.typing @@ -0,0 +1,28 @@ +{ + "type": "object", + "title": "Typing Event", + "description": "Informs the client of the list of users currently typing.", + "properties": { + "content": { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The list of user IDs typing in this room, if any." + } + }, + "required": ["user_ids"] + }, + "type": { + "type": "string", + "enum": ["m.typing"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["type", "room_id", "content"] +} diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema new file mode 120000 index 00000000..945c9b46 --- /dev/null +++ b/event-schemas/schema/v1/v1-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/jenkins.sh b/jenkins.sh new file mode 100755 index 00000000..0936de9d --- /dev/null +++ b/jenkins.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -ex + +(cd event-schemas/ && ./check_examples.py) +(cd api && ./check_examples.py) +(cd scripts && ./gendoc.py) +(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") +(cd event-schemas/ && ./check.sh) diff --git a/scripts/codehighlight.css b/scripts/codehighlight.css new file mode 100644 index 00000000..5c9b0c36 --- /dev/null +++ b/scripts/codehighlight.css @@ -0,0 +1,6 @@ +pre.code .comment, code .comment { color: green } +pre.code .keyword, code .keyword { color: darkred; font-weight: bold } +pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold } +pre.code .literal.number, code .literal.number { color: blue } +pre.code .name.tag, code .name.tag { color: darkgreen } +pre.code .literal.string, code .literal.string { color: darkblue } diff --git a/scripts/continuserv/README b/scripts/continuserv/README new file mode 100644 index 00000000..8ce37850 --- /dev/null +++ b/scripts/continuserv/README @@ -0,0 +1,6 @@ +continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. + +To run it, you must install the `go` tool. You will also need to install fsnotify by running: + `go get gopkg.in/fsnotify.v1` +You can then run continuserv by running: + `go run main.go` diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go new file mode 100644 index 00000000..573c2c95 --- /dev/null +++ b/scripts/continuserv/main.go @@ -0,0 +1,161 @@ +// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. +// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished. +// It does not currently pre-empt stale generations, but will block until they are complete. +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + fsnotify "gopkg.in/fsnotify.v1" +) + +var ( + port = flag.Int("port", 8000, "Port on which to serve HTTP") + + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + wg sync.WaitGroup // Indicates how many updates are pending. + mu sync.Mutex // Prevent multiple updates in parallel. +) + +func main() { + flag.Parse() + + w, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("Error making watcher: %v", err) + } + + dir, err := os.Getwd() + if err != nil { + log.Fatalf("Error getting wd: %v", err) + } + for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) { + if dir == "/" { + log.Fatalf("Could not find git root") + } + } + + filepath.Walk(dir, makeWalker(w)) + + wg.Add(1) + populateOnce(dir) + + ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes + go doPopulate(ch, dir) + + go watchFS(ch, w) + fmt.Printf("Listening on port %d\n", *port) + http.HandleFunc("/", serve) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) + +} + +func watchFS(ch chan struct{}, w *fsnotify.Watcher) { + for { + select { + case e := <-w.Events: + if filter(e) { + fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) + ch <- struct{}{} + } + } + } +} + +func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { + return func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatalf("Error walking: %v", err) + } + if err := w.Add(path); err != nil { + log.Fatalf("Failed to add watch: %v", err) + } + return nil + } +} + +// Return true if event should trigger re-population +func filter(e fsnotify.Event) bool { + // vim is *really* noisy about how it writes files + if e.Op != fsnotify.Write { + return false + } + // Avoid some temp files that vim writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + return false + } + + // Ignore the .git directory - It's very noisy + if strings.Contains(e.Name, "/.git/") { + return false + } + + // Avoid infinite cycles being caused by writing actual output + if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") { + return false + } + return true +} + +func serve(w http.ResponseWriter, req *http.Request) { + wg.Wait() + b := toServe.Load().([]byte) + w.Write(b) +} + +func populateOnce(dir string) { + defer wg.Done() + mu.Lock() + defer mu.Unlock() + cmd := exec.Command("python", "gendoc.py") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + return + } + specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + if err != nil { + toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + return + } + toServe.Store(specBytes) +} + +func doPopulate(ch chan struct{}, dir string) { + var pending int + for { + select { + case <-ch: + if pending == 0 { + wg.Add(1) + } + pending++ + case <-time.After(10 * time.Millisecond): + if pending > 0 { + pending = 0 + populateOnce(dir) + } + } + } +} + +func exists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f8ec6316..1655d6f0 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,22 +1,231 @@ #! /usr/bin/env python +from argparse import ArgumentParser from docutils.core import publish_file +import copy import fileinput import glob import os +import re import shutil import subprocess import sys +import yaml + +os.chdir(os.path.dirname(os.path.abspath(__file__))) stylesheets = { - "stylesheet_path": ["basic.css", "nature.css"] + "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] } -def glob_spec_to(out_file_name): - with open(out_file_name, "wb") as outfile: - for f in sorted(glob.glob("../specification/*.rst")): - with open(f, "rb") as infile: - outfile.write(infile.read()) +VERBOSE = False + +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" +def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): + rst_lines = [] + title_chars = "".join(title_styles) + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") + + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None + for i, line in enumerate(file_stream, 1): + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): + rst_lines.append(line) + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): + rst_lines.append(line) + prev_non_title_line = line + continue + + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) + + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + logv((" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0])) + + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) + ) + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): + raise Exception( + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) + ) + + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + logv( + "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + (filename, line_title_style, title_styles[adjusted_level], + file_offset, title_level) + ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + + return "".join(rst_lines) + + +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): + # string are file paths to RST blobs + if isinstance(file_info, basestring): + log("%s %s" % (">" * (1 + title_level), file_info)) + with open(os.path.join(spec_dir, file_info), "r") as f: + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + if rst[-2:] != "\n\n": + raise Exception( + ("File %s should end with TWO new-line characters to ensure " + + "file concatenation works correctly.") % (file_info,) + ) + return rst + # dicts look like {0: filepath, 1: filepath} where the key is the title level + elif isinstance(file_info, dict): + levels = sorted(file_info.keys()) + rst = [] + for l in levels: + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + # lists are multiple file paths e.g. [filepath, filepath] + elif isinstance(file_info, list): + rst = [] + for f in file_info: + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + raise Exception( + "The following 'file' entry in this target isn't a string, list or dict. " + + "It really really should be. Entry: %s" % (file_info,) + ) + + +def build_spec(target, out_filename): + with open(out_filename, "wb") as outfile: + for file_info in target["files"]: + section = get_rst( + file_info=file_info, + title_level=0, + title_styles=target["title_styles"], + spec_dir="../specification/", + adjust_titles=True + ) + outfile.write(section) + + +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] + relative_title_chars = [ + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + def rst2html(i, o): @@ -31,18 +240,126 @@ def rst2html(i, o): settings_overrides=stylesheets ) -def run_through_template(input): - null = open(os.devnull, 'w') - subprocess.check_output( - [ - 'python', 'build.py', - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ], - stderr=null, - cwd="../templating", - ) + +def run_through_template(input, set_verbose): + tmpfile = './tmp/output' + try: + with open(tmpfile, 'w') as out: + args = [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ] + if set_verbose: + args.insert(2, "-v") + log("EXEC: %s" % " ".join(args)) + log(" ==== build.py output ==== ") + print subprocess.check_output( + args, + stderr=out, + cwd="../templating" + ) + except subprocess.CalledProcessError as e: + with open(tmpfile, 'r') as f: + sys.stderr.write(f.read() + "\n") + raise + + +""" +Extract and resolve groups for the given target in the given targets listing. +Args: + targets_listing (str): The path to a YAML file containing a list of targets + target_name (str): The name of the target to extract from the listings. +Returns: + dict: Containing "filees" (a list of file paths), "relative_title_styles" + (a dict of relative style keyword to title character) and "title_styles" + (a list of characters which represent the global title style to follow, + with the top section title first, the second section second, and so on.) +""" +def get_build_target(targets_listing, target_name): + build_target = { + "title_styles": [], + "relative_title_styles": {}, + "files": [] + } + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + + build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + + def get_group(group_id, depth): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '%s' but it doesn't exist." % group_name + ) + if not isinstance(group, list): + raise Exception( + "Expected group '%s' to be a list but it isn't." % group_name + ) + # deep copy so changes to depths don't contaminate multiple uses of this group + group = copy.deepcopy(group) + # swap relative depths for absolute ones + for i, entry in enumerate(group): + if isinstance(entry, dict): + group[i] = { + (rel_depth + depth): v for (rel_depth, v) in entry.items() + } + return group + + resolved_files = [] + for file_entry in target["files"]: + # file_entry is a group id + if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + group = get_group(file_entry, 0) + # The group may be resolved to a list of file entries, in which case + # we want to extend the array to insert each of them rather than + # insert the entire list as a single element (which is what append does) + if isinstance(group, list): + resolved_files.extend(group) + else: + resolved_files.append(group) + # file_entry is a dict which has more file entries as values + elif isinstance(file_entry, dict): + resolved_entry = {} + for (depth, entry) in file_entry.iteritems(): + if not isinstance(entry, basestring): + raise Exception( + "Double-nested depths are not supported. Entry: %s" % (file_entry,) + ) + if entry.startswith("group:"): + resolved_entry[depth] = get_group(entry, depth) + else: + # map across without editing (e.g. normal file path) + resolved_entry[depth] = entry + resolved_files.append(resolved_entry) + continue + # file_entry is just a plain ol' file path + else: + resolved_files.append(file_entry) + build_target["files"] = resolved_files + return build_target + +def log(line): + print "gendoc: %s" % line + +def logv(line): + if VERBOSE: + print "gendoc:V: %s" % line + def prepare_env(): try: @@ -53,30 +370,49 @@ def prepare_env(): os.makedirs("./tmp") except OSError: pass - + + def cleanup_env(): shutil.rmtree("./tmp") -def main(): + +def main(target_name, keep_intermediates): prepare_env() - glob_spec_to("tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + log("Building spec [target=%s]" % target_name) + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst", VERBOSE) + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") - cleanup_env() + if not keep_intermediates: + cleanup_env() + if __name__ == '__main__': - if len(sys.argv) > 1: - # we accept no args, so they don't know what they're doing! - print "gendoc.py - Generate the Matrix specification as HTML." - print "Usage:" - print " python gendoc.py" - print "" - print "The specification can then be found in the gen/ folder." - print "" - print "Requirements:" - print " - This script requires Jinja2 and rst2html (docutils)." - sys.exit(0) - main() + parser = ArgumentParser( + "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." + ) + parser.add_argument( + "--nodelete", "-n", action="store_true", + help="Do not delete intermediate files. They will be found in tmp/" + ) + parser.add_argument( + "--target", "-t", default="main", + help="Specify the build target to build from specification/targets.yaml" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Turn on verbose mode." + ) + args = parser.parse_args() + if not args.target: + parser.print_help() + sys.exit(1) + VERBOSE = args.verbose + main(args.target, args.nodelete) diff --git a/scripts/nature.css b/scripts/nature.css index 9d67f689..0fdcc55a 100644 --- a/scripts/nature.css +++ b/scripts/nature.css @@ -244,10 +244,6 @@ div.viewcode-block:target { border-bottom: 1px solid #ac9; } -p { - margin: 0; -} - ul li dd { margin-top: 0; } @@ -282,3 +278,9 @@ td[colspan]:not([colspan="1"]) { thead { background: #eeeeee; } + +div.admonition-rationale { + background-color: #efe; + border: 1px solid #ccc; +} + diff --git a/scripts/speculator/README b/scripts/speculator/README new file mode 100644 index 00000000..0a9f53fd --- /dev/null +++ b/scripts/speculator/README @@ -0,0 +1,10 @@ +speculator allows you to preview pull requests to the matrix.org specification. + +It serves the following HTTP endpoints: + - / lists open pull requests + - /spec/123 which renders the spec as html at pull request 123. + - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. + - /diff/html/123 which gives a diff of the spec's HTML at pull request 123. + +To run it, you must install the `go` tool, and run: + `go run main.go` diff --git a/scripts/speculator/htmldiff.pl b/scripts/speculator/htmldiff.pl new file mode 100755 index 00000000..b1689067 --- /dev/null +++ b/scripts/speculator/htmldiff.pl @@ -0,0 +1,564 @@ +#!/usr/bin/perl +# +# htmldiff - present a diff marked version of two html documents +# +# Copyright (c) 1998-2006 MACS, Inc. +# +# Copyright (c) 2007 SiSco, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# See http://www.themacs.com for more information. +# +# usage: htmldiff [[-c] [-l] [-o] oldversion newversion [output]] +# +# -c - disable metahtml comment processing +# -o - disable outputting of old text +# -l - use navindex to create sequence of diffs +# oldversion - the previous version of the document +# newversion - the newer version of the document +# output - a filename to place the output in. If omitted, the output goes to +# standard output. +# +# if invoked with no options or arguments, operates as a CGI script. It then +# takes the following parameters: +# +# oldfile - the URL of the original file +# newfile - the URL of the new file +# mhtml - a flag to indicate whether it should be aware of MetaHTML comments. +# +# requires GNU diff utility +# also requires the perl modules Getopt::Std +# +# NOTE: The markup created by htmldiff may not validate against the HTML 4.0 +# DTD. This is because the algorithm is realtively simple, and there are +# places in the markup content model where the span element is not allowed. +# Htmldiff is NOT aware of these places. +# +# $Source: /u/sources/public/2009/htmldiff/htmldiff.pl,v $ +# $Revision: 1.1 $ +# +# $Log: htmldiff.pl,v $ +# Revision 1.1 2014/01/06 08:04:51 dom +# added copy of htmldiff perl script since aptest.com repo no longer available +# +# Revision 1.5 2008/03/05 13:23:16 ahby +# Fixed a problem with leading whitespace before markup. +# +# Revision 1.4 2007/12/13 13:09:16 ahby +# Updated copyright and license. +# +# Revision 1.3 2007/12/13 12:53:34 ahby +# Changed use of span to ins and del +# +# Revision 1.2 2002/02/13 16:27:23 ahby +# Changed processing model. +# Improved handling of old text and changed styles. +# +# Revision 1.1 2000/07/12 12:20:04 ahby +# Updated to remove empty spans - this fixes validation problems under +# strict. +# +# Revision 1.11 1999/12/08 19:46:45 ahby +# Fixed validation errors introduced by placing markup where it didn't +# belong. +# +# Revision 1.10 1999/10/18 13:42:58 ahby +# Added -o to the usage message. +# +# Revision 1.9 1999/05/04 12:29:11 ahby +# Added an option to turn off the display of old text. +# +# Revision 1.8 1999/04/09 14:37:27 ahby +# Fixed a perl syntax error. +# +# Revision 1.7 1999/04/09 14:35:49 ahby +# Added reference to MACS homepage. +# +# Revision 1.6 1999/04/09 14:35:09 ahby +# Added comment about validity of generated markup. +# +# Revision 1.5 1999/02/22 22:17:54 ahby +# Changed to use stylesheets. +# Changed to rely upon span. +# Changed to work around content model problems. +# +# Revision 1.4 1999/02/08 02:32:22 ahby +# Added a copyright statement. +# +# Revision 1.3 1999/02/08 02:30:40 ahby +# Added header processing. +# +# Revision 1.2 1998/12/10 17:31:31 ahby +# Fixed to escape less-thans in change blocks and to not permit change +# markup within specific elements (like TITLE). +# +# Revision 1.1 1998/11/26 00:09:22 ahby +# Initial revision +# +# + +use Getopt::Std; + +sub usage { + print STDERR "htmldiff [-c] [-o] oldversion newversion [output]\n"; + exit; +} + +sub url_encode { + my $str = shift; + $str =~ s/([\x00-\x1f\x7F-\xFF])/ + sprintf ('%%%02x', ord ($1))/eg; + return $str; +} + +# markit - diff-mark the streams +# +# markit(file1, file2) +# +# markit relies upon GNUdiff to mark up the text. +# +# The markup is encoded using special control sequences: +# +# a block wrapped in control-a is deleted text +# a block wrapped in control-b is old text +# a block wrapped in control-c is new text +# +# The main processing loop attempts to wrap the text blocks in appropriate +# SPANs based upon the type of text that it is. +# +# When the loop encounters a < in the text, it stops the span. Then it outputs +# the element that is defined, then it restarts the span. + +sub markit { + my $retval = ""; + my($file1) = shift; + my($file2) = shift; +# my $old="deleted text: %c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $old="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $new="%c'\012'%c'\003'%c'\012'%>%c'\012'%c'\003'%c'\012'"; + my $unchanged="%="; + my $changed="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + if ($opt_o) { + $old = ""; + $changed = "%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + } +# my $old="%c'\002'deleted text:%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\012'%c'\002'"; +# my $new="%c'\002'%c'\012'%c'\002'%>%c'\002'%c'\002'%c'\012'"; +# my $unchanged="%="; +# my $changed="%c'\002'%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\002'%c'\012'%>%c'\012'%c'\002'%c'\002'%c'\012'"; + + my @span; + $span[0]=""; + $span[1]=""; + $span[2]=""; + $span[3]=""; + $span[4]=""; + + my @diffEnd ; + $diffEnd[1] = ''; + $diffEnd[2] = ''; + $diffEnd[3] = ''; + $diffEnd[4] = ''; + + my $diffcounter = 0; + + open(FILE, qq(diff -d --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 |)) || die("Diff failed: $!"); +# system (qq(diff --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 > /tmp/output)); + + my $state = 0; + my $inblock = 0; + my $temp = ""; + my $lineCount = 0; + +# strategy: +# +# process the output of diff... +# +# a link with control A-D means the start/end of the corresponding ordinal +# state (1-4). Resting state is state 0. +# +# While in a state, accumulate the contents for that state. When exiting the +# state, determine if it is appropriate to emit the contents with markup or +# not (basically, if the accumulated buffer contains only empty lines or lines +# with markup, then we don't want to emit the wrappers. We don't need them. +# +# Note that if there is markup in the "old" block, that markup is silently +# removed. It isn't really that interesting, and it messes up the output +# something fierce. + + while () { + my $anchor = $opt_l ? qq[] : "" ; + my $anchorEnd = $opt_l ? q[] : "" ; + $lineCount ++; + if ($state == 0) { # if we are resting and we find a marker, + # then we must be entering a block + if (m/^([\001-\004])/) { + $state = ord($1); + $_ = ""; + } +# if (m/^\001/) { +# $state = 1; +# s/^/$span[1]/; +# } elsif (m/^\002/) { +# $state = 2; +# s/^/$span[2]/; +# } elsif (m/^\003/) { +# $state = 3; +# s/^/$span[3]/; +# } elsif (m/^\004/) { +# $state = 4; +# s/^/$span[4]/; +# } + } else { + # if we are in "old" state, remove markup + if (($state == 1) || ($state == 2)) { + s/\<.*\>//; # get rid of any old markup + s/\/>/g; + } + # if we found another marker, we must be exiting the state + if (m/^([\001-\004])/) { + if ($temp ne "") { + $_ = $span[$state] . $anchor . $temp . $anchorEnd . $diffEnd[$state] . "\n"; + $temp = ""; + } else { + $_ = "" ; + } + $state = 0; + } elsif (m/^\s*\ +.diff-old-a { + font-size: smaller; + color: red; +} + +.diff-new { background-color: yellow; } +.diff-chg { background-color: lime; } +.diff-new:before, +.diff-new:after + { content: "\2191" } +.diff-chg:before, .diff-chg:after + { content: "\2195" } +.diff-old { text-decoration: line-through; background-color: #FBB; } +.diff-old:before, +.diff-old:after + { content: "\2193" } +:focus { border: thin red solid} + +); + if ($opt_t) { + $styles .= q( + +); + + } + + if ($stripheader) { + open(HEADER, ">$headertmp"); + } + + my $incomment = 0; + my $inhead = 1; + open(FILE, $filename) || die("File $filename cannot be opened: $!"); + while () { + if ($inhead == 1) { + if (m/\<\/head/i) { + print HEADER $styles; + } + if (m/\ +); + } + close HEADER; + } else { + print HEADER; + } + } else { + if ($incomment) { + if (m;-->;) { + $incomment = 0; + s/.*-->//; + } else { + next; + } + } + if (m;;) { + s///; + } + if (m; - m.call.candidate --------> - [more candidates events] - User answers call - <------ m.call.answer - [...] - <------ m.call.hangup - -Or a rejected call: - -:: - - Caller Callee - m.call.invite -----------> - m.call.candidate --------> - [more candidates events] - User rejects call - <------- m.call.hangup - -Calls are negotiated according to the WebRTC specification. - - -Glare -~~~~~ -This specification aims to address the problem of two users calling each other -at roughly the same time and their invites crossing on the wire. It is a far -better experience for the users if their calls are connected if it is clear -that their intention is to set up a call with one another. - -In Matrix, calls are to rooms rather than users (even if those rooms may only -contain one other user) so we consider calls which are to the same room. - -The rules for dealing with such a situation are as follows: - - - If an invite to a room is received whilst the client is preparing to send an - invite to the same room, the client should cancel its outgoing call and - instead automatically accept the incoming call on behalf of the user. - - If an invite to a room is received after the client has sent an invite to - the same room and is waiting for a response, the client should perform a - lexicographical comparison of the call IDs of the two calls and use the - lesser of the two calls, aborting the greater. If the incoming call is the - lesser, the client should accept this call on behalf of the user. - -The call setup should appear seamless to the user as if they had simply placed -a call and the other party had accepted. Thusly, any media stream that had been -setup for use on a call should be transferred and used for the call that -replaces it. - diff --git a/specification/25_application_service_api.rst b/specification/3-application_service_api.rst similarity index 93% rename from specification/25_application_service_api.rst rename to specification/3-application_service_api.rst index 8236b2de..bdde7789 100644 --- a/specification/25_application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -4,11 +4,9 @@ Application Service API The Matrix client-server API and server-server APIs provide the means to implement a consistent self-contained federated messaging fabric. However, they provide limited means of implementing custom server-side behaviour in Matrix -(e.g. gateways, filters, extensible hooks etc). - -The Application Service API defines a standard API to allow such extensible -functionality to be implemented irrespective of the underlying homeserver -implementation. +(e.g. gateways, filters, extensible hooks etc). The Application Service API +defines a standard API to allow such extensible functionality to be implemented +irrespective of the underlying homeserver implementation. .. TODO-spec Add in Client-Server services? Overview of bots? Seems weird to be in the spec @@ -18,12 +16,10 @@ Passive Application Services ---------------------------- "Passive" application services can only observe events from a given home server. They cannot prevent events from being sent, nor can they modify the content of -the event being sent. - -In order to observe events from a homeserver, the homeserver needs to be -configured to pass certain types of traffic to the application service. This -is achieved by manually configuring the homeserver with information about the -AS.. +the event being sent. In order to observe events from a homeserver, the +homeserver needs to be configured to pass certain types of traffic to the +application service. This is achieved by manually configuring the homeserver +with information about the AS. .. NOTE:: Previously, application services could register with a homeserver via HTTP @@ -66,13 +62,13 @@ An example HS configuration required to pass traffic to the AS is: application service is merely augmenting the room itself (e.g. providing logging or searching facilities). - Namespaces are represented by POSIX extended regular expressions, - e.g.: + e.g: .. code-block:: yaml users: - exclusive: true - regex: @irc.freenode.net/.* + regex: @irc.freenode.net_.* Home Server -> Application Service API @@ -326,7 +322,7 @@ but only if the application service has defined the namespace as ``exclusive``. ID conventions ~~~~~~~~~~~~~~ -.. NOTE:: +.. TODO-spec - Giving HSes the freedom to namespace still feels like the Right Thing here. - Exposing a public API provides the consistency which was the main complaint against namespacing. @@ -345,7 +341,7 @@ types, including: - MSISDNs (``tel``) - Email addresses (``mailto``) - IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04) -- XMPP (xep-0032) +- XMPP (XEP-0032) - SIP URIs (RFC 3261) As a result, virtual user IDs SHOULD relate to their URI counterpart. This @@ -401,21 +397,3 @@ client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix -Active Application Services ----------------------------- - -.. TODO-spec - API that provides hooks into the server so that you can intercept and - manipulate events, and/or insert virtual users & rooms into the server. - -Policy Servers -============== -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - We should mention them in the Architecture section at least: how they fit - into the picture. - -Enforcing policies ------------------- diff --git a/specification/30_server_server_api.rst b/specification/4-server_server_api.rst similarity index 93% rename from specification/30_server_server_api.rst rename to specification/4-server_server_api.rst index 1c7bf3ed..c5ff2b87 100644 --- a/specification/30_server_server_api.rst +++ b/specification/4-server_server_api.rst @@ -2,10 +2,9 @@ Federation API ============== Matrix home servers use the Federation APIs (also known as server-server APIs) -to communicate with each other. -Home servers use these APIs to push messages to each other in real-time, to -request historic messages from each other, and to query profile and presence -information about users on each other's servers. +to communicate with each other. Home servers use these APIs to push messages to +each other in real-time, to request historic messages from each other, and to +query profile and presence information about users on each other's servers. The APIs are implemented using HTTPS GETs and PUTs between each of the servers. These HTTPS requests are strongly authenticated using public key @@ -21,7 +20,7 @@ Persisted Data Units (PDUs): context. Like email, it is the responsibility of the originating server of a PDU - to deliver that event to its recepient servers. However PDUs are signed + to deliver that event to its recipient servers. However PDUs are signed using the originating server's public key so that it is possible to deliver them through third-party servers. @@ -60,13 +59,11 @@ and an optional TLS port. .. ** If the port is present then the server is discovered by looking up an AAAA or -A record for the DNS name and connecting to the specified TLS port. - -If the port is absent then the server is discovered by looking up a -``_matrix._tcp`` SRV record for the DNS name. If this record does not exist -then the server is discovered by looking up an AAAA or A record on the DNS -name and taking the default fallback port number of 8448. - +A record for the DNS name and connecting to the specified TLS port. If the port +is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV +record for the DNS name. If this record does not exist then the server is +discovered by looking up an AAAA or A record on the DNS name and taking the +default fallback port number of 8448. Home servers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. @@ -84,18 +81,19 @@ directly or by querying an intermediate notary server using a response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. -This approach is borrowed from the Perspectives Project -(http://perspectives-project.org/), but modified to include the NACL keys and to -use JSON instead of XML. It has the advantage of avoiding a single trust-root -since each server is free to pick which notary servers they trust and can -corroborate the keys returned by a given notary server by querying other -servers. +This approach is borrowed from the `Perspectives Project`_, but modified to +include the NACL keys and to use JSON instead of XML. It has the advantage of +avoiding a single trust-root since each server is free to pick which notary +servers they trust and can corroborate the keys returned by a given notary +server by querying other servers. + +.. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON -object at ``/_matrix/key/v2/server/${key_id}``. The response contains a list of +object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the server and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list @@ -178,7 +176,7 @@ events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a @@ -510,7 +508,7 @@ To backfill events on a given context:: Retrieves a sliding-window history of previous PDUs that occurred on the given context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that -preceeded it are retrieved, up to a total number given by the "limit" argument. +preceded it are retrieved, up to a total number given by the "limit" argument. These are then returned in a new Transaction containing all of the PDUs. @@ -554,9 +552,7 @@ Every HTTP request made by a homeserver is authenticated using public key digital signatures. The request method, target and body are signed by wrapping them in a JSON object and signing it using the JSON signing algorithm. The resulting signatures are added as an Authorization header with an auth scheme -of X-Matrix. - -Note that the target field should include the full path starting with +of X-Matrix. Note that the target field should include the full path starting with ``/_matrix/...``, including the ``?`` and any query parameters if present, but should not include the leading ``https:``, nor the destination server's hostname. @@ -656,12 +652,12 @@ State Conflict Resolution - How does this work with deleting current state - How do we reject invalid federation traffic? -[[TODO(paul): At this point we should probably have a long description of how -State management works, with descriptions of clobbering rules, power levels, etc -etc... But some of that detail is rather up-in-the-air, on the whiteboard, and -so on. This part needs refining. And writing in its own document as the details -relate to the server/system as a whole, not specifically to server-server -federation.]] + [[TODO(paul): At this point we should probably have a long description of how + State management works, with descriptions of clobbering rules, power levels, etc + etc... But some of that detail is rather up-in-the-air, on the whiteboard, and + so on. This part needs refining. And writing in its own document as the details + relate to the server/system as a whole, not specifically to server-server + federation.]] Presence -------- @@ -677,8 +673,8 @@ Performing a presence update and poll subscription request:: Each should be an object with the following keys: user_id: string containing a User ID presence: "offline"|"unavailable"|"online"|"free_for_chat" - status_msg: (optional) string of freeform text - last_active_ago: miliseconds since the last activity by the user + status_msg: (optional) string of free-form text + last_active_ago: milliseconds since the last activity by the user poll: (optional): list of strings giving User IDs @@ -696,7 +692,7 @@ removed until explicitly requested by a later ``unpoll``. On receipt of a message containing a non-empty ``poll`` list, the receiving server should immediately send the sending server a presence update EDU of its own, containing in a ``push`` list the current state of every user that was in -the orginal EDU's ``poll`` list. +the original EDU's ``poll`` list. Sending a presence invite:: @@ -721,7 +717,7 @@ Rejecting a presence invite:: Content keys - as for m.presence_invite .. TODO-doc - - Explain the timing-based roundtrip reduction mechanism for presence + - Explain the timing-based round-trip reduction mechanism for presence messages - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence @@ -742,8 +738,8 @@ Querying profile information:: field: (optional) string giving a field name Returns: JSON object containing the following keys: - displayname: string of freeform text - avatar_url: string containing an http-scheme URL + displayname: string of free-form text + avatar_url: string containing an HTTP-scheme URL If the query contains the optional ``field`` key, it should give the name of a result field. If such is present, then the result should contain only a field @@ -769,3 +765,4 @@ Querying directory information:: The list of join candidates is a list of server names that are likely to hold the given room; these are servers that the requesting server may wish to try joining with. This list may or may not include the server answering the query. + diff --git a/specification/40_content_repository.rst b/specification/40_content_repository.rst deleted file mode 100644 index df04605a..00000000 --- a/specification/40_content_repository.rst +++ /dev/null @@ -1,76 +0,0 @@ -Content repository -================== - -HTTP API --------- - -Uploads are POSTed to a resource which returns a token which is used to GET -the download. Uploads are POSTed to the sender's local homeserver, but are -downloaded from the recipient's local homeserver, which must thus first transfer -the content from the origin homeserver using the same API (unless the origin -and destination homeservers are the same). The upload/download API is:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content-uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - -Clients can get thumbnails by supplying a desired width and height and -thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - -The thumbnail methods are "crop" and "scale". "scale" trys to return an -image where either the width or the height is smaller than the requested -size. The client should then scale and letterbox the image if it needs to -fit within a given rectangle. "crop" trys to return an image where the -width and height are close to the requested size and the aspect matches -the requested size. The client should scale the image if it needs to fit -within a given rectangle. - -Homeservers may generate thumbnails for content uploaded to remote -homeservers themselves or may rely on the remote homeserver to thumbnail -the content. Homeservers may return thumbnails of a different size to that -requested. However homeservers should provide exact matches where reasonable. -Homeservers must never upscale images. - -Security --------- - -Clients may try to upload very large files. Homeservers should not store files -that are too large and should not serve them to clients. - -Clients may try to upload very large images. Homeservers should not attempt to -generate thumbnails for images that are too large. - -Remote homeservers may host very large files or images. Homeserver should not -proxy or thumbnail large files or images from remote homeservers. - -Clients may try to upload a large number of files. Homeservers should limit the -number and total size of media that can be uploaded by clients. - -Clients may try to access a large number of remote files through a homeserver. -Homeservers should restrict the number and size of remote files that it caches. - -Clients or remote homeservers may try to upload malicious files targeting -vulnerabilities in either the homeserver thumbnailing or the client decoders. - diff --git a/specification/45_typing_notifications.rst b/specification/45_typing_notifications.rst deleted file mode 100644 index 7df9a238..00000000 --- a/specification/45_typing_notifications.rst +++ /dev/null @@ -1,58 +0,0 @@ -Typing Notifications -==================== - -Client APIs ------------ - -To set "I am typing for the next N msec":: - PUT .../rooms//typing/ - Content: { "typing": true, "timeout": N } - # timeout is in msec; I suggest no more than 20 or 30 seconds - -This should be re-sent by the client to continue informing the server the user -is still typing; I suggest a safety margin of 5 seconds before the expected -timeout runs out. Just keep declaring a new timeout, it will replace the old -one. - -To set "I am no longer typing":: - PUT ../rooms//typing/ - Content: { "typing": false } - -Client Events -------------- - -All room members will receive an event on the event stream:: - - { - "type": "m.typing", - "room_id": "!room-id-here:matrix.org", - "content": { - "user_ids": ["list of", "every user", "who is", "currently typing"] - } - } - -The client must use this list to *REPLACE* its knowledge of every user who is -currently typing. The reason for this is that the server DOES NOT remember -users who are not currently typing, as that list gets big quickly. The client -should mark as not typing, any user ID who is not in that list. - -Server APIs ------------ - -Servers will emit EDUs in the following form:: - - { - "type": "m.typing", - "content": { - "room_id": "!room-id-here:matrix.org", - "user_id": "@user-id-here:matrix.org", - "typing": true/false, - } - } - -Server EDUs don't (currently) contain timing information; it is up to -originating HSes to ensure they eventually send "stop" notifications. - -((This will eventually need addressing, as part of the wider typing/presence -timer addition work)) - diff --git a/specification/5-identity_servers.rst b/specification/5-identity_servers.rst new file mode 100644 index 00000000..6ec013bd --- /dev/null +++ b/specification/5-identity_servers.rst @@ -0,0 +1,8 @@ +Identity Servers +================ +.. NOTE:: + This section is a work in progress. + +.. TODO-doc Dave + - 3PIDs and identity server, functions + diff --git a/specification/50_appendices.rst b/specification/6-appendices.rst similarity index 96% rename from specification/50_appendices.rst rename to specification/6-appendices.rst index 295c8f69..de1ac290 100644 --- a/specification/50_appendices.rst +++ b/specification/6-appendices.rst @@ -128,11 +128,3 @@ Threat: Disclosure to Servers Within Chatroom An attacker could take control of a server within a chatroom to expose message contents or metadata for messages in that room. - -Identity Servers -================ -.. NOTE:: - This section is a work in progress. - -.. TODO-doc Dave - - 3PIDs and identity server, functions diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst new file mode 100644 index 00000000..9eee9843 --- /dev/null +++ b/specification/modules/_template.rst @@ -0,0 +1,56 @@ +Module Heading +============== + +.. _module:short-name: + +A short summary of the module. What features does this module provide? An anchor +should be specified at the top of the module using the format ``module:name``. + +Complicated modules may wish to have architecture diagrams or event flows +(e.g. VoIP call flows) here. Custom subsections can be included but they should +be used *sparingly* to reduce the risk of putting client or server behaviour +information in these custom sections. + +Events +------ +List the new event types introduced by this module, if any. If there are no +new events, this section can be omitted. Event types should be done as +subsections. This section is intended to document the "common shared event +structure" between client and server. Deviations from this shared structure +should be documented in the relevant behaviour section. + +``m.example.event.type`` +~~~~~~~~~~~~~~~~~~~~~~~~ +There should be JSON Schema docs for this event. Once there is JSON schema, +there will be a template variable with dots in the event type replaced with +underscores and the suffix ``_event``. You can insert a template like so: + +{{m_example_event_type_event}} + +Client behaviour +---------------- +List any new HTTP endpoints. These endpoints should be documented using Swagger. +Once there is Swagger, there will be a template variable based on the name of +the YAML file with the suffix ``_http_api``. You can insert a template for +swagger docs like so: + +{{name-of-yaml-file-without-file-ext_http_api}} + +List the steps the client needs to take to +correctly process this module. List what data structures the client should be +storing in order to aid implementation. + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. Federation-specific logic should be included +here. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst new file mode 100644 index 00000000..9ac5e199 --- /dev/null +++ b/specification/modules/content_repo.rst @@ -0,0 +1,74 @@ +Content repository +================== + +.. _module:content: + +This module allows users to upload content to their homeserver which is +retrievable from other homeservers. Its' purpose is to allow users to share +attachments in a room. Content locations are represented as Matrix Content (MXC) +URIs. They look like:: + + mxc:/// + + : The name of the homeserver where this content originated, e.g. matrix.org + : An opaque ID which identifies the content. + +Uploads are POSTed to a resource on the user's local homeserver which returns a +token which is used to GET the download. Content is downloaded from the +recipient's local homeserver, which must first transfer the content from the +origin homeserver using the same API (unless the origin and destination +homeservers are the same). + +Client behaviour +---------------- + +Clients can upload and download content using the following HTTP APIs. + +{{content_repo_http_api}} + +Thumbnails +~~~~~~~~~~ +The thumbnail methods are "crop" and "scale". "scale" tries to return an +image where either the width or the height is smaller than the requested +size. The client should then scale and letterbox the image if it needs to +fit within a given rectangle. "crop" tries to return an image where the +width and height are close to the requested size and the aspect matches +the requested size. The client should scale the image if it needs to fit +within a given rectangle. + +Server behaviour +---------------- + +Homeservers may generate thumbnails for content uploaded to remote +homeservers themselves or may rely on the remote homeserver to thumbnail +the content. Homeservers may return thumbnails of a different size to that +requested. However homeservers should provide exact matches where reasonable. +Homeservers must never upscale images. + +Security considerations +----------------------- + +The HTTP GET endpoint does not require any authentication. Knowing the URL of +the content is sufficient to retrieve the content, even if the entity isn't in +the room. + +Homeservers have additional concerns: + + - Clients may try to upload very large files. Homeservers should not store files + that are too large and should not serve them to clients. + + - Clients may try to upload very large images. Homeservers should not attempt to + generate thumbnails for images that are too large. + + - Remote homeservers may host very large files or images. Homeservers should not + proxy or thumbnail large files or images from remote homeservers. + + - Clients may try to upload a large number of files. Homeservers should limit the + number and total size of media that can be uploaded by clients. + + - Clients may try to access a large number of remote files through a homeserver. + Homeservers should restrict the number and size of remote files that it caches. + + - Clients or remote homeservers may try to upload malicious files targeting + vulnerabilities in either the homeserver thumbnailing or the client decoders. + diff --git a/specification/41_end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst similarity index 96% rename from specification/41_end_to_end_encryption.rst rename to specification/modules/end_to_end_encryption.rst index 02388152..e3a52613 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -1,6 +1,8 @@ End-to-End Encryption ===================== +.. _module:e2e: + .. TODO-doc - Why is this needed. - Overview of process diff --git a/specification/47_history_visibility.rst b/specification/modules/history_visibility.rst similarity index 93% rename from specification/47_history_visibility.rst rename to specification/modules/history_visibility.rst index b1630f1e..371282bd 100644 --- a/specification/47_history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -1,5 +1,7 @@ Room History Visibility -======================= +----------------------- + +.. _module:history-visibility: Whether a member of a room can see the events that happened in a room from before they joined the room is controlled by the ``history_visibility`` key diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst new file mode 100644 index 00000000..43a06aa1 --- /dev/null +++ b/specification/modules/instant_messaging.rst @@ -0,0 +1,29 @@ +Instant Messaging +================= + +.. _module:im: + +Events +------ + +{{m_room_message_event}} + +{{m_room_message_feedback_event}} + +{{m_room_name_event}} + +{{m_room_topic_event}} + +m.room.message msgtypes +----------------------- + +.. TODO-spec + How a client should handle unknown message types. + + +Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type +of message being sent. Each type has their own required and optional keys, as +outlined below. + +{{msgtype_events}} + diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst new file mode 100644 index 00000000..79151c4f --- /dev/null +++ b/specification/modules/presence.rst @@ -0,0 +1,112 @@ +Presence +======== + +.. _module:presence: + +Each user has the concept of presence information. This encodes: + + * Whether the user is currently online + * How recently the user was last active (as seen by the server) + * Whether a given client considers the user to be currently idle + * Arbitrary information about the user's current status (e.g. "in a meeting"). + +This information is collated from both per-device (``online``, ``idle``, +``last_active``) and per-user (status) data, aggregated by the user's homeserver +and transmitted as an ``m.presence`` event. This is one of the few events which +are sent *outside the context of a room*. Presence events are sent to all users +who subscribe to this user's presence through a presence list or by sharing +membership of a room. + +A presence list is a list of user IDs whose presence the user wants to follow. +To be added to this list, the user being added must be invited by the list owner +who must accept the invitation. + +User's presence state is represented by the ``presence`` key, which is an enum +of one of the following: + + - ``online`` : The default state when the user is connected to an event + stream. + - ``unavailable`` : The user is not reachable at this time e.g. they are + idle. + - ``offline`` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. + - ``free_for_chat`` : The user is generally willing to receive messages + moreso than default. + +Events +------ + +{{presence_events}} + +Client behaviour +---------------- + +Clients can manually set/get their presence/presence list using the HTTP APIs +listed below. + +{{presence_http_api}} + +Idle timeout +~~~~~~~~~~~~ + +Clients SHOULD implement an "idle timeout". This is a timer which fires after +a period of inactivity on the client. The definition of inactivity varies +depending on the client. For example, web implementations may determine +inactivity to be not moving the mouse for a certain period of time. When this +timer fires it should set the presence state to ``unavailable``. When the user +becomes active again (e.g. by moving the mouse) the client should set the +presence state to ``online``. A timeout value between 1 and 5 minutes is +recommended. + +Server behaviour +---------------- + +Each user's home server stores a "presence list" per user. Once a user accepts +a presence list, both user's HSes must track the subscription. + +Propagating profile information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields SHOULD cause an +automatic propagation event to occur, informing likely-interested parties of the +new values. One of these change mechanisms SHOULD be via ``m.presence`` events. +These events should set ``displayname`` and ``avatar_url`` to the new values +along with the presence-specific keys. This SHOULD be done automatically by the +home server when a user successfully changes their display name or avatar URL. + +.. admonition:: Rationale + + The intention for sending this information in ``m.presence`` is so that any + "user list" can display the *current* name/presence for a user ID outside the + scope of a room e.g. for a user page. This is bundled into a single event for + several reasons. The user's display name can change per room. This + event provides the "canonical" name for the user. In addition, the name is + bundled into a single event for the ease of client implementations. If this + was not done, the client would need to search all rooms for their own + membership event to pull out the display name. + + +Last active ago +~~~~~~~~~~~~~~~ +The server maintains a timestamp of the last time it saw a +pro-active event from the user. A pro-active event may be sending a message to a +room or changing presence state to a higher level of availability. Levels of +availability are defined from low to high as follows: + + - ``offline`` + - ``unavailable`` + - ``online`` + - ``free_for_chat`` + +Based on this list, changing state from ``unavailable`` to ``online`` counts as +a pro-active event, whereas ``online`` to ``unavailable`` does not. This +timestamp is presented via a key called ``last_active_ago`` which gives the +relative number of milliseconds since the pro-active event. + +Security considerations +----------------------- + +Presence information is shared with all users who share a room with the target +user. In large public rooms this could be undesirable. + diff --git a/specification/43_push_cs_api.rst b/specification/modules/push_cs_api.rst similarity index 93% rename from specification/43_push_cs_api.rst rename to specification/modules/push_cs_api.rst index 022f50c3..c6301926 100644 --- a/specification/43_push_cs_api.rst +++ b/specification/modules/push_cs_api.rst @@ -70,7 +70,7 @@ Room Rules Sender These rules configure notification behaviour for messages from a specific, named Matrix user ID. The rule_id of Sender rules is always the Matrix user - ID of the user whose messages theyt apply to. + ID of the user whose messages they'd apply to. Underride These are identical to override rules, but have a lower priority than content, room and sender rules. @@ -99,20 +99,17 @@ be redundant. Actions for the highest priority rule and only that rule apply (for example, a set_tweak action in a lower priority rule will not apply if a higher priority rule matches, even if that rule does not specify any tweaks). -Rules also have an identifier, rule_id, which is a string. The rule_id is -unique within the kind of rule and scope: rule_ids need not be unique between -rules of the same kind on different devices. - -A home server may also have server default rules of each kind and in each scope. -Server default rules are lower priority than user-defined rules in each scope. -Server default rules (and only server default rules) begin with a dot ('.') -character. - -In addition, all rules may be enabled or disabled. Disabled rules never match. +Rules also have an identifier, ``rule_id``, which is a string. The ``rule_id`` +is unique within the kind of rule and scope: ``rule_ids`` need not be unique +between rules of the same kind on different devices. A home server may also have +server default rules of each kind and in each scope. Server default rules are +lower priority than user-defined rules in each scope. Server default rules (and +only server default rules) begin with a dot ('.') character. In addition, all +rules may be enabled or disabled. Disabled rules never match. If no rules match an event, the Home Server should not notify for the message (that is to say, the default action is "dont-notify"). Events that the user sent -themself are never alerted for. +themselves are never alerted for. Predefined Rules ---------------- @@ -128,7 +125,7 @@ with these IDs, their semantics should match those given below: { "rule_id": ".m.rule.contains_user_name" - "pattern": "[the lcoal part of the user's Matrix ID]", + "pattern": "[the local part of the user's Matrix ID]", "actions": [ "notify", { @@ -220,7 +217,7 @@ with these IDs, their semantics should match those given below: Push Rules: Actions: -------------------- All rules have an associated list of 'actions'. An action affects if and how a -notification is delievered for a matching event. This standard defines the +notification is delivered for a matching event. This standard defines the following actions, although if Home servers wish to support more, they are free to do so: @@ -241,11 +238,11 @@ set_tweak Actions that have no parameters are represented as a string. Otherwise, they are represented as a dictionary with a key equal to their name and other keys as -their parameters, eg. { "set_tweak": "sound", "value": "default" } +their parameters, e.g. ``{ "set_tweak": "sound", "value": "default" }`` Push Rules: Actions: Tweaks --------------------------- -The 'set_tweak' key action is used to add an entry to the 'tweaks' dictionary +The ``set_tweak`` key action is used to add an entry to the 'tweaks' dictionary that is sent in the notification poke. The following tweaks are defined: sound @@ -275,7 +272,7 @@ do so: event_match This is a glob pattern match on a field of the event. Parameters: - * 'key': The dot-separated field of the event to match, eg. content.body + * 'key': The dot-separated field of the event to match, e.g. content.body * 'pattern': The glob-style pattern to match against. Patterns with no special glob characters should be treated as having asterisks prepended and appended when testing the condition. @@ -295,7 +292,7 @@ room_member_count '>=' or '<='. A prefix of '<' matches rooms where the member count is strictly less than the given number and so forth. If no prefix is present, this matches rooms where the member count is exactly equal to the given - number (ie. the same as '=='). + number (i.e. the same as '=='). Room, Sender, User and Content rules do not have conditions in the same way, but instead have predefined conditions, the behaviour of which can be configured @@ -314,7 +311,7 @@ scope Either 'global' or 'device/' to specify global rules or device rules for the given profile_tag. kind - The kind of rule, ie. 'override', 'underride', 'sender', 'room', 'content'. + The kind of rule, i.e. 'override', 'underride', 'sender', 'room', 'content'. rule_id The identifier for the rule. @@ -330,7 +327,7 @@ after rule. All requests to the push rules API also require an access_token as a query -paraemter. +parameter. The content of the PUT request is a JSON object with a list of actions under the 'actions' key and either conditions (under the 'conditions' key) or the diff --git a/specification/42_push_overview.rst b/specification/modules/push_overview.rst similarity index 99% rename from specification/42_push_overview.rst rename to specification/modules/push_overview.rst index 972a8eea..46028283 100644 --- a/specification/42_push_overview.rst +++ b/specification/modules/push_overview.rst @@ -1,6 +1,8 @@ Push Notifications ================== +.. _module:push: + Overview -------- diff --git a/specification/44_push_push_gw_api.rst b/specification/modules/push_push_gw_api.rst similarity index 93% rename from specification/44_push_push_gw_api.rst rename to specification/modules/push_push_gw_api.rst index b182503b..c970e93e 100644 --- a/specification/44_push_push_gw_api.rst +++ b/specification/modules/push_push_gw_api.rst @@ -1,7 +1,7 @@ HTTP Notification Protocol -------------------------- -This describes the format used by "http" pushers to send notifications of +This describes the format used by "HTTP" pushers to send notifications of events. Notifications are sent as HTTP POST requests to the URL configured when the @@ -77,10 +77,10 @@ counts This is a dictionary of the current number of unacknowledged communications for the recipient user. Counts whose value is zero are omitted. unread - The number of unread messages a user has accross all of the rooms they are a + The number of unread messages a user has across all of the rooms they are a member of. missed_calls - The number of unacknowledged missed calls a user has accross all rooms of + The number of unacknowledged missed calls a user has across all rooms of which they are a member. device This is an array of devices that the notification should be sent to. @@ -104,13 +104,13 @@ And additional key is defined but only present on member events: user_is_target This is true if the user receiving the notification is the subject of a member - event (ie. the state_key of the member event is equal to the user's Matrix + event (i.e. the state_key of the member event is equal to the user's Matrix ID). The recipient of an HTTP notification should respond with an HTTP 2xx response when the notification has been processed. If the endpoint returns an HTTP error code, the Home Server should retry for a reasonable amount of time with a -reasonable backoff scheme. +reasonable back-off scheme. The endpoint should return a JSON dictionary as follows:: diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst new file mode 100644 index 00000000..a8ad3cd3 --- /dev/null +++ b/specification/modules/receipts.rst @@ -0,0 +1,84 @@ +Receipts +======== + +.. _module:receipts: + +This module adds in support for receipts. These receipts are a form of +acknowledgement of an event. This module defines a single acknowledgement: +``m.read`` which indicates that the user has read up to a given event. + +Sending a receipt for each event can result in sending large amounts of traffic +to a homeserver. To prevent this from becoming a problem, receipts are implemented +using "up to" markers. This marker indicates that the acknowledgement applies +to all events "up to and including" the event specified. For example, marking +an event as "read" would indicate that the user had read all events *up to* the +referenced event. + +Events +------ +Each ``user_id``, ``receipt_type`` pair must be associated with only a +single ``event_id``. + +{{m_receipt_event}} + +Client behaviour +---------------- + +In v1 ``/initialSync``, receipts are listed in a separate top level ``receipts`` +key. In v2 ``/sync``, receipts are contained in the ``ephemeral`` block for a +room. New receipts that come down the event streams are deltas which update +existing mappings. Clients should replace older receipt acknowledgements based +on ``user_id`` and ``receipt_type`` pairs. For example:: + + Client receives m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $aaa:example.com + + Client receives another m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $bbb:example.com + + The client should replace the older acknowledgement for $aaa:example.com with + this one for $bbb:example.com + +Clients should send read receipts when there is some certainty that the event in +question has been **displayed** to the user. Simply receiving an event does not +provide enough certainty that the user has seen the event. The user SHOULD need +to *take some action* such as viewing the room that the event was sent to or +dismissing a notification in order for the event to count as "read". + +A client can update the markers for its user by interacting with the following +HTTP APIs. + +{{v2_receipts_http_api}} + +Server behaviour +---------------- + +For efficiency, receipts SHOULD be batched into one event per room before +delivering them to clients. + +Receipts are sent across federation as EDUs with type ``m.receipt``. The +format of the EDUs are:: + + { + : { + : { + : { } + }, + ... + }, + ... + } + +These are always sent as deltas to previously sent receipts. Currently only a +single ```` should be used: ``m.read``. + +Security considerations +----------------------- + +As receipts are sent outside the context of the event graph, there are no +integrity checks performed on the contents of ``m.receipt`` events. + diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst new file mode 100644 index 00000000..d2253632 --- /dev/null +++ b/specification/modules/typing_notifications.rst @@ -0,0 +1,66 @@ +Typing Notifications +==================== + +.. _module:typing: + +Users may wish to be informed when another user is typing in a room. This can be +achieved using typing notifications. These are ephemeral events scoped to a +``room_id``. This means they do not form part of the `Event Graph`_ but still +have a ``room_id`` key. + +.. _Event Graph: `sect:event-graph`_ + +Events +------ + +{{m_typing_event}} + +Client behaviour +---------------- + +When a client receives an ``m.typing`` event, it MUST use the user ID list to +**REPLACE** its knowledge of every user who is currently typing. The reason for +this is that the server *does not remember* users who are not currently typing +as that list gets big quickly. The client should mark as not typing any user ID +who is not in that list. + +It is recommended that clients store a ``boolean`` indicating whether the user +is typing or not. Whilst this value is ``true`` a timer should fire periodically +every N seconds to send a typing HTTP request. The value of N is recommended to +be no more than 20-30 seconds. This request should be re-sent by the client to +continue informing the server the user is still typing. As subsequent +requests will replace older requests, a safety margin of 5 seconds before the +expected timeout runs out is recommended. When the user stops typing, the +state change of the ``boolean`` to ``false`` should trigger another HTTP request +to inform the server that the user has stopped typing. + +{{typing_http_api}} + +Server behaviour +---------------- + +Servers MUST emit typing EDUs in a different form to ``m.typing`` events which +are shown to clients. This form looks like:: + + { + "type": "m.typing", + "content": { + "room_id": "!room-id-here:matrix.org", + "user_id": "@user-id-here:matrix.org", + "typing": true/false + } + } + +This does not contain timing information so it is up to originating homeservers +to ensure they eventually send "stop" notifications. + +.. TODO + ((This will eventually need addressing, as part of the wider typing/presence + timer addition work)) + +Security considerations +----------------------- + +Clients may not wish to inform everyone in a room that they are typing and +instead only specific users in the room. + diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst new file mode 100644 index 00000000..a7b02538 --- /dev/null +++ b/specification/modules/voip_events.rst @@ -0,0 +1,101 @@ +Voice over IP +============= + +.. _module:voip: + +This module outlines how two users in a room can set up a Voice over IP (VoIP) +call to each other. Voice and video calls are built upon the WebRTC 1.0 standard. +Call signalling is achieved by sending `message events`_ to the room. As a result, +this means that clients MUST only send call events to rooms with exactly two +participants as currently the WebRTC standard is based around two-party +communication. + +.. _message events: `sect:events`_ + +Events +------ + +{{voip_events}} + +Client behaviour +---------------- + +A call is set up with message events exchanged as follows: + +:: + + Caller Callee + [Place Call] + m.call.invite -----------> + m.call.candidate --------> + [..candidates..] --------> + [Answers call] + <--------------- m.call.answer + [Call is active and ongoing] + <--------------- m.call.hangup + +Or a rejected call: + +:: + + Caller Callee + m.call.invite ------------> + m.call.candidate ---------> + [..candidates..] ---------> + [Rejects call] + <-------------- m.call.hangup + +Calls are negotiated according to the WebRTC specification. + +Glare +~~~~~ + +"Glare" is a problem which occurs when two users call each other at roughly the +same time. This results in the call failing to set up as there already is an +incoming/outgoing call. A glare resolution algorithm can be used to determine +which call to hangup and which call to answer. If both clients implement the +same algorithm then they will both select the same call and the call will be +successfully connected. + + +As calls are "placed" to rooms rather than users, the glare resolution algorithm +outlined below is only considered for calls which are to the same room. The +algorithm is as follows: + + - If an ``m.call.invite`` to a room is received whilst the client is + **preparing to send** an ``m.call.invite`` to the same room: + + * the client should cancel its outgoing call and instead + automatically accept the incoming call on behalf of the user. + + - If an ``m.call.invite`` to a room is received **after the client has sent** + an ``m.call.invite`` to the same room and is waiting for a response: + + * the client should perform a lexicographical comparison of the call IDs of + the two calls and use the *lesser* of the two calls, aborting the + greater. If the incoming call is the lesser, the client should accept + this call on behalf of the user. + + +The call setup should appear seamless to the user as if they had simply placed +a call and the other party had accepted. This means any media stream that had been +setup for use on a call should be transferred and used for the call that +replaces it. + +Server behaviour +---------------- + +The homeserver MAY provide a TURN server which clients can use to contact the +remote party. The following HTTP API endpoints will be used by clients in order +to get information about the TURN server. + +{{voip_http_api}} + + +Security considerations +----------------------- + +Calls should only be placed to rooms with one other user in them. If they are +placed to group chat rooms it is possible that another user will intercept and +answer the call. + diff --git a/specification/targets.yaml b/specification/targets.yaml new file mode 100644 index 00000000..d77bf8b5 --- /dev/null +++ b/specification/targets.yaml @@ -0,0 +1,40 @@ +targets: + main: # arbitrary name to identify this build target + files: # the sort order of files to cat + - 0-intro.rst + - 1-client_server_api.rst + - { 1: 0-events.rst } + - { 1: 0-event_signing.rst } + - 2-modules.rst + - { 1: 0-feature_profiles.rst } + - { 1: "group:modules" } # reference a group of files + - 3-application_service_api.rst + - 4-server_server_api.rst + - 5-identity_servers.rst + - 6-appendices.rst +groups: # reusable blobs of files when prefixed with 'group:' + modules: + - modules/instant_messaging.rst + - modules/voip_events.rst + - modules/typing_notifications.rst + - modules/receipts.rst + - modules/presence.rst + - modules/content_repo.rst + - modules/end_to_end_encryption.rst + - modules/history_visibility.rst + - modules/push_overview.rst + # relative depth + - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } + +title_styles: ["=", "-", "~", "+", "^"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 11c34fb3..c18c9a5f 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -16,7 +16,7 @@ class Sections(object): def log(self, text): if self.debug: - print text + print "batesian:sections: %s" % text def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -27,12 +27,38 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if not isinstance(section, basestring): + if isinstance(section, basestring): + if section_key in section_dict: + raise Exception( + ("%s : Section %s already exists. It must have been " + + "generated dynamically. Check which render_ methods " + + "return a dict.") % + (func_name, section_key) + ) + section_dict[section_key] = section + self.log( + " Generated. Snippet => %s" % section[:60].replace("\n","") + ) + elif isinstance(section, dict): + self.log(" Generated multiple sections:") + for (k, v) in section.iteritems(): + if not isinstance(k, basestring) or not isinstance(v, basestring): + raise Exception( + ("Method %s returned multiple sections as a dict but " + + "expected the dict elements to be strings but they aren't.") % + (func_name, ) + ) + if k in section_dict: + raise Exception( + "%s tried to produce section %s which already exists." % + (func_name, k) + ) + section_dict[k] = v + self.log( + " %s => %s" % (k, v[:60].replace("\n","")) + ) + else: raise Exception( - "Section function '%s' didn't return a string!" % func_name + "Section function '%s' didn't return a string/dict!" % func_name ) - section_dict[section_key] = section - self.log( - " Generated. Snippet => %s" % section[:60].replace("\n","") - ) return section_dict \ No newline at end of file diff --git a/templating/batesian/units.py b/templating/batesian/units.py index c20a2d1f..144cf245 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,11 @@ class Units(object): def log(self, text): if self.debug: - print text + func_name = "" + trace = inspect.stack() + if len(trace) > 1 and len(trace[1]) > 2: + func_name = trace[1][3] + ":" + print "batesian:units:%s %s" % (func_name, text) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/build.py b/templating/build.py index dadf91ce..b91e1da2 100755 --- a/templating/build.py +++ b/templating/build.py @@ -38,7 +38,7 @@ Processing """ from batesian import AccessKeyStore -from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template +from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta from argparse import ArgumentParser, FileType import importlib import json @@ -52,8 +52,8 @@ def create_from_template(template, sections): def check_unaccessed(name, store): unaccessed_keys = store.get_unaccessed_set() if len(unaccessed_keys) > 0: - print "Found %s unused %s keys." % (len(unaccessed_keys), name) - print unaccessed_keys + log("Found %s unused %s keys." % (len(unaccessed_keys), name)) + log(unaccessed_keys) def main(input_module, file_stream=None, out_dir=None, verbose=False): if out_dir and not os.path.exists(out_dir): @@ -78,8 +78,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): def wrap(input, wrap=80, initial_indent=""): if len(input) == 0: return initial_indent + # TextWrapper collapses newlines into single spaces; we do our own + # splitting on newlines to prevent this, so that newlines can actually + # be intentionally inserted in text. + input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) - return wrapper.fill(input) + output_lines = [wrapper.fill(line) for line in input_lines] + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters env = Environment( @@ -116,17 +121,31 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return # check the input files and substitute in sections where required - print "Parsing input template: %s" % file_stream.name - temp = Template(file_stream.read()) - print "Creating output for: %s" % file_stream.name + log("Parsing input template: %s" % file_stream.name) + temp_str = file_stream.read().decode("utf-8") + # do sanity checking on the template to make sure they aren't reffing things + # which will never be replaced with a section. + ast = env.parse(temp_str) + template_vars = meta.find_undeclared_variables(ast) + unused_vars = [var for var in template_vars if var not in sections] + if len(unused_vars) > 0: + raise Exception( + "You have {{ variables }} which are not found in sections: %s" % + (unused_vars,) + ) + # process the template + temp = Template(temp_str) + log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: - f.write(output) - print "Output file for: %s" % file_stream.name + f.write(output.encode("utf-8")) + log("Output file for: %s" % file_stream.name) check_unaccessed("units", units) +def log(line): + print "batesian: %s" % line if __name__ == '__main__': parser = ArgumentParser( @@ -170,7 +189,7 @@ if __name__ == '__main__': sys.exit(0) if not args.file: - print "No file supplied." + log("No file supplied.") parser.print_help() sys.exit(1) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 7bc25095..e75a75af 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,28 +72,40 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def render_profile_http_api(self): - return self._render_http_api_group( - "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" - ) - def render_sync_http_api(self): - return self._render_http_api_group( - "sync" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_http_apis(self): + # map all swagger_apis to the form $GROUP_http_api + swagger_groups = self.units.get("swagger_apis").keys() + renders = {} + for group in swagger_groups: + sortFnOrPathList = None + if group == "presence": + sortFnOrPathList = ["status"] + elif group == "profile": + sortFnOrPathList=["displayname", "avatar_url"] + renders[group + "_http_api"] = self._render_http_api_group( + group, sortFnOrPathList + ) + return renders - def render_presence_http_api(self): - return self._render_http_api_group( - "presence", - sortFnOrPathList=["status"], - title_kind="~" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_events(self): + # map all event schemata to the form $EVENTTYPE_event with s/./_/g + # e.g. m_room_topic_event + schemas = self.units.get("event_schemas") + renders = {} + for event_type in schemas: + renders[event_type.replace(".", "_") + "_event"] = self._render_events( + lambda x: x == event_type, sorted + ) + return renders def render_room_events(self): def filterFn(eventType): @@ -102,6 +119,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -117,7 +137,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -138,12 +159,17 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") ce_types = self.units.get("common_event_fields") - return template.render(common_event=ce_types[type]) + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + return template.render( + common_event=ce_types[type], title_kind=subtitle_title_char + ) def render_common_event_fields(self): return self._render_ce_type("event") @@ -153,3 +179,4 @@ class MatrixSections(Sections): def render_common_state_event_fields(self): return self._render_ce_type("state_event") + diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 2a3f7ec5..3f16be3d 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,5 +1,5 @@ {{common_event.title}} Fields -{{(7 + common_event.title | length) * '-'}} +{{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 5e80f168..472c9d7a 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,5 +1,10 @@ ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} +{% if "alias_for_path" in endpoint -%} +``{{endpoint.path}}`` is an alias for `{{endpoint.alias_for_path}}`_. + +.. _`{{endpoint.alias_for_path}}`: #{{endpoint.alias_link}} +{% else -%} {{endpoint.desc | wrap(80)}} @@ -13,7 +18,7 @@ Request format: ================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* --------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} {{param.key}}{{param.type|indent(19-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(37)}} {% endfor -%} @@ -26,18 +31,18 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== +=================== ================= ========================================== Param Type Description -================== ================= =========================================== +=================== ================= ========================================== {% for row in table.rows -%} {# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} +{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} +{# It also needs to then wrap inside the desc col (42 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(20-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(18 - (row.type|length))) |indent_block(38)}} {% endfor -%} -================== ================= =========================================== +=================== ================= ========================================== {% endfor %} {% endif -%} @@ -46,6 +51,22 @@ Example request:: {{endpoint.example.req | indent_block(2)}} -Example response:: +{% if endpoint.example.responses|length > 0 -%} +Response{{"s" if endpoint.example.responses|length > 1 else "" }}: + +{% endif -%} + +{% for res in endpoint.example.responses -%} - {{endpoint.example.res | indent_block(2)}} \ No newline at end of file +**Status code {{res["code"]}}:** + +{{res["description"]}} + +Example + +.. code:: json + + {{res["example"] | indent_block(2)}} + +{% endfor %} +{% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160..f7862451 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index d607160f..eebfba84 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -1,4 +1,12 @@ -"""Contains all the units for the spec.""" +""" +Contains all the units for the spec. + +This file loads swagger and JSON schema files and parses out the useful bits +and returns them as Units for use in Batesian. + +For the actual conversion of data -> RST (including templates), see the sections +file instead. +""" from batesian.units import Units import inspect import json @@ -8,6 +16,18 @@ import subprocess import urllib import yaml +V1_CLIENT_API = "../api/client-server/v1" +V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" +V1_EVENT_SCHEMA = "../event-schemas/schema/v1" +V2_CLIENT_API = "../api/client-server/v2_alpha" +CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" + +ROOM_EVENT = "core-event-schema/room_event.json" +STATE_EVENT = "core-event-schema/state_event.json" + + def get_json_schema_object_fields(obj, enforce_title=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -30,8 +50,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): } tables = [fields] - props = obj.get("properties") parents = obj.get("allOf") + props = obj.get("properties") + if not props: + props = obj.get("patternProperties") + if props: + # try to replace horrible regex key names with pretty x-pattern ones + for key_name in props.keys(): + pretty_key = props[key_name].get("x-pattern") + if pretty_key: + props[pretty_key] = props[key_name] + del props[key_name] if not props and not parents: raise Exception( "Object %s has no properties or parents." % obj @@ -51,10 +80,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): if props[key_name]["type"] == "object": if props[key_name].get("additionalProperties"): # not "really" an object, just a KV store - value_type = ( - "{string: %s}" % - props[key_name]["additionalProperties"]["type"] - ) + prop_val = props[key_name]["additionalProperties"]["type"] + if prop_val == "object": + nested_object = get_json_schema_object_fields( + props[key_name]["additionalProperties"], + enforce_title=True + ) + value_type = "{string: %s}" % nested_object[0]["title"] + if not nested_object[0].get("no-table"): + tables += nested_object + else: + value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( props[key_name], @@ -87,6 +123,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): desc += ( " Must be '%s'." % props[key_name]["enum"][0] ) + if isinstance(value_type, list): + value_type = " or ".join(value_type) fields["rows"].append({ "key": key_name, @@ -105,21 +143,23 @@ class MatrixUnits(Units): for path in api["paths"]: for method in api["paths"][path]: single_api = api["paths"][path][method] + full_path = api.get("basePath", "") + path endpoint = { "title": single_api.get("summary", ""), "desc": single_api.get("description", single_api.get("summary", "")), "method": method.upper(), - "path": api.get("basePath", "") + path, + "path": full_path, "requires_auth": "security" in single_api, "rate_limited": 429 in single_api.get("responses", {}), "req_params": [], "res_tables": [], "example": { "req": "", - "res": "" + "responses": [], + "good_response": "" } } - self.log(".o.O.o. Endpoint: %s %s" % (method, path)) + self.log(" ------- Endpoint: %s %s ------- " % (method, path)) for param in single_api.get("parameters", []): # description desc = param.get("description", "") @@ -128,6 +168,13 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + + if param.get("enum"): + val_type = "enum" + desc += ( + " One of: %s" % json.dumps(param.get("enum")) + ) + refType = Units.prop(param, "schema/$ref/") # Error,Event schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads if not val_type and refType: @@ -147,8 +194,8 @@ class MatrixUnits(Units): # object with some keys; we'll add entries f.e one) if "schema" not in param: raise Exception( - "API endpoint group=%s path=%s method=%s param=%s"+ - " has no valid parameter value." % ( + ("API endpoint group=%s path=%s method=%s param=%s"+ + " has no valid parameter value.") % ( group_name, path, method, param ) ) @@ -168,6 +215,9 @@ class MatrixUnits(Units): "desc": json_body[key]["description"] }) # endfor[param] + for row in endpoint["req_params"]: + self.log("Request parameter: %s" % row) + # group params by location to ease templating endpoint["req_param_by_loc"] = { # path: [...], query: [...], body: [...] @@ -177,11 +227,19 @@ class MatrixUnits(Units): endpoint["req_param_by_loc"][p["loc"]] = [] endpoint["req_param_by_loc"][p["loc"]].append(p) - # add example response if it has one - res = single_api["responses"][200] # get the 200 OK response - endpoint["example"]["res"] = res.get("examples", {}).get( - "application/json", "" - ) + good_response = None + for code, res in single_api.get("responses", {}).items(): + if not good_response and code == 200: + good_response = res + description = res.get("description", "") + example = res.get("examples", {}).get("application/json", "") + if description and example: + endpoint["example"]["responses"].append({ + "code": code, + "description": description, + "example": example, + }) + # form example request if it has one. It "has one" if all params # have either "x-example" or a "schema" with an "example". params_missing_examples = [ @@ -216,25 +274,79 @@ class MatrixUnits(Units): ) # add response params if this API has any. - res_type = Units.prop(res, "schema/type") - if res_type and res_type not in ["object", "array"]: - # response is a raw string or something like that - endpoint["res_tables"].append({ - "title": None, - "rows": [{ - "key": res["schema"].get("name", ""), - "type": res_type, - "desc": res.get("description", "") - }] - }) - elif res_type and Units.prop(res, "schema/properties"): # object - res_tables = get_json_schema_object_fields(res["schema"]) - for table in res_tables: - if "no-table" not in table: - endpoint["res_tables"].append(table) + if good_response: + self.log("Found a 200 response for this API") + res_type = Units.prop(good_response, "schema/type") + res_name = Units.prop(good_response, "schema/name") + if res_type and res_type not in ["object", "array"]: + # response is a raw string or something like that + good_table = { + "title": None, + "rows": [{ + "key": "<" + res_type + ">" if not res_name else res_name, + "type": res_type, + "desc": res.get("description", ""), + "req_str": "" + }] + } + if good_response.get("headers"): + for (header_name, header) in good_response.get("headers").iteritems(): + good_table["rows"].append({ + "key": header_name, + "type": "Header<" + header["type"] + ">", + "desc": header["description"], + "req_str": "" + }) + endpoint["res_tables"].append(good_table) + elif res_type and Units.prop(good_response, "schema/properties"): + # response is an object: + schema = good_response["schema"] + res_tables = get_json_schema_object_fields(schema) + for table in res_tables: + if "no-table" not in table: + endpoint["res_tables"].append(table) + elif res_type and Units.prop(good_response, "schema/items"): + # response is an array: + # FIXME: Doesn't recurse at all. + schema = good_response["schema"] + array_type = Units.prop(schema, "items/type") + if Units.prop(schema, "items/allOf"): + array_type = ( + Units.prop(schema, "items/title") + ) + endpoint["res_tables"].append({ + "title": schema.get("title", ""), + "rows": [{ + "key": "N/A", + "type": ("[%s]" % array_type), + "desc": schema.get("description", ""), + "req_str": "" + }] + }) + + for response_table in endpoint["res_tables"]: + self.log("Response: %s" % response_table["title"]) + for r in response_table["rows"]: + self.log("Row: %s" % r) + if len(endpoint["res_tables"]) == 0: + self.log( + "This API appears to have no response table. Are you " + + "sure this API returns no parameters?" + ) endpoints.append(endpoint) + aliases = single_api.get("x-alias", None) + if aliases: + alias_link = aliases["canonical-link"] + for alias in aliases["aliases"]: + endpoints.append({ + "method": method.upper(), + "path": alias, + "alias_for_path": full_path, + "alias_link": alias_link + }) + return { "base": api.get("basePath"), "group": group_name, @@ -242,34 +354,57 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - path = "../api/client-server/v1" + paths = [ + V1_CLIENT_API, V2_CLIENT_API + ] apis = {} - for filename in os.listdir(path): - if not filename.endswith(".yaml"): + for path in paths: + is_v2 = (path == V2_CLIENT_API) + if not os.path.exists(V2_CLIENT_API): + self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) continue - self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: - # strip .yaml - group_name = filename[:-5] - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) - apis[group_name] = api + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + self.log("Reading swagger API: %s" % filename) + with open(os.path.join(path, filename), "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + if is_v2: + group_name = "v2_" + group_name + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta(api, group_name) + apis[group_name] = api return apis def load_common_event_fields(self): - path = "../event-schemas/schema/v1/core" + path = CORE_EVENT_SCHEMA event_types = {} - with open(path, "r") as f: - core_json = json.loads(f.read()) - for event_type in core_json["definitions"]: + + for (root, dirs, files) in os.walk(path): + for filename in files: + if not filename.endswith(".json"): + continue + + event_type = filename[:-5] # strip the ".json" + filepath = os.path.join(root, filename) + with open(filepath) as f: + try: + event_info = json.load(f) + except Exception as e: + raise ValueError( + "Error reading file %r" % (filepath,), e + ) + if "event" not in event_type: continue # filter ImageInfo and co - event_info = core_json["definitions"][event_type] + table = { "title": event_info["title"], "desc": event_info["description"], "rows": [] } + for prop in sorted(event_info["properties"]): row = { "key": prop, @@ -277,11 +412,12 @@ class MatrixUnits(Units): "desc": event_info["properties"][prop].get("description","") } table["rows"].append(row) + event_types[event_type] = table return event_types def load_event_examples(self): - path = "../event-schemas/examples/v1" + path = V1_EVENT_EXAMPLES examples = {} for filename in os.listdir(path): if not filename.startswith("m."): @@ -293,7 +429,7 @@ class MatrixUnits(Units): return examples def load_event_schemas(self): - path = "../event-schemas/schema/v1" + path = V1_EVENT_SCHEMA schemata = {} for filename in os.listdir(path): @@ -322,8 +458,8 @@ class MatrixUnits(Units): # add typeof base_defs = { - "core#/definitions/room_event": "Message Event", - "core#/definitions/state_event": "State Event" + ROOM_EVENT: "Message Event", + STATE_EVENT: "State Event" } if type(json_schema.get("allOf")) == list: schema["typeof"] = base_defs.get( @@ -360,7 +496,6 @@ class MatrixUnits(Units): "`m.room.message msgtypes`_." ) - # Assign state key info if it has some if schema["typeof"] == "State Event": skey_desc = Units.prop( @@ -374,7 +509,7 @@ class MatrixUnits(Units): return schemata def load_spec_meta(self): - path = "../CHANGELOG.rst" + path = CHANGELOG title_part = None version = None changelog_lines = [] @@ -405,7 +540,7 @@ class MatrixUnits(Units): if re.match("^v[0-9\.]+$", word): version = word[1:] # strip the 'v' - self.log("Version: %s Title part: %s Changelog lines: %s" % ( + self.log("Version: %s Title part: %s Changelog line count: %s" % ( version, title_part, len(changelog_lines) )) if not version or len(changelog_lines) == 0: @@ -416,6 +551,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__))