diff --git a/.gitignore b/.gitignore index df0135f5e..74b1c7d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ scripts/gen +scripts/continuserv/continuserv +scripts/speculator/speculator templating/out *.pyc +supporting-docs/_site +supporting-docs/.sass-cache +api/node_modules diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78aebddc9..6e3198ef9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,34 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.2.0 (2015-10-02) +============================================ + +This update fundamentally restructures the specification. The specification has +been split into more digestible "modules" which each describe a particular +function (e.g. typing). This was done in order make the specification easier to +maintain and help define which modules are mandatory for certain types +of clients. Types of clients along with the mandatory modules can be found in a +new "Feature Profiles" section. This update also begins to aggressively +standardise on using Swagger and JSON Schema to document HTTP endpoints and +Events respectively. It also introduces a number of new concepts to Matrix. + +Additions: + - New section: Feature Profiles. + - New section: Receipts. + - New section: Room history visibility. + - New event: ``m.receipt``. + - New event: ``m.room.canonical_alias`` + - New event: ``m.room.history_visibility`` + - New keys: ``/createRoom`` - allows room "presets" using ``preset`` and + ``initial_state`` keys. + - New endpoint: ``/tokenrefresh`` - Related to refreshing access tokens. + +Modifications: + - Convert most of the older HTTP APIs to Swagger documentation. + - Convert most of the older event formats to JSON Schema. + - Move selected client-server sections to be "Modules". + Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. diff --git a/api/check_examples.py b/api/check_examples.py new file mode 100755 index 000000000..ee3c773c1 --- /dev/null +++ b/api/check_examples.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python + +import sys +import json +import os + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + + +def check_parameter(filepath, request, parameter): + schema = parameter.get("schema") + example = None + try: + example_json = schema.get('example') + if example_json and not schema.get("format") == "byte": + example = json.loads(example_json) + except Exception as e: + raise ValueError("Error parsing JSON example request for %r" % ( + request + ), e) + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking request schema for: %r %r" % ( + filepath, request + )) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r" % ( + request + ), e) + + +def check_response(filepath, request, code, response): + example = None + try: + example_json = response.get('examples', {}).get('application/json') + if example_json: + example = json.loads(example_json) + except Exception as e: + raise ValueError("Error parsing JSON example response for %r %r" % ( + request, code + ), e) + schema = response.get('schema') + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking response schema for: %r %r %r" % ( + filepath, request, code + )) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + + +def check_swagger_file(filepath): + with open(filepath) as f: + swagger = yaml.load(f) + + for path, path_api in swagger.get('paths', {}).items(): + + for method, request_api in path_api.items(): + request = "%s %s" % (method.upper(), path) + for parameter in request_api.get('parameters', ()): + if parameter['in'] == 'body': + check_parameter(filepath, request, parameter) + + try: + responses = request_api['responses'] + except KeyError: + raise ValueError("No responses for %r" % (request,)) + for code, response in responses.items(): + check_response(filepath, request, code, response) + + +if __name__ == '__main__': + paths = sys.argv[1:] + if not paths: + paths = [] + for (root, dirs, files) in os.walk(os.curdir): + for filename in files: + if filename.endswith(".yaml"): + paths.append(os.path.join(root, filename)) + for path in paths: + try: + check_swagger_file(path) + except Exception as e: + raise ValueError("Error checking file %r" % (path,), e) diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/v1/application_service.yaml new file mode 100644 index 000000000..1a5ec8382 --- /dev/null +++ b/api/client-server/v1/application_service.yaml @@ -0,0 +1,201 @@ +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "/" +consumes: + - application/json +produces: + - application/json +paths: + "/transactions/{txnId}": + put: + summary: Send some events to the application service. + description: |- + This API is called by the HS when the HS wants to push an event (or + batch of events) to the AS. + parameters: + - in: path + name: txnId + type: string + description: |- + The transaction ID for this set of events. Homeservers generate + these IDs and they are used to ensure idempotency of requests. + required: true + x-example: "35" + - in: body + name: body + description: A list of events + schema: + type: object + example: |- + { + "events": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + }, + { + "age": 1984, + "content": { + "body": "another incoming message", + "msgtype": "m.text" + }, + "event_id": "$1228055551ffsef:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + description: "Transaction informations" + properties: + events: + type: array + description: A list of events + items: + type: object + title: Event + required: ["events"] + responses: + 200: + description: The transaction was processed successfully. + examples: + application/json: |- + {} + schema: + type: object + + "/rooms/{roomAlias}": + get: + summary: Query if a room alias should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given room alias. The homeserver will only query room + aliases inside the application service's ``aliases`` namespace. The + homeserver will send this request when it receives a request to join a + room alias within the application service's namespace. + parameters: + - in: path + name: roomAlias + type: string + description: The room alias being queried. + required: true + x-example: "#magicforest:example.com" + responses: + 200: + description: |- + The application service indicates that this room alias exists. The + application service MUST have created a room and associated it with + the queried room alias using the client-server API. Additional + information about the room such as its name and topic can be set + before responding. + examples: + application/json: |- + {} + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: |- + The application service indicates that this room alias does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + type: object + "/users/{userId}": + get: + summary: Query if a user should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given user ID. The homeserver will only query user IDs + inside the application service's ``users`` namespace. The homeserver will + send this request when it receives an event for an unknown user ID in + the application service's namespace. + parameters: + - in: path + name: userId + type: string + description: The user ID being queried. + required: true + x-example: "@alice:example.com" + responses: + 200: + description: |- + The application service indicates that this user exists. The application + service MUST create the user using the client-server API. + examples: + application/json: |- + {} + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: |- + The application service indicates that this user does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: |- + { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + type: object + diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index fe3d1dc37..8e6e8d1a4 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -15,16 +15,22 @@ paths: summary: Upload some content to the content repository. produces: ["application/json"] parameters: + - in: header + name: Content-Type + type: string + description: The content type of the file being uploaded + x-example: "Content-Type: audio/mpeg" - in: body - name: content + name: "" description: The content to be uploaded. required: true schema: type: string + example: "" format: byte responses: 200: - description: Information about the uploaded content. + description: The MXC URI for the uploaded content. schema: type: object required: ["content_uri"] @@ -32,6 +38,11 @@ paths: content_uri: type: string description: "The MXC URI to the uploaded content." + examples: + "application/json": |- + { + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + } "/download/{serverName}/{mediaId}": get: summary: "Download content from the content repository." @@ -40,18 +51,27 @@ paths: - in: path type: string name: serverName + x-example: matrix.org required: true description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) responses: 200: - description: "The content downloaded." + description: "The content that was previously uploaded." + headers: + Content-Type: + description: "The content type of the file that was previously uploaded." + type: "string" + Content-Disposition: + description: "The name of the file that was previously uploaded, if set." + type: "string" schema: type: file "/thumbnail/{serverName}/{mediaId}": @@ -63,30 +83,44 @@ paths: type: string name: serverName required: true + x-example: matrix.org description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) - in: query type: integer + x-example: 64 name: width - description: The desired width of the thumbnail. + description: |- + The *desired* width of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: integer + x-example: 64 name: height - description: The desired height of the thumbnail. + description: |- + The *desired* height of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: string enum: ["crop", "scale"] name: method + x-example: "scale" description: The desired resizing method. responses: 200: description: "A thumbnail of the requested content." + headers: + Content-Type: + description: "The content type of the thumbnail." + type: "string" + enum: ["image/jpeg", "image/png"] schema: type: file diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema new file mode 120000 index 000000000..045aecb02 --- /dev/null +++ b/api/client-server/v1/core-event-schema @@ -0,0 +1 @@ +v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/v1/create_room.yaml new file mode 100644 index 000000000..051c4b843 --- /dev/null +++ b/api/client-server/v1/create_room.yaml @@ -0,0 +1,148 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/createRoom": + post: + summary: Create a new room + description: |- + Create a new room with various configuration options. + security: + - accessToken: [] + parameters: + - in: body + name: body + description: The desired room configuration. + schema: + type: object + example: |- + { + "preset": "public_chat", + "room_alias_name": "thepub", + "name": "The Grand Duke Pub", + "topic": "All about happy hour", + "creation_content": { + "m.federate": false + } + } + properties: + visibility: + type: string + enum: ["public", "private"] + description: |- + A ``public`` visibility indicates that the room will be shown + in the published room list. A ``private`` visibility will hide + the room from the published room list. Rooms default to + ``private`` visibility if this key is not included. NB: This + should not be confused with ``join_rules`` which also uses the + word ``public``. + room_alias_name: + type: string + description: |- + The desired room alias **local part**. If this is included, a + room alias will be created and mapped to the newly created + room. The alias will belong on the *same* home server which + created the room. For example, if this was set to "foo" and + sent to the homeserver "example.com" the complete room alias + would be ``#foo:example.com``. + name: + type: string + description: |- + If this is included, an ``m.room.name`` event will be sent + into the room to indicate the name of the room. See Room + Events for more information on ``m.room.name``. + topic: + type: string + description: |- + If this is included, an ``m.room.topic`` event will be sent + into the room to indicate the topic for the room. See Room + Events for more information on ``m.room.topic``. + invite: + type: array + description: |- + A list of user IDs to invite to the room. This will tell the + server to invite everyone in the list to the newly created room. + items: + type: string + creation_content: + title: CreationContent + type: object + description: |- + Extra keys to be added to the content of the ``m.room.create``. + The server will clober the following keys: ``creator``. Future + versions of the specification may allow the server to clobber + other keys. + initial_state: + type: array + description: |- + A list of state events to set in the new room. This allows + the user to override the default state events set in the new + room. The expected format of the state events are an object + with type, state_key and content keys set. + Takes precedence over events set by ``presets``, but gets + overriden by ``name`` and ``topic`` keys. + items: + type: object + title: StateEvent + properties: + type: + type: string + state_key: + type: string + content: + type: string + preset: + type: string + enum: ["private_chat", "public_chat", "trusted_private_chat"] + description: |- + Convenience parameter for setting various default state events + based on a preset. Must be either: + + ``private_chat`` => + ``join_rules`` is set to ``invite``. + ``history_visibility`` is set to ``shared``. + + ``trusted_private_chat`` => + ``join_rules`` is set to ``invite``. + ``history_visibility`` is set to ``shared``. + All invitees are given the same power level as the room creator. + + ``public_chat``: => + ``join_rules`` is set to ``public``. + ``history_visibility`` is set to ``shared``. + + responses: + 200: + description: Information about the newly created room. + schema: + type: object + description: Information about the newly created room. + properties: + room_id: + type: string + description: |- + The created room's ID. + examples: + application/json: |- + { + "room_id": "!sefiuhWgwghwWgh:example.com" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. diff --git a/api/client-server/v1/definitions/event.yaml b/api/client-server/v1/definitions/event.yaml deleted file mode 100644 index cfb1b924c..000000000 --- a/api/client-server/v1/definitions/event.yaml +++ /dev/null @@ -1,7 +0,0 @@ -type: object -description: A Matrix Event -properties: - event_id: - type: string - description: An event ID. -required: ["event_id"] \ No newline at end of file diff --git a/api/client-server/v1/definitions/push_condition.json b/api/client-server/v1/definitions/push_condition.json new file mode 100644 index 000000000..1d84955ce --- /dev/null +++ b/api/client-server/v1/definitions/push_condition.json @@ -0,0 +1,9 @@ +{ + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": ["event_match", "profile_tag", "contains_display_name", "room_member_count"] + } + } +} \ No newline at end of file diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/v1/definitions/push_rule.json new file mode 100644 index 000000000..4df93f67b --- /dev/null +++ b/api/client-server/v1/definitions/push_rule.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties": { + "default": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "rule_id": { + "type": "string" + }, + "actions": { + "items": { + "type": ["object", "string"] + }, + "type": "array" + } + } +} \ No newline at end of file diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/v1/definitions/push_ruleset.json new file mode 100644 index 000000000..e03727012 --- /dev/null +++ b/api/client-server/v1/definitions/push_ruleset.json @@ -0,0 +1,60 @@ +{ + "type": "object", + "properties": { + "content": { + "items": { + "type": "object", + "allOf": [ + { + "$ref": "push_rule.json" + } + ] + }, + "type": "array" + }, + "override": { + "items": { + "type": "object", + "allOf": [ + { + "$ref": "push_rule.json" + } + ] + }, + "type": "array" + }, + "sender": { + "items": { + "type": "object", + "allOf": [ + { + "$ref": "push_rule.json" + } + ] + }, + "type": "array" + }, + "underride": { + "items": { + "type": "object", + "allOf": [ + { + "$ref": "push_rule.json" + } + ] + }, + "type": "array" + }, + "room": { + "items": { + "type": "object", + "allOf": [ + { + "$ref": "push_rule.json" + } + ] + }, + "type": "array" + } + } +} \ No newline at end of file diff --git a/api/client-server/v1/definitions/room_event.yaml b/api/client-server/v1/definitions/room_event.yaml deleted file mode 100644 index 5a9e3a09c..000000000 --- a/api/client-server/v1/definitions/room_event.yaml +++ /dev/null @@ -1,9 +0,0 @@ -type: object -description: A Matrix Room Event -properties: - event_id: - type: string - description: An event ID. - room_id: - type: string -required: ["event_id", "room_id"] \ No newline at end of file diff --git a/api/client-server/v1/definitions/state_event.yaml b/api/client-server/v1/definitions/state_event.yaml deleted file mode 100644 index f8e6f4d4e..000000000 --- a/api/client-server/v1/definitions/state_event.yaml +++ /dev/null @@ -1,11 +0,0 @@ -type: object -description: A Matrix State Event -properties: - event_id: - type: string - description: An event ID. - room_id: - type: string - state_key: - type: string -required: ["event_id", "room_id", "state_key"] \ No newline at end of file diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml new file mode 100644 index 000000000..bbab46dfa --- /dev/null +++ b/api/client-server/v1/login.yaml @@ -0,0 +1,160 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Registration and Login API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/login": + post: + summary: Authenticates the user. + description: |- + Authenticates the user by password, and issues an access token they can + use to authorize themself in subsequent requests. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "type": "m.login.pasword", + "user": "cheeky_monkey", + "password": "ilovebananas" + } + properties: + type: + type: string + description: The login type being used. Currently only "m.login.password" is supported. + user: + type: string + description: The fully qualified user ID or just local part of the user ID, to log in. + password: + type: string + description: The user's password. + required: ["type", "user", "password"] + responses: + 200: + description: The user has been authenticated. + examples: + application/json: |- + { + "user_id": "@cheeky_monkey:matrix.org", + "access_token": "abc123", + "home_server": "matrix.org" + } + schema: + type: object + properties: + user_id: + type: string + description: The fully-qualified Matrix ID that has been registered. + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + There is no specific error message to indicate that a request has failed because + an access token has expired; instead, if a client has reason to believe its + access token is valid, and it receives an auth error, they should attempt to + refresh for a new token on failure, and retry the request with the new token. + refresh_token: + type: string + # TODO: Work out how to linkify /tokenrefresh + description: |- + (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. + home_server: + type: string + description: The hostname of the Home Server on which the account has been registered. + 400: + description: |- + Part of the request was invalid. For example, the login type may not be recognised. + examples: + application/json: |- + { + "errcode": "M_UNKNOWN", + "error": "Bad login type." + } + 403: + description: |- + The login attempt failed. For example, the password may have been incorrect. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/tokenrefresh": + post: + summary: Exchanges a refresh token for an access token. + description: |- + Exchanges a refresh token for a new access token. + This is intended to be used if the access token has expired. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "refresh_token": "a1b2c3" + } + properties: + refresh_token: + type: string + description: The refresh token which was issued by the server. + required: ["refresh_token"] + responses: + 200: + description: |- + The refresh token was accepted, and a new access token has been issued. + The passed refresh token is no longer valid and cannot be used. + A new refresh token will have been returned unless some policy does + not allow the user to continue to renew their session. + examples: + application/json: |- + { + "access_token": "bearwithme123", + "refresh_token": "exchangewithme987" + } + schema: + type: object + properties: + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + refresh_token: + type: string + description: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the TODO Linkify /tokenrefresh API endpoint. + 403: + description: |- + The exchange attempt failed. For example, the refresh token may have already been used. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml new file mode 100644 index 000000000..f8dfdea55 --- /dev/null +++ b/api/client-server/v1/membership.yaml @@ -0,0 +1,237 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} + + # With an extra " " to disambiguate from the 3pid invite endpoint + # The extra space makes it sort first for what I'm sure is a good reason. + "/rooms/{roomId}/invite ": + post: + summary: Invite a user to participate in a particular room. + description: |- + *Note that there are two forms of this API, which are documented separately. + This version of the API requires that the inviter knows the Matrix + identifier of the invitee.* + + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + Only users currently in a particular room can invite other users to + join that room. + + If the user was invited to the room, the home server will append a + ``m.room.member`` event to the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the invitee. + required: ["user_id"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + description: |- + *Note that there are two forms of this API, which are documented separately. + This version of the API does not require that the inviter know the Matrix + identifier of the invitee, and instead relies on third party identifiers. + The homeserver uses an identity server to perform the mapping from + third party identifier to a Matrix identifier.* + + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + Only users currently in a particular room can invite other users to + join that room. + + If the identity server did know the Matrix user identifier for the + third party identifier, the home server will append a ``m.room.member`` + event to the room. + + If the identity server does not know a Matrix user identifier for the + passed third party identifier, the homeserver will issue an invitation + which can be accepted upon providing proof of ownership of the third + party identifier. This is achieved by the identity server generating a + token, which it gives to the inviting homeserver. The homeserver will + add an ``m.room.third_party_invite`` event into the graph for the room, + containing that token. + + When the invitee binds the invited third party identifier to a Matrix + user ID, the identity server will give the user a list of pending + invitations, each containing: + + - The room ID to which they were invited + + - The token given to the homeserver + + - A signature of the token, signed with the identity server's private key + + - The matrix user ID who invited them to the room + + If a token is requested from the identity server, the home server will + append a ``m.room.third_party_invite`` event to the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "id_server": "matrix.org", + "medium": "email", + "address": "cheeky@monkey.com", + "display_name": "A very cheeky monkey" + } + properties: + id_server: + type: string + description: The hostname+port of the identity server which should be used for third party identifier lookups. + medium: + type: string + # TODO: Link to identity service spec when it eixsts + description: The kind of address being passed in the address field, for example ``email``. + address: + type: string + description: The invitee's third party identifier. + display_name: + type: string + description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. + required: ["id_server", "medium", "address", "display_name"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/v1/message_pagination.yaml new file mode 100644 index 000000000..d2bc0554d --- /dev/null +++ b/api/client-server/v1/message_pagination.yaml @@ -0,0 +1,131 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Rooms API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/messages": + get: + summary: Get a list of events for this room + description: |- + This API returns a list of message and state events for a room. It uses + pagination query parameters to paginate history in the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to get events from. + required: true + x-example: "!636q39766251:example.com" + - in: query + type: string + name: from + description: |- + The token to start returning events from. This token can be obtained + from the initial sync API. + required: true + x-example: "s345_678_333" + - in: query + type: string + enum: ["b", "f"] + name: dir + description: |- + The direction to return events from. + required: true + x-example: "b" + - in: query + type: integer + name: limit + description: |- + The maximum number of events to return. Default: 10. + x-example: "3" + responses: + 200: + description: A list of messages with a new token to request more. + schema: + type: object + description: A list of messages with a new token to request more. + properties: + start: + type: string + description: |- + The token to start paginating from. If ``dir=b`` this will be + the token supplied in ``from``. + end: + type: string + description: |- + The token the pagination ends at. If ``dir=b`` this token should + be used again to request even earlier events. + chunk: + type: array + description: |- + A list of room events. + items: + type: object + title: RoomEvent + examples: + application/json: |- + { + "start": "t47429-4392820_219380_26003_2265", + "end": "t47409-4357353_219380_26003_2265", + "chunk": [ + { + "origin_server_ts": 1444812213737, + "user_id": "@alice:example.com", + "event_id": "$1444812213350496Caaaa:example.com", + "content": { + "body": "hello world", + "msgtype":"m.text" + }, + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.message", + "age": 1042 + }, + { + "origin_server_ts": 1444812194656 , + "user_id": "@bob:example.com", + "event_id": "$1444812213350496Cbbbb:example.com", + "content": { + "body": "the world is big", + "msgtype":"m.text" + }, + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.message", + "age": 20123 + }, + { + "origin_server_ts": 1444812163990, + "user_id": "@bob:example.com", + "event_id": "$1444812213350496Ccccc:example.com", + "content": { + "name": "New room name" + }, + "prev_content": { + "name": "Old room name" + }, + "state_key": "", + "room_id":"!Xq3620DUiqCaoxq:example.com", + "type":"m.room.name", + "age": 50789 + } + ] + } + 403: + description: > + You aren't a member of the room. diff --git a/api/client-server/v1/presence.yaml b/api/client-server/v1/presence.yaml index 166653414..5684398b0 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/v1/presence.yaml @@ -101,7 +101,7 @@ paths: The length of time in milliseconds since an action was performed by this user. status_msg: - type: string + type: [string, "null"] description: The state message for this user if one was set. 404: description: |- @@ -185,7 +185,7 @@ paths: "last_active_ago": 395, "presence": "offline", "user_id": "@alice:matrix.org" - } + }, "type": "m.presence" }, { @@ -195,7 +195,7 @@ paths: "last_active_ago": 16874, "presence": "online", "user_id": "@marisa:matrix.org" - } + }, "type": "m.presence" } ] @@ -205,5 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "definitions/event.yaml" - \ No newline at end of file + - "$ref": "core-event-schema/event.json" diff --git a/api/client-server/v1/push_notifier.yaml b/api/client-server/v1/push_notifier.yaml new file mode 100644 index 000000000..be3b5f74d --- /dev/null +++ b/api/client-server/v1/push_notifier.yaml @@ -0,0 +1,192 @@ +swagger: '2.0' +info: + title: "Matrix Push Notification API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/push/v1 +consumes: + - application/json +produces: + - application/json +paths: + "/notify": + post: + summary: Notify a push gateway about an event. + description: |- + This endpoint is invoked by HTTP pushers to notify a push gateway about + an event. + *NB: Notifications are sent to the URL configured when the pusher is + created. This means that the HTTP path may be different depending on the + push gateway.* + parameters: + - in: body + name: notification + description: Information about the push notification. + required: true + schema: + type: object + example: |- + { + "notification": { + "id": "$3957tyerfgewrf384", + "room_id": "!slw48wfj34rtnrf:example.com", + "type": "m.room.message", + "sender": "@exampleuser:matrix.org", + "sender_display_name": "Major Tom", + "room_name": "Mission Control", + "room_alias": "#exampleroom:matrix.org", + "prio": "high", + "content": { + "msgtype": "m.text", + "body": "I'm floating in a most peculiar way." + } + }, + "counts": { + "unread" : 2, + "missed_calls": 1 + }, + "devices": [ + { + "app_id": "org.matrix.matrixConsole.ios", + "pushkey": "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/", + "pushkey_ts": 12345678, + "data" : {}, + "tweaks": { + "sound": "bing" + } + } + ] + } + required: ["notification", "counts", "devices"] + properties: + notification: + type: object + description: Information about the push notification + required: ["id", "room_id", "type", "sender"] + properties: + id: + type: string + description: |- + An identifier for this notification that may be used to + detect duplicate notification requests. This is not + necessarily the ID of the event that triggered the + notification. + room_id: + type: string + description: The ID of the room in which this event occurred. + type: + type: string + description: The type of the event as in the event's ``type`` field. + sender: + type: string + description: The sender of the event as in the corresponding event field. + sender_display_name: + type: string + description: |- + The current display name of the sender in the room in which + the event occurred. + room_name: + type: string + description: The name of the room in which the event occurred. + room_alias: + type: string + description: An alias to display for the room in which the event occurred. + user_is_target: + type: boolean + description: |- + This is true if the user receiving the notification is the + subject of a member event (i.e. the ``state_key`` of the + member event is equal to the user's Matrix ID). + prio: + type: string + enum: ["high", "low"] + description: |- + The priority of the notification. If omitted, ``high`` is + assumed. This may be used by push gateways to deliver less + time-sensitive notifications in a way that will preserve + battery power on mobile devices. + content: + type: object + title: EventContent + description: |- + The ``content`` field from the event, if present. If the + event had no content field, this field is omitted. + counts: + type: object + description: |- + This is a dictionary of the current number of unacknowledged + communications for the recipient user. Counts whose value is + zero are omitted. + properties: + unread: + type: integer + description: |- + The number of unread messages a user has across all of the + rooms they are a member of. + missed_calls: + type: integer + description: |- + The number of unacknowledged missed calls a user has + across all rooms of which they are a member. + devices: + type: array + title: Devices + description: |- + This is an array of devices that the notification should be sent to. + items: + type: object + properties: + app_id: + type: string + description: |- + The app_id given when the pusher was created. + pushkey: + type: string + description: The pushkey given when the pusher was created. + pushkey_ts: + type: integer + description: |- + The unix timestamp (in seconds) when the + pushkey was last updated. + data: + type: object + title: PusherData + description: |- + A dictionary of additional pusher-specific data. For + 'http' pushers, this is the data dictionary passed in at + pusher creation minus the ``url`` key. + tweaks: + type: object + title: Tweaks + description: |- + A dictionary of customisations made to the way this + notification is to be presented. These are added by push rules. + responses: + 200: + description: A list of rejected push keys. + examples: + application/json: |- + { + "rejected": [ "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/" ] + } + schema: + type: object # empty json object + properties: + rejected: + type: array + description: |- + A list of all pushkeys given in the notification request that + are not valid. These could have been rejected by an upstream + gateway because they have expired or have never been valid. + Homeservers must cease sending notification requests for these + pushkeys and remove the associated pushers. It may not + necessarily be the notification in the request that failed: + it could be that a previous notification to the same pushkey + failed. + items: + type: string + description: A pushkey + diff --git a/api/client-server/v1/pusher.yaml b/api/client-server/v1/pusher.yaml new file mode 100644 index 000000000..8c243f2bc --- /dev/null +++ b/api/client-server/v1/pusher.yaml @@ -0,0 +1,144 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Push API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/pushers/set": + post: + summary: Modify a pusher for this user on the homeserver. + description: |- + This endpoint allows the creation, modification and deletion of `pushers`_ + for this user ID. The behaviour of this endpoint varies depending on the + values in the JSON body. + security: + - accessToken: [] + parameters: + - in: body + name: pusher + description: The pusher information + required: true + schema: + type: object + example: |- + { + "lang": "en", + "kind": "http", + "app_display_name": "Mat Rix", + "device_display_name": "iPhone 9", + "app_id": "com.example.app.ios", + "profile_tag": "4bea66906d0111e59d70feff819cdc9f", + "pushkey": "APA91bHPRgkF3JUikC4ENAHEeMrd41Zxv3hVZjC9KtT8OvPVGJ-hQMRKRrZuJAEcl7B338qju59zJMjw2DELjzEvxwYv7hH5Ynpc1ODQ0aT4U4OFEeco8ohsN5PjL1iC2dNtk2BAokeMCg2ZXKqpc8FXKmhX94kIxQ", + "data": { + "url": "https://push-gateway.location.here" + }, + "append": false + } + properties: + pushkey: + type: string + description: |- + This is a unique identifier for this pusher. The value you + should use for this is the routing or destination address + information for the notification, for example, the APNS token + for APNS or the Registration ID for GCM. If your notification + client has no such concept, use any unique identifier. + Max length, 512 bytes. + kind: + type: string + enum: ["http", null] + description: |- + The kind of pusher to configure. ``"http"`` makes a pusher that + sends HTTP pokes. ``null`` deletes the pusher. + profile_tag: + type: string + description: |- + This is a string that determines what set of device rules will + be matched when evaluating push rules for this pusher. It is + an arbitrary string. Multiple devices may use the same + ``profile_tag``. It is advised that when an app's data is + copied or restored to a different device, this value remain + the same. Client apps should offer ways to change the + ``profile_tag``, optionally copying rules from the old + profile tag. Max length, 32 bytes. + app_id: + type: string + description: |- + This is a reverse-DNS style identifier for the application. + It is recommended that this end with the platform, such that + different platform versions get different app identifiers. + Max length, 64 chars. + app_display_name: + type: string + description: |- + A string that will allow the user to identify what application + owns this pusher. + device_display_name: + type: string + description: |- + A string that will allow the user to identify what device owns + this pusher. + lang: + type: string + description: |- + The preferred language for receiving notifications (e.g. 'en' + or 'en-US') + data: + type: object + description: |- + A dictionary of information for the pusher implementation + itself. If ``kind`` is ``http``, this should contain ``url`` + which is the URL to use to send notifications to. + properties: + url: + type: string + description: |- + Required if ``kind`` is ``http``. The URL to use to send + notifications to. + append: + type: boolean + description: |- + If true, the homeserver should add another pusher with the + given pushkey and App ID in addition to any others with + different user IDs. Otherwise, the Home Server must remove any + other pushers with the same App ID and pushkey for different + users. The default is ``false``. + required: ['profile_tag', 'kind', 'app_id', 'app_display_name', + 'device_display_name', 'pushkey', 'lang', 'data'] + responses: + 200: + description: The pusher was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 400: + description: One or more of the pusher values were invalid. + examples: + application/json: |- + { + "error": "Missing parameters: lang, data", + "errcode": "M_MISSING_PARAM" + } + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/api/client-server/v1/pushrules.yaml b/api/client-server/v1/pushrules.yaml new file mode 100644 index 000000000..31e84f554 --- /dev/null +++ b/api/client-server/v1/pushrules.yaml @@ -0,0 +1,488 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Push Rules API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/pushrules/": + get: + summary: Retrieve all push rulesets. + description: |- + Retrieve all push rulesets for this user. Clients can "drill-down" on + the rulesets by suffixing a ``scope`` to this path e.g. + ``/pushrules/global/``. This will return a subset of this data under the + specified key e.g. the ``global`` key. + security: + - accessToken: [] + responses: + 200: + description: All the push rulesets for this user. + schema: + type: object + required: ["device", "global"] + properties: + device: + type: object + title: Devices + description: A dictionary of profile tags to rulesets. + additionalProperties: + x-pattern: "$PROFILE_TAG" + type: object + description: The ruleset for this profile tag. + title: Ruleset + allOf: [ + "$ref": "definitions/push_ruleset.json" + ] + global: + type: object + description: The global ruleset. + title: Ruleset + allOf: [ + "$ref": "definitions/push_ruleset.json" + ] + examples: + application/json: |- + { + "device": {}, + "global": { + "content": [ + { + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight" + } + ], + "default": true, + "enabled": true, + "pattern": "alice", + "rule_id": ".m.rule.contains_user_name" + } + ], + "override": [ + { + "actions": [ + "dont_notify" + ], + "conditions": [], + "default": true, + "enabled": false, + "rule_id": ".m.rule.master" + }, + { + "actions": [ + "dont_notify" + ], + "conditions": [ + { + "key": "content.msgtype", + "kind": "event_match", + "pattern": "m.notice" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.suppress_notices" + } + ], + "room": [], + "sender": [], + "underride": [ + { + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "ring" + }, + { + "set_tweak": "highlight", + "value": false + } + ], + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.call.invite" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.call" + }, + { + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight" + } + ], + "conditions": [ + { + "kind": "contains_display_name" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.contains_display_name" + }, + { + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight", + "value": false + } + ], + "conditions": [ + { + "is": "2", + "kind": "room_member_count" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.room_one_to_one" + }, + { + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight", + "value": false + } + ], + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.room.member" + }, + { + "key": "content.membership", + "kind": "event_match", + "pattern": "invite" + }, + { + "key": "state_key", + "kind": "event_match", + "pattern": "@alice:example.com" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.invite_for_me" + }, + { + "actions": [ + "notify", + { + "set_tweak": "highlight", + "value": false + } + ], + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.room.member" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.member_event" + }, + { + "actions": [ + "notify", + { + "set_tweak": "highlight", + "value": false + } + ], + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.room.message" + } + ], + "default": true, + "enabled": true, + "rule_id": ".m.rule.message" + } + ] + } + } + "/pushrules/{scope}/{kind}/{ruleId}": + get: + summary: Retrieve a push rule. + description: |- + Retrieve a single specified push rule. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: scope + required: true + x-example: "global" + description: |- + Either ``global`` or ``device/`` to specify global + rules or device rules for the given ``profile_tag``. + - in: path + type: string + name: kind + required: true + x-example: room + enum: ["override", "underride", "sender", "room", "content"] + description: | + The kind of rule + - in: path + type: string + name: ruleId + required: true + x-example: "#spam:example.com" + description: | + The identifier for the rule. + responses: + 200: + description: |- + The specific push rule. This will also include keys specific to the + rule itself such as the rule's ``actions`` and ``conditions`` if set. + examples: + application/json: |- + { + "actions": [ + "dont_notify" + ], + "rule_id": "#spam:matrix.org", + "enabled": true + } + schema: + type: object + description: The push rule. + title: PushRule + allOf: [ + "$ref": "definitions/push_rule.json" + ] + delete: + summary: Delete a push rule. + description: |- + This endpoint removes the push rule defined in the path. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: scope + required: true + x-example: "global" + description: |- + Either ``global`` or ``device/`` to specify global + rules or device rules for the given ``profile_tag``. + - in: path + type: string + name: kind + required: true + x-example: room + enum: ["override", "underride", "sender", "room", "content"] + description: | + The kind of rule + - in: path + type: string + name: ruleId + required: true + x-example: "#spam:example.com" + description: | + The identifier for the rule. + responses: + 200: + description: The push rule was deleted. + examples: + application/json: |- + {} + schema: + type: object # empty json object + put: + summary: Add or change a push rule. + description: |- + This endpoint allows the creation, modification and deletion of pushers + for this user ID. The behaviour of this endpoint varies depending on the + values in the JSON body. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: scope + required: true + x-example: "global" + description: |- + Either ``global`` or ``device/`` to specify global + rules or device rules for the given ``profile_tag``. + - in: path + type: string + name: kind + required: true + x-example: room + enum: ["override", "underride", "sender", "room", "content"] + description: | + The kind of rule + - in: path + type: string + name: ruleId + required: true + x-example: "#spam:example.com" + description: | + The identifier for the rule. + - in: query + type: string + name: before + required: false + x-example: someRuleId + description: |- + Use 'before' with a ``rule_id`` as its value to make the new rule the + next-most important rule with respect to the given rule. + - in: query + type: string + name: after + required: false + x-example: anotherRuleId + description: |- + This makes the new rule the next-less important rule relative to the + given rule. + - in: body + name: pushrule + description: |- + The push rule data. Additional top-level keys may be present depending + on the parameters for the rule ``kind``. + required: true + schema: + type: object + example: |- + { + "pattern": "cake*lie", + "actions": ["notify"] + } + properties: + actions: + type: array + description: |- + The action(s) to perform when the conditions for this rule are met. + items: + type: string + enum: ["notify", "dont_notify", "coalesce", "set_tweak"] + # TODO: type: object e.g. {"set_sound":"beeroclock.wav"} :/ + conditions: + type: array + description: |- + The conditions that must hold true for an event in order for a + rule to be applied to an event. A rule with no conditions + always matches. + items: + type: object + title: conditions + allOf: [ "$ref": "definitions/push_condition.json" ] + required: ["actions"] + responses: + 200: + description: The pusher was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 400: + description: There was a problem configuring this push rule. + examples: + application/json: |- + { + "error": "before/after rule not found: someRuleId", + "errcode": "M_UNKNOWN" + } + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/pushrules/{scope}/{kind}/{ruleId}/enabled": + put: + summary: "Enable or disable a push rule." + description: |- + This endpoint allows clients to enable or disable the specified push rule. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: scope + required: true + x-example: "global" + description: |- + Either ``global`` or ``device/`` to specify global + rules or device rules for the given ``profile_tag``. + - in: path + type: string + name: kind + required: true + x-example: room + enum: ["override", "underride", "sender", "room", "content"] + description: | + The kind of rule + - in: path + type: string + name: ruleId + required: true + x-example: "#spam:example.com" + description: | + The identifier for the rule. + - in: body + name: + description: | + Whether the push rule is enabled or not. + required: true + schema: + type: boolean + example: |- + true + responses: + 200: + description: The push rule was enabled or disabled. + examples: + application/json: |- + {} + schema: + type: object # empty json object diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml new file mode 100644 index 000000000..9c9273dfe --- /dev/null +++ b/api/client-server/v1/room_send.yaml @@ -0,0 +1,125 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 message event send API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/send/{eventType}/{txnId}": + put: + summary: Send a message event to the given room. + description: |- + This endpoint is used to send a message event to a room. Message events + allow access to historical events and pagination, making them suited + for "once-off" activity in a room. + + The body of the request should be the content object of the event; the + fields in this object will vary depending on the type of event. See + `Room Events`_ for the m. event specification. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to send the event to. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.message" + - in: path + name: txnId + type: string + description: |- + The transaction ID for this event. Clients should generate a + unique ID; it will be used by the server to ensure idempotency of requests. + required: true + x-example: "35" + - in: body + name: body + schema: + type: object + example: |- + { + "msgtype": "m.text", + "body": "hello" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. + "/rooms/{roomId}/send/{eventType}": + post: + summary: Send a message event to the given room. + description: |- + This endpoint can be used to send a message event to a room; however + the lack of a transaction ID means that it is possible to cause message + duplication if events are resent on error, so it is preferable to use + `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to send the event to. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.message" + - in: body + name: body + schema: + type: object + example: |- + { + "msgtype": "m.text", + "body": "hello" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/v1/room_state.yaml new file mode 100644 index 000000000..ec30fd347 --- /dev/null +++ b/api/client-server/v1/room_state.yaml @@ -0,0 +1,80 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 state event send API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/state/{eventType}/{stateKey}": + put: + summary: Send a message event to the given room. + description: | + State events can be sent using this endpoint. These events will be + overwritten if ````, ```` and ```` all + match. If the state event has an empty ``state_key``, it can be + omitted from the path. + + Requests to this endpoint **cannot use transaction IDs** + like other ``PUT`` paths because they cannot be differentiated from the + ``state_key``. Furthermore, ``POST`` is unsupported on state paths. + + The body of the request should be the content object of the event; the + fields in this object will vary depending on the type of event. See + `Room Events`_ for the ``m.`` event specification. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to set the state in + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of event to send. + required: true + x-example: "m.room.name" + - in: path + type: string + name: stateKey + description: The state_key for the state to send. Defaults to the empty string. + required: true + x-example: "" + - in: body + name: body + schema: + type: object + example: |- + { + "name": "New name for the room" + } + responses: + 200: + description: "An ID for the sent event." + examples: + application/json: |- + { + "event_id": "YUwRidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml new file mode 100644 index 000000000..9300d7d10 --- /dev/null +++ b/api/client-server/v1/rooms.yaml @@ -0,0 +1,448 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Rooms API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/state/{eventType}/{stateKey}": + get: + summary: Get the state identified by the type and key. + description: |- + Looks up the contents of a state event in a room. If the user is + joined to the room then the state is taken from the current + state of the room. If the user has left the room then the state is + taken from the state of the room when they left. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to look up the state in. + required: true + x-example: "!636q39766251:example.com" + - in: path + type: string + name: eventType + description: The type of state to look up. + required: true + x-example: "m.room.name" + - in: path + type: string + name: stateKey + description: The key of the state to look up. Defaults to the empty string. + required: true + x-example: "" + responses: + 200: + description: The content of the state event. + examples: + application/json: |- + {"name": "Example room name"} + schema: + type: object + 404: + description: The room has no state with the given type or key. + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/state": + get: + summary: Get all state events in the current state of a room. + description: |- + Get the state events for the current state of a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to look up the state for. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: The current state of the room + examples: + application/json: |- + [ + { + "age": 7148266897, + "content": { + "join_rule": "public" + }, + "event_id": "$14259997323TLwtb:example.com", + "origin_server_ts": 1425999732392, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.join_rules", + "user_id": "@alice:example.com" + }, + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 7148267200, + "content": { + "creator": "@alice:example.com" + }, + "event_id": "$14259997320KhbwJ:example.com", + "origin_server_ts": 1425999732089, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.create", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + }, + { + "age": 7148267004, + "content": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "kick": 50, + "redact": 50, + "state_default": 50, + "users": { + "@alice:example.com": 100 + }, + "users_default": 0 + }, + "event_id": "$14259997322mqfaq:example.com", + "origin_server_ts": 1425999732285, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.power_levels", + "user_id": "@alice:example.com" + } + ] + schema: + type: array + title: RoomState + description: |- + If the user is a member of the room this will be the + current state of the room as a list of events. If the user + has left the room then this will be the state of the room + when they left as a list of events. + items: + title: StateEvent + type: object + allOf: + - "$ref": "core-event-schema/state_event.json" + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/initialSync": + get: + summary: Snapshot the current state of a room and its most recent messages. + description: |- + Get a copy of the current state and the most recent messages in a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room to get the data. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: The current state of the room + examples: + application/json: |- + { + "membership": "join", + "messages": { + "chunk": [ + { + "age": 343513403, + "content": { + "body": "foo", + "msgtype": "m.text" + }, + "event_id": "$14328044851tzTJS:example.com", + "origin_server_ts": 1432804485886, + "room_id": "!636q39766251:example.com", + "type": "m.room.message", + "user_id": "@alice:example.com" + }, + { + "age": 343511809, + "content": { + "body": "bar", + "msgtype": "m.text" + }, + "event_id": "$14328044872spjFg:example.com", + "origin_server_ts": 1432804487480, + "room_id": "!636q39766251:example.com", + "type": "m.room.message", + "user_id": "@bob:example.com" + } + ], + "end": "s3456_9_0", + "start": "t44-3453_9_0" + }, + "room_id": "!636q39766251:example.com", + "state": [ + { + "age": 7148266897, + "content": { + "join_rule": "public" + }, + "event_id": "$14259997323TLwtb:example.com", + "origin_server_ts": 1425999732392, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.join_rules", + "user_id": "@alice:example.com" + }, + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 7148267200, + "content": { + "creator": "@alice:example.com" + }, + "event_id": "$14259997320KhbwJ:example.com", + "origin_server_ts": 1425999732089, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.create", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + }, + { + "age": 7148267004, + "content": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "kick": 50, + "redact": 50, + "state_default": 50, + "users": { + "@alice:example.com": 100 + }, + "users_default": 0 + }, + "event_id": "$14259997322mqfaq:example.com", + "origin_server_ts": 1425999732285, + "room_id": "!636q39766251:example.com", + "state_key": "", + "type": "m.room.power_levels", + "user_id": "@alice:example.com" + } + ], + "visibility": "private" + } + schema: + title: RoomInfo + type: object + properties: + room_id: + type: string + description: "The ID of this room." + membership: + type: string + description: "The user's membership state in this room." + enum: ["invite", "join", "leave", "ban"] + messages: + type: object + title: PaginationChunk + description: "The pagination chunk for this room." + properties: + start: + type: string + description: |- + A token which correlates to the first value in ``chunk``. + Used for pagination. + end: + type: string + description: |- + A token which correlates to the last value in ``chunk``. + Used for pagination. + chunk: + type: array + description: |- + If the user is a member of the room this will be a + list of the most recent messages for this room. If + the user has left the room this will be the + messages that preceeded them leaving. This array + will consist of at most ``limit`` elements. + items: + type: object + title: RoomEvent + allOf: + - "$ref": "core-event-schema/room_event.json" + required: ["start", "end", "chunk"] + state: + type: array + description: |- + If the user is a member of the room this will be the + current state of the room as a list of events. If the + user has left the room this will be the state of the + room when they left it. + items: + title: StateEvent + type: object + allOf: + - "$ref": "core-event-schema/state_event.json" + visibility: + type: string + enum: ["private", "public"] + description: |- + Whether this room is visible to the ``/publicRooms`` API + or not." + required: ["room_id", "membership"] + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + + "/rooms/{roomId}/members": + get: + summary: Get the m.room.member events for the room. + description: + Get the list of members for this room. + parameters: + - in: path + type: string + name: roomId + description: The room to get the member events for. + required: true + x-example: "!636q39766251:example.com" + responses: + 200: + description: |- + A list of members of the room. If you are joined to the room then + this will be the current members of the room. If you have left te + room then this will be the members of the room when you left. + examples: + application/json: |- + { + "chunk": [ + { + "age": 6547561012, + "content": { + "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", + "displayname": null, + "membership": "join" + }, + "event_id": "$1426600438280zExKY:example.com", + "membership": "join", + "origin_server_ts": 1426600438277, + "room_id": "!636q39766251:example.com", + "state_key": "@alice:example.com", + "type": "m.room.member", + "user_id": "@alice:example.com" + }, + { + "age": 1622568720, + "content": { + "avatar_url": "mxc://example.com/GCmhgzMPRjqgpODLsNQzVuHZ#auto", + "displayname": "Bob", + "membership": "join" + }, + "event_id": "$1431525430134MxlLX:example.com", + "origin_server_ts": 1431525430569, + "replaces_state": "$142652023736BSXcM:example.com", + "room_id": "!636q39766251:example.com", + "state_key": "@bob:example.com", + "type": "m.room.member", + "user_id": "@bob:example.com" + } + ] + } + schema: + type: object + properties: + chunk: + type: array + items: + title: MemberEvent + type: object + allOf: + - "$ref": "v1-event-schema/m.room.member" + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. + diff --git a/api/client-server/v1/search.yaml b/api/client-server/v1/search.yaml new file mode 100644 index 000000000..cc2990564 --- /dev/null +++ b/api/client-server/v1/search.yaml @@ -0,0 +1,146 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Search API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/search": + post: + summary: Search server side for things. + description: |- + Performs a full text search across different categories. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "search_categories": { + "room_events": { + "keys": [ + "content.body" + ], + "search_term": "martians and men" + } + } + } + properties: + search_categories: + type: object + title: "Categories" + description: Describes which categories to search in and + their criteria. + properties: + room_events: + type: object + title: "Room Events" + description: Mapping of category name to search criteria. + properties: + search_term: + type: string + description: The string to search events for + keys: + type: array + items: + type: string + enum: ["content.body", "content.name", "content.topic"] + description: The keys to search. Defaults to all. + filter: + type: object + title: Filter + description: |- + The filter to apply to search results. + This has the same format as v2 filter API. + required: ["search_term"] + required: ["search_categories"] + responses: + 200: + description: Results of the search. + schema: + type: object + title: Results + required: ["search_categories"] + properties: + search_categories: + type: object + title: Categories + description: Describes which categories to search in and + their criteria. + properties: + room_events: + type: object + title: Room Event Results + description: Mapping of category name to search criteria. + properties: + count: + type: number + description: Total number of results found + results: + type: object + title: Results + description: Mapping of event_id to result. + additionalProperties: + type: object + title: Result + description: The result object. + properties: + rank: + type: number + description: A number that describes how closely + this result matches the search. Higher is + closer. + result: + type: object + title: Event + description: The event that matched. + allOf: + - "$ref": "core-event-schema/room_event.json" + examples: + application/json: |- + { + "search_categories": { + "room_events": { + "count": 24, + "results": { + "$144429830826TWwbB:localhost": { + "rank": 0.00424866, + "result": { + "age": 526228296, + "content": { + "body": "Test content", + "msgtype": "m.text" + }, + "event_id": "$144429830826TWwbB:localhost", + "origin_server_ts": 1444298308034, + "room_id": "!qPewotXpIctQySfjSy:localhost", + "type": "m.room.message", + "user_id": "@test:localhost" + } + } + } + } + } + } + 400: + description: Part of the request was invalid. + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index a08146b47..d07e9399c 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -30,7 +30,9 @@ paths: - in: query type: string name: from - description: The token to stream from. + description: |- + The token to stream from. This token is either from a previous + request to this API or from the initial sync API. required: false x-example: "s3456_9_0" - in: query @@ -68,21 +70,21 @@ paths: start: type: string description: |- - A token which correlates to the first value in ``chunk``. Used - for pagination. + A token which correlates to the first value in ``chunk``. This + is usually the same token supplied to ``from=``. end: type: string description: |- - A token which correlates to the last value in ``chunk``. Used - for pagination. + A token which correlates to the last value in ``chunk``. This + token should be used in the next request to ``/events``. chunk: type: array description: "An array of events." items: type: object - title: RoomEvent + title: Event allOf: - - "$ref": "definitions/room_event.yaml" + - "$ref": "core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -100,6 +102,16 @@ paths: description: The maximum number of messages to return for each room. required: false x-example: "2" + - in: query + type: boolean + name: archived + description: |- + Whether to include rooms that the user has left. If ``false`` then + only rooms that the user has been invited to or has joined are + included. If set to ``true`` then rooms that the user has left are + included as well. By default this is ``false``. + required: false + x-example: "true" responses: 200: description: The user's current state. @@ -253,7 +265,7 @@ paths: type: object title: Event allOf: - - "$ref": "definitions/event.yaml" + - "$ref": "core-event-schema/event.json" rooms: type: array items: @@ -267,6 +279,12 @@ paths: type: string description: "The user's membership state in this room." enum: ["invite", "join", "leave", "ban"] + invite: + type: object + title: "InviteEvent" + description: "The invite event if ``membership`` is ``invite``" + allOf: + - "$ref": "v1-event-schema/m.room.member" messages: type: object title: PaginationChunk @@ -285,24 +303,29 @@ paths: chunk: type: array description: |- - A list of the most recent messages for this room. This - array will consist of at most ``limit`` elements. + If the user is a member of the room this will be a + list of the most recent messages for this room. If + the user has left the room this will be the + messages that preceeded them leaving. This array + will consist of at most ``limit`` elements. items: type: object title: RoomEvent allOf: - - "$ref": "definitions/room_event.yaml" + - "$ref": "core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array description: |- - A list of state events representing the current state - of the room. + If the user is a member of the room this will be the + current state of the room as a list of events. If the + user has left the room this will be the state of the + room when they left it. items: title: StateEvent type: object allOf: - - "$ref": "definitions/state_event.yaml" + - "$ref": "core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -338,13 +361,13 @@ paths: "body": "Hello world!", "msgtype": "m.text" }, - "room_id:" "!wfgy43Sg4a:matrix.org", + "room_id:": "!wfgy43Sg4a:matrix.org", "user_id": "@bob:matrix.org", "event_id": "$asfDuShaf7Gafaw:matrix.org", "type": "m.room.message" } schema: allOf: - - "$ref": "definitions/event.yaml" + - "$ref": "core-event-schema/event.json" 404: - description: The event was not found or you do not have permission to read this event. \ No newline at end of file + description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/typing.yaml b/api/client-server/v1/typing.yaml new file mode 100644 index 000000000..737c69284 --- /dev/null +++ b/api/client-server/v1/typing.yaml @@ -0,0 +1,77 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Typing API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/typing/{userId}": + put: + summary: Informs the server that the user has started or stopped typing. + description: |- + This tells the server that the user is typing for the next N + milliseconds where N is the value specified in the ``timeout`` key. + Alternatively, if ``typing`` is ``false``, it tells the server that the + user has stopped typing. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + description: The user who has started to type. + required: true + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + description: The room in which the user is typing. + required: true + x-example: "!wefh3sfukhs:example.com" + - in: body + name: typingState + description: The current typing state. + required: true + schema: + type: object + example: |- + { + "typing": true, + "timeout": 30000 + } + properties: + typing: + type: boolean + description: |- + Whether the user is typing or not. If ``false``, the ``timeout`` + key can be omitted. + timeout: + type: integer + description: The length of time in milliseconds to mark this user as typing. + required: ["typing"] + responses: + 200: + description: The new typing state was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/api/client-server/v1/v1-event-schema b/api/client-server/v1/v1-event-schema new file mode 120000 index 000000000..7a0d0326b --- /dev/null +++ b/api/client-server/v1/v1-event-schema @@ -0,0 +1 @@ +../../../event-schemas/schema/v1 \ No newline at end of file diff --git a/api/client-server/v1/voip.yaml b/api/client-server/v1/voip.yaml new file mode 100644 index 000000000..5fdf1ca7a --- /dev/null +++ b/api/client-server/v1/voip.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Voice over IP API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/turnServer": + get: + summary: Obtain TURN server credentials. + description: |- + This API provides credentials for the client to use when initiating + calls. + security: + - accessToken: [] + responses: + 200: + description: The TURN server credentials. + examples: + application/json: |- + { + "username":"1443779631:@user:example.com", + "password":"JlKfBy1QwLrO20385QyAtEyIv0=", + "uris":[ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp" + ], + "ttl":86400 + } + schema: + type: object + properties: + username: + type: string + description: |- + The username to use. + password: + type: string + description: |- + The password to use. + uris: + type: array + items: + type: string + description: A list of TURN URIs + ttl: + type: integer + description: The time-to-live in seconds + required: ["username", "password", "uris", "ttl"] + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema new file mode 120000 index 000000000..b020e6da4 --- /dev/null +++ b/api/client-server/v2_alpha/core-event-schema @@ -0,0 +1 @@ +../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/api/client-server/v2_alpha/definitions/definitions @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/error.yaml b/api/client-server/v2_alpha/definitions/error.yaml new file mode 100644 index 000000000..20312ae4d --- /dev/null +++ b/api/client-server/v2_alpha/definitions/error.yaml @@ -0,0 +1,10 @@ +type: object +description: A Matrix-level Error +properties: + errcode: + type: string + description: An error code. + error: + type: string + description: A human-readable error message. +required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json new file mode 100644 index 000000000..75762d758 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of events", + "items": { + "type": "object" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json new file mode 100644 index 000000000..1cdcb1f41 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -0,0 +1,42 @@ +{ + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": + "The maximum number of events to return." + }, + "types": { + "type": "array", + "description": + "A list of event types to include. If this list is absent then all event types are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_types": { + "type": "array", + "description": + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "senders": { + "type": "array", + "description": + "A list of senders IDs to include. If this list is absent then all senders are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_senders": { + "type": "array", + "description": + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json new file mode 100644 index 000000000..fcf148f36 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of event ids", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json new file mode 100644 index 000000000..86375781c --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -0,0 +1,22 @@ +{ + "type": "object", + "allOf": [{"$ref": "definitions/event_filter.json"}], + "properties": { + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json new file mode 100644 index 000000000..0cd6a7986 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "room": { + "type": "object", + "properties": { + "state": { + "description": + "The state events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "timeline": { + "description": + "The message and state update events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "ephemeral": { + "description": + "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + } + } + }, + "presence": { + "description": + "The presence updates to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "event_format": { + "description": + "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", + "type": "string", + "enum": ["client", "federation"] + }, + "event_fields": { + "type": "array", + "description": + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A literal '.' character in a field name may be escaped using a '\\'. A server may include more fields than were requested.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json new file mode 100644 index 000000000..ddf8d3416 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "properties": { + "limited": { + "type": "boolean", + "description": "Whether there are more events on the server" + }, + "prev_batch": { + "type": "string", + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + } + } +} diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml new file mode 100644 index 000000000..0c2761b7b --- /dev/null +++ b/api/client-server/v2_alpha/filter.yaml @@ -0,0 +1,139 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 filter API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/filter": + post: + summary: Upload a new filter. + description: |- + Uploads a new filter definition to the homeserver. + Returns a filter ID that may be used in /sync requests to + retrict which events are returned to the client. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: + The id of the user uploading the filter. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: body + name: filter + required: true + description: The filter to upload. + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "ephemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + responses: + 200: + description: The filter was created. + examples: + application/json: |- + { + "filter_id": "66696p746572" + } + schema: + type: object + properties: + filter_id: + type: string + description: |- + The ID of the filter that was created. + "/user/{userId}/filter/{filterId}": + get: + summary: Download a filter + parameters: + - in: path + name: userId + type: string + description: |- + The user ID to download a filter for. + x-example: "@alice:example.com" + required: true + - in: path + name: filterId + type: string + description: |- + The filter ID to download. + x-example: "66696p746572" + required: true + responses: + 200: + description: |- + "The filter defintion" + examples: + application/json: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "ephemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/v2_alpha/receipts.yaml new file mode 100644 index 000000000..b60f72e6b --- /dev/null +++ b/api/client-server/v2_alpha/receipts.yaml @@ -0,0 +1,69 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Receipts API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/receipt/{receiptType}/{eventId}": + post: + summary: Send a receipt for the given event ID. + description: |- + This API updates the marker for the given receipt type to the event ID + specified. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room in which to send the event. + required: true + x-example: "!wefuh21ffskfuh345:example.com" + - in: path + type: string + name: receiptType + description: The type of receipt to send. + required: true + x-example: "m.read" + enum: ["m.read"] + - in: path + type: string + name: eventId + description: The event ID to acknowledge up to. + required: true + x-example: "$1924376522eioj:example.com" + - in: body + name: receipt + description: |- + Extra receipt information to attach to ``content`` if any. The + server will automatically set the ``ts`` field. + schema: + type: object + example: |- + {} + responses: + 200: + description: The receipt was sent. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml new file mode 100644 index 000000000..2bd86e730 --- /dev/null +++ b/api/client-server/v2_alpha/registration.yaml @@ -0,0 +1,106 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Registration API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +paths: + "/register": + post: + summary: Register for an account on this homeserver. + description: |- + Register for an account on this homeserver. + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "username": "cheeky_monkey", + "password": "ilovebananas", + "bind_email": false + } + properties: + bind_email: + type: boolean + description: |- + If true, the server binds the email used for authentication to + the Matrix ID with the ID Server. + username: + type: string + description: |- + The local part of the desired Matrix ID. If omitted, + the homeserver MUST generate a Matrix ID local part. + password: + type: string + description: The desired password for the account. + required: ["password"] + responses: + 200: + description: The account has been registered. + examples: + application/json: |- + { + "user_id": "@cheeky_monkey:matrix.org", + "access_token": "abc123", + "home_server": "matrix.org", + "refresh_token": "def456" + } + schema: + type: object + properties: + user_id: + type: string + description: The fully-qualified Matrix ID that has been registered. + access_token: + type: string + description: |- + An access token for the account. + This access token can then be used to authorize other requests. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. + There is no specific error message to indicate that a request has failed because + an access token has expired; instead, if a client has reason to believe its + access token is valid, and it receives an auth error, they should attempt to + refresh for a new token on failure, and retry the request with the new token. + refresh_token: + type: string + # TODO: Work out how to linkify /tokenrefresh + description: |- + (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. + home_server: + type: string + description: The hostname of the Home Server on which the account has been registered. + 400: + description: |- + Part of the request was invalid. This may include one of the following error codes: + + * ``M_USER_IN_USE`` : The desired user ID is already taken. + * ``M_EXCLUSIVE`` : The desired user ID is in the exclusive namespace + claimed by an application service. + + These errors may be returned at any stage of the registration process, + including after authentication if the requested user ID was registered + whilst the client was performing authentication. + + Home Servers MUST perform the relevant checks and return these codes before + performing `User-Interactive Authentication`_, although they may also return + them after authentication is completed if, for example, the requested user ID + was registered whilst the client was performing authentication. + examples: + application/json: |- + { + "errcode": "M_USER_IN_USE", + "error": "Desired user ID is already taken." + } + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml new file mode 100644 index 000000000..a2d5a2b89 --- /dev/null +++ b/api/client-server/v2_alpha/sync.yaml @@ -0,0 +1,307 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 sync API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/sync": + get: + summary: Synchronise the client's state and receive new messages. + description: |- + Synchronise the client's state with the latest state on the server. + Clients use this API when they first log in to get an initial snapshot + of the state on the server, and then continue to call this API to get + incremental deltas to the state, and to receive new messages. + security: + - accessToken: [] + parameters: + - in: query + name: filter + type: string + description: |- + The ID of a filter created using the filter API. + x-example: "66696p746572" + - in: query + name: since + type: string + description: |- + A point in time to continue a sync from. + x-example: "s72594_4483_1934" + - in: query + name: full_state + type: boolean + description: |- + Controls whether to include the full state for all rooms the user + is a member of. + + If this is set to ``true``, then all state events will be returned, + even if ``since`` is non-empty. The timeline will still be limited + by the ``since`` parameter. In this case, the ``timeout`` parameter + will be ignored and the query will return immediately, possibly with + an empty timeline. + + If ``false``, and ``since`` is non-empty, only state which has + changed since the point indicated by ``since`` will be returned. + + By default, this is ``false``. + x-example: "false" + - in: query + name: set_presence + type: string + enum: ["offline"] + description: |- + Controls whether the client is automatically marked as online by + polling this API. If this parameter is omitted then the client is + automatically marked as online when it uses this API. Otherwise if + the parameter is set to "offline" then the client is not marked as + being online when it uses this API. + x-example: "offline" + - in: query + name: timeout + type: integer + description: |- + The maximum time to poll in milliseconds before returning this + request. + x-example: 30000 + responses: + 200: + description: + The initial snapshot or delta for the client to use to update their + state. + schema: + type: object + properties: + next_batch: + type: string + description: |- + The batch token to supply in the ``since`` param of the next + ``/sync`` request. + rooms: + title: Rooms + type: object + description: |- + Updates to rooms. + properties: + joined: + title: Joined + type: object + additionalProperties: + title: Joined Room + type: object + properties: + event_map: + title: EventMap + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + title: Event + description: An event object. + type: object + allOf: + - $ref: "core-event-schema/event.json" + state: + title: State + type: object + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + title: Timeline + type: object + description: |- + The timeline of messages and state changes in the + room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + title: Ephemeral + type: object + description: |- + The ephemeral events in the room that aren't + recorded in the timeline or state of the room. + e.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + invited: + title: Invited + type: object + description: |- + The rooms that the user has been invited to. + additionalProperties: + title: Invited Room + type: object + properties: + invite_state: + title: InviteState + type: object + description: |- + The state of a room that the user has been invited + to. These state events may only have the ``sender``, + ``type``, ``state_key`` and ``content`` keys + present. These events do not replace any state that + the client already has for the room, for example if + the client has archived the room. Instead the + client should keep two separate copies of the + state: the one from the ``invite_state`` and one + from the archived ``state``. If the client joins + the room then the current state will be given as a + delta against the archived ``state`` not the + ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" + archived: + title: Archived + type: object + description: |- + The rooms that the user has left or been banned from. The + entries in the room_map will lack an ``ephemeral`` key. + additionalProperties: + title: Archived Room + type: object + properties: + event_map: + title: EventMap + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + title: Event + description: An event object. + type: object + allOf: + - $ref: "core-event-schema/event.json" + state: + title: State + type: object + description: |- + The state updates for the room up to the point when + the user left. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + title: Timeline + type: object + description: |- + The timeline of messages and state changes in the + room up to the point when the user left. + allOf: + - $ref: "definitions/timeline_batch.json" + presence: + title: Presence + type: object + description: |- + The updates to the presence status of other users. + allOf: + - $ref: "definitions/event_batch.json" + examples: + application/json: |- + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + } + ] + }, + "rooms": { + "joined": { + "!726s6s6q:example.com": { + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "unsigned": { + "prev_content": {"membership": "invite"} + }, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797 + } + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] + } + } + }, + "invited": { + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "invite"} + } + ] + } + } + }, + "archived": {} + } + } diff --git a/api/package.json b/api/package.json index 151934935..84b9dd7b5 100644 --- a/api/package.json +++ b/api/package.json @@ -10,6 +10,6 @@ "license": "ISC", "dependencies": { "nopt": "^3.0.2", - "swagger-parser": "^2.4.1" + "swagger-parser": "^3.2.1" } } diff --git a/api/validator.js b/api/validator.js index 6eef652f3..0d76c09d3 100644 --- a/api/validator.js +++ b/api/validator.js @@ -26,11 +26,10 @@ if (!opts.schema) { } -var errFn = function(err, api, metadata) { +var errFn = function(err, api) { if (!err) { return; } - console.log(metadata); console.error(err); process.exit(1); }; @@ -44,12 +43,14 @@ if (isDir) { process.exit(1); } files.forEach(function(f) { - if (f.indexOf(".yaml") > 0) { - parser.parse(path.join(opts.schema, f), function(err, api, metadata) { + var suffix = ".yaml"; + if (f.indexOf(suffix, f.length - suffix.length) > 0) { + parser.validate(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); } else { + console.error("%s is not valid.", f); errFn(err, api, metadata); } }); @@ -58,12 +59,12 @@ if (isDir) { }); } else{ - parser.parse(opts.schema, function(err, api, metadata) { + parser.validate(opts.schema, function(err, api) { if (!err) { console.log("%s is valid", opts.schema); } else { - errFn(err, api, metadata); + errFn(err, api); } }); }; diff --git a/specification/49_other_non_core_apis.rst b/drafts/address-book-repo.rst similarity index 100% rename from specification/49_other_non_core_apis.rst rename to drafts/address-book-repo.rst index 8855ce9e7..d6315a965 100644 --- a/specification/49_other_non_core_apis.rst +++ b/drafts/address-book-repo.rst @@ -3,10 +3,10 @@ Address book repository .. NOTE:: This section is a work in progress. - Do we even need it? Clients can use out-of-band addressbook servers for now; - this should definitely not be core. .. TODO-spec + Do we even need it? Clients can use out-of-band addressbook servers for now; + this should definitely not be core. - format: POST(?) wodges of json, some possible processing, then return wodges of json on GET. - processing may remove dupes, merge contacts, pepper with extra info (e.g. matrix-ability of contacts), etc. diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst new file mode 100644 index 000000000..93622c3d4 --- /dev/null +++ b/drafts/macaroons_caveats.rst @@ -0,0 +1,34 @@ +Macaroon Caveats +================ + +`Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. + +.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf + +Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. + +Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. + +All caveats must take the form: + +`key` `operator` `value` +where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] +`operator` is a non-empty string which does not contain whitespace +`value` is a non-empty string +And these are joined by single space characters. + +Specified caveats: + ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ +| Caveat name | Description | Legal Values | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ +| gen | Generation of the macaroon caveat spec. | 1 | +| user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | +| refresh - only used to authorize a token refresh | +| time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | +| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| Note that exact equality of time is largely meaningless. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ diff --git a/event-schemas/check.sh b/event-schemas/check.sh index fac36cf85..a6d03b5a4 100755 --- a/event-schemas/check.sh +++ b/event-schemas/check.sh @@ -1,5 +1,11 @@ #!/bin/bash -e # Runs z-schema over all of the schema files (looking for matching examples) + +if ! which z-schema; then + echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema" + exit 1 +fi + find schema/v1/m.* | while read line do split_path=(${line///// }) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py new file mode 100755 index 000000000..e54d3a1ca --- /dev/null +++ b/event-schemas/check_examples.py @@ -0,0 +1,76 @@ +#! /usr/bin/env python + +import sys +import json +import os +import traceback + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + + +def check_example_file(examplepath, schemapath): + with open(examplepath) as f: + example = yaml.load(f) + + with open(schemapath) as f: + schema = yaml.load(f) + + fileurl = "file://" + os.path.abspath(schemapath) + + print ("Checking schema for: %r %r" % (examplepath, schemapath)) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + try: + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + examplepath, schemapath + ), e) + + +def check_example_dir(exampledir, schemadir): + errors = [] + for root, dirs, files in os.walk(exampledir): + for filename in files: + if filename.startswith("."): + # Skip over any vim .swp files. + continue + examplepath = os.path.join(root, filename) + schemapath = examplepath.replace(exampledir, schemadir) + try: + check_example_file(examplepath, schemapath) + except Exception as e: + errors.append(sys.exc_info()) + for (exc_type, exc_value, exc_trace) in errors: + traceback.print_exception(exc_type, exc_value, exc_trace) + if errors: + raise ValueError("Error validating examples") + +if __name__ == '__main__': + try: + check_example_dir("examples", "schema") + except: + sys.exit(1) diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/v1/m.receipt new file mode 100644 index 000000000..bd0b726c3 --- /dev/null +++ b/event-schemas/examples/v1/m.receipt @@ -0,0 +1,13 @@ +{ + "type": "m.receipt", + "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", + "content": { + "$1435641916114394fHBLK:matrix.org": { + "m.read": { + "@rikj:jki.re": { + "ts": 1436451550453 + } + } + } + } +} diff --git a/event-schemas/examples/v1/m.room.avatar b/event-schemas/examples/v1/m.room.avatar new file mode 100644 index 000000000..9fb1189c1 --- /dev/null +++ b/event-schemas/examples/v1/m.room.avatar @@ -0,0 +1,18 @@ +{ + "age": 242352, + "content": { + "info": { + "h": 398, + "w": 394, + "mimetype": "image/jpeg", + "size": 31037 + }, + "url": "mxc://localhost/JWEIFJgwEIhweiWJE" + }, + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.avatar", + "state_key": "", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.canonical_alias b/event-schemas/examples/v1/m.room.canonical_alias new file mode 100644 index 000000000..0203a851e --- /dev/null +++ b/event-schemas/examples/v1/m.room.canonical_alias @@ -0,0 +1,12 @@ +{ + "age": 242352, + "content": { + "alias": "#somewhere:localhost" + }, + "state_key": "", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.canonical_alias", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.history_visibility b/event-schemas/examples/v1/m.room.history_visibility new file mode 100644 index 000000000..fcc3f881b --- /dev/null +++ b/event-schemas/examples/v1/m.room.history_visibility @@ -0,0 +1,12 @@ +{ + "age": 242352, + "content": { + "history_visibility": "shared" + }, + "state_key": "", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.history_visibility", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index b9cd26715..2c174e8a6 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,8 +3,39 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid" + "displayname": "Alice Margatroid", + "third_party_invite": { + "token": "pc98", + "public_key": "abc123", + "key_validity_url": "https://magic.forest/verifykey", + "signed": { + "mxid": "@alice:localhost", + "token": "pc98", + "signatures": { + "magic.forest": { + "ed25519:0": "poi098" + } + } + }, + "sender": "@zun:zun.soft" + } }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/examples/v1/m.room.third_party_invite b/event-schemas/examples/v1/m.room.third_party_invite new file mode 100644 index 000000000..82ac52487 --- /dev/null +++ b/event-schemas/examples/v1/m.room.third_party_invite @@ -0,0 +1,14 @@ +{ + "age": 242352, + "content": { + "display_name": "Alice Margatroid", + "key_validity_url": "https://magic.forest/verifykey", + "public_key": "abc123" + }, + "state_key": "pc98", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.third_party_invite", + "room_id": "!Cuyf34gef24t:localhost", + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/v1/m.typing new file mode 100644 index 000000000..bd53f6fb6 --- /dev/null +++ b/event-schemas/examples/v1/m.typing @@ -0,0 +1,7 @@ +{ + "type": "m.typing", + "room_id": "!z0mnsuiwhifuhwwfw:matrix.org", + "content": { + "user_ids": ["@alice:matrix.org", "@bob:example.com"] + } +} \ No newline at end of file diff --git a/event-schemas/schema/v1/core b/event-schemas/schema/v1/core deleted file mode 100644 index ed374a698..000000000 --- a/event-schemas/schema/v1/core +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "definitions": { - "event": { - "title": "Event", - "description": "The basic set of fields all events must have.", - "type": "object", - "properties": { - "event_id": { - "type": "string", - "description": "The globally unique event identifier." - }, - "user_id": { - "type": "string", - "description": "Contains the fully-qualified ID of the user who *sent* this event." - }, - "content": { - "type": "object", - "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." - }, - "type": { - "type": "string", - "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'" - } - }, - "required": ["event_id", "user_id", "content", "type"] - }, - "room_event": { - "type": "object", - "title": "Room Event", - "description": "In addition to the Event fields, Room Events MUST have the following additional field.", - "allOf":[{ - "$ref": "#/definitions/event" - }], - "properties": { - "room_id": { - "type": "string", - "description": "The ID of the room associated with this event." - } - }, - "required": ["room_id"] - }, - "state_event": { - "type": "object", - "title": "State Event", - "description": "In addition to the Room Event fields, State Events have the following additional fields.", - "allOf":[{ - "$ref": "#/definitions/room_event" - }], - "properties": { - "state_key": { - "type": "string", - "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event." - }, - "prev_content": { - "type": "object", - "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." - } - }, - "required": ["state_key"] - }, - "msgtype_infos": { - "image_info": { - "type": "object", - "title": "ImageInfo", - "description": "Metadata about an image.", - "properties": { - "size": { - "type": "integer", - "description": "Size of the image in bytes." - }, - "w": { - "type": "integer", - "description": "The width of the image in pixels." - }, - "h": { - "type": "integer", - "description": "The height of the image in pixels." - }, - "mimetype": { - "type": "string", - "description": "The mimetype of the image, e.g. ``image/jpeg``." - } - } - } - } - } -} diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/core-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/v1/core-event-schema/event.json new file mode 100644 index 000000000..e73aec809 --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/event.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "title": "Event", + "description": "The basic set of fields all events must have.", + "properties": { + "content": { + "type": "object", + "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." + }, + "type": { + "type": "string", + "description": "The type of event. This SHOULD be namespaced similar to Java package naming conventions e.g. 'com.example.subdomain.event.type'" + } + } +} diff --git a/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json b/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json new file mode 100644 index 000000000..ee75745ea --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "ImageInfo", + "description": "Metadata about an image.", + "properties": { + "size": { + "type": "integer", + "description": "Size of the image in bytes." + }, + "w": { + "type": "integer", + "description": "The width of the image in pixels." + }, + "h": { + "type": "integer", + "description": "The height of the image in pixels." + }, + "mimetype": { + "type": "string", + "description": "The mimetype of the image, e.g. ``image/jpeg``." + } + } +} diff --git a/event-schemas/schema/v1/core-event-schema/room_event.json b/event-schemas/schema/v1/core-event-schema/room_event.json new file mode 100644 index 000000000..d5413f8a0 --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/room_event.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "Room Event", + "description": "In addition to the Event fields, Room Events MUST have the following additional field.", + "allOf":[{ + "$ref": "core-event-schema/event.json" + }], + "properties": { + "room_id": { + "type": "string", + "description": "The ID of the room associated with this event." + }, + "event_id": { + "type": "string", + "description": "The globally unique event identifier." + }, + "user_id": { + "type": "string", + "description": "Contains the fully-qualified ID of the user who *sent* this event." + } + }, + "required": ["room_id"] +} diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json new file mode 100644 index 000000000..88b4900ae --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -0,0 +1,19 @@ +{ + "type": "object", + "title": "State Event", + "description": "In addition to the Room Event fields, State Events have the following additional fields.", + "allOf":[{ + "$ref": "core-event-schema/room_event.json" + }], + "properties": { + "state_key": { + "type": "string", + "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'." + }, + "prev_content": { + "type": "object", + "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." + } + }, + "required": ["state_key"] +} diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/v1/m.call.answer index 598554f02..f11a61ba5 100644 --- a/event-schemas/schema/v1/m.call.answer +++ b/event-schemas/schema/v1/m.call.answer @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by the callee when they wish to answer the call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/v1/m.call.candidates index 6475cf51b..52e067053 100644 --- a/event-schemas/schema/v1/m.call.candidates +++ b/event-schemas/schema/v1/m.call.candidates @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by callers after sending an invite and by the callee after answering. Its purpose is to give the other party additional ICE candidates to try using to communicate.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/v1/m.call.hangup index 3704ec753..026676352 100644 --- a/event-schemas/schema/v1/m.call.hangup +++ b/event-schemas/schema/v1/m.call.hangup @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "Sent by either party to signal their termination of the call. This can be sent either once the call has has been established or before to abort the call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/v1/m.call.invite index 2be014663..585403d05 100644 --- a/event-schemas/schema/v1/m.call.invite +++ b/event-schemas/schema/v1/m.call.invite @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "description": "This event is sent by the caller when they wish to establish a call.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.presence b/event-schemas/schema/v1/m.presence index bb69ce404..79852ac69 100644 --- a/event-schemas/schema/v1/m.presence +++ b/event-schemas/schema/v1/m.presence @@ -1,5 +1,4 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Presence Event", "description": "Informs the client of a user's presence state change.", diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt new file mode 100644 index 000000000..d0f79ac4a --- /dev/null +++ b/event-schemas/schema/v1/m.receipt @@ -0,0 +1,48 @@ +{ + "type": "object", + "title": "Receipt Event", + "description": "Informs the client of new receipts.", + "properties": { + "content": { + "type": "object", + "patternProperties": { + "^\\$": { + "type": "object", + "x-pattern": "$EVENT_ID", + "title": "Receipts", + "description": "The mapping of event ID to a collection of receipts for this event ID. The event ID is the ID of the event being acknowledged and *not* an ID for the receipt itself.", + "properties": { + "m.read": { + "type": "object", + "title": "Users", + "description": "A collection of users who have sent ``m.read`` receipts for this event.", + "patternProperties": { + "^@": { + "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", + "properties": { + "ts": { + "type": "number", + "description": "The timestamp the receipt was sent at." + } + } + } + } + } + } + } + }, + "additionalProperties": false + }, + "type": { + "type": "string", + "enum": ["m.receipt"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["room_id", "type", "content"] +} diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/v1/m.room.aliases index 6585f13d0..43cc9dbcd 100644 --- a/event-schemas/schema/v1/m.room.aliases +++ b/event-schemas/schema/v1/m.room.aliases @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Informs the room about what room aliases it has been given.", "description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.avatar b/event-schemas/schema/v1/m.room.avatar new file mode 100644 index 000000000..276c975c0 --- /dev/null +++ b/event-schemas/schema/v1/m.room.avatar @@ -0,0 +1,64 @@ +{ + "title": "RoomAvatar", + "description": "A picture that is associated with the room. This can be displayed alongside the room information.", + "type": "object", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to the image." + }, + "thumbnail_url": { + "type": "string", + "description": "The URL to the thumbnail of the image." + }, + "thumbnail_info": { + "type": "object", + "title": "ImageInfo", + "description": "Metadata about the image referred to in ``thumbnail_url``.", + "allOf": [{ + "$ref": "core-event-schema/msgtype_infos/image_info.json" + }] + }, + "info": { + "type": "object", + "title": "ImageInfo", + "description": "Metadata about the image referred to in ``url``.", + "properties": { + "size": { + "type": "integer", + "description": "Size of the image in bytes." + }, + "w": { + "type": "integer", + "description": "The width of the image in pixels." + }, + "h": { + "type": "integer", + "description": "The height of the image in pixels." + }, + "mimetype": { + "type": "string", + "description": "The mimetype of the image, e.g. ``image/jpeg``." + } + } + } + }, + "required": ["url"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.avatar"] + } + } +} diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias new file mode 100644 index 000000000..25cd00c08 --- /dev/null +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -0,0 +1,29 @@ +{ + "type": "object", + "title": "Informs the room as to which alias is the canonical one.", + "description": "This event is used to inform the room about which alias should be considered the canonical one. This could be for display purposes or as suggestion to users which alias to use to advertise the room.", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "alias": { + "type": "string", + "description": "The canonical alias." + } + }, + "required": ["alias"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.canonical_alias"] + } + } +} diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index 8f74e69ba..348b6d86e 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "The first event in the room.", "description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { @@ -13,6 +12,10 @@ "creator": { "type": "string", "description": "The ``user_id`` of the room creator. This is set by the homeserver." + }, + "m.federate": { + "type": "boolean", + "description": "Whether users on other servers can join this room. Defaults to ``true`` if key does not exist." } }, "required": ["creator"] diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/v1/m.room.history_visibility new file mode 100644 index 000000000..9abcf5b84 --- /dev/null +++ b/event-schemas/schema/v1/m.room.history_visibility @@ -0,0 +1,30 @@ +{ + "type": "object", + "title": "Controls visibility of history.", + "description": "This event controls whether a user can see the events that happened in a room from before they joined.", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "history_visibility": { + "type": "string", + "description": "Who can see the room history.", + "enum": ["invited","joined","shared","world_readable"] + } + }, + "required": ["history_visibility"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.history_visibility"] + } + } +} diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/v1/m.room.join_rules index 2ba20b89a..70c36ccb1 100644 --- a/event-schemas/schema/v1/m.room.join_rules +++ b/event-schemas/schema/v1/m.room.join_rules @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Describes how users are allowed to join the room.", "description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room. Currently, ``knock`` and ``private`` are reserved keywords which are not implemented.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index ab616176a..45f2ad705 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { @@ -20,8 +19,51 @@ "description": "The avatar URL for this user, if any. This is added by the homeserver." }, "displayname": { - "type": "string", + "type": ["string", "null"], "description": "The display name for this user, if any. This is added by the homeserver." + }, + "third_party_invite": { + "type": "object", + "title": "Invite", + "properties": { + "token": { + "type": "string", + "description": "A token which must be correctly signed, in order to join the room." + }, + "key_validity_url": { + "type": "string", + "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." + }, + "public_key": { + "type": "string", + "description": "A base64-encoded ed25519 key with which token must be signed." + }, + "signed": { + "type": "object", + "title": "signed", + "properties": { + "mxid": { + "type": "string", + "description": "The invited matrix user ID. Must be equal to the user_id property of the event." + }, + "token": { + "type": "string", + "description": "The token property of the containing third_party_invite object." + }, + "signatures": { + "type": "object", + "description": "A single signature from the verifying server, in the format specified by the Signing Events section.", + "title": "Signatures" + } + }, + "required": ["mxid", "signatures", "token"] + }, + "sender": { + "type": "string", + "description": "The matrix user ID of the user who send the invite which is being used." + } + }, + "required": ["token", "key_validity_url", "public_key", "sender", "signed"] } }, "required": ["membership"] @@ -33,6 +75,32 @@ "type": { "type": "string", "enum": ["m.room.member"] + }, + "invite_room_state": { + "type": "array", + "description": "A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``", + "items": { + "type": "object", + "title": "StrippedState", + "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.", + "required": ["type", "state_key", "content"], + "properties": { + "type": { + "type": "string", + "description": "The ``type`` for the event.", + "enum": ["m.room.join_rules", "m.room.canonical_alias", "m.room.avatar", "m.room.name"] + }, + "state_key": { + "type": "string", + "description": "The ``state_key`` for the event." + }, + "content": { + "title": "EventContent", + "type": "object", + "description": "The ``content`` for the event." + } + } + } } } } diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/v1/m.room.message index a849f07f5..91c04b7fe 100644 --- a/event-schemas/schema/v1/m.room.message +++ b/event-schemas/schema/v1/m.room.message @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Message", - "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.", + "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message. This allows clients to display *something* even if it is just plain text.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/v1/m.room.message#m.audio index 9910cb0c8..0a6625f54 100644 --- a/event-schemas/schema/v1/m.room.message#m.audio +++ b/event-schemas/schema/v1/m.room.message#m.audio @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "AudioMessage", "description": "This message represents a single audio clip.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/v1/m.room.message#m.emote index 886abafc0..690600f46 100644 --- a/event-schemas/schema/v1/m.room.message#m.emote +++ b/event-schemas/schema/v1/m.room.message#m.emote @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "EmoteMessage", "description": "This message is similar to ``m.text`` except that the sender is 'performing' the action contained in the ``body`` key, similar to ``/me`` in IRC. This message should be prefixed by the name of the sender. This message could also be represented in a different colour to distinguish it from regular ``m.text`` messages.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/v1/m.room.message#m.file index cf6d65524..c97480ef8 100644 --- a/event-schemas/schema/v1/m.room.message#m.file +++ b/event-schemas/schema/v1/m.room.message#m.file @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "FileMessage", "description": "This message represents a generic file.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -50,7 +49,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/v1/m.room.message#m.image index 8c010b0d7..af78096d6 100644 --- a/event-schemas/schema/v1/m.room.message#m.image +++ b/event-schemas/schema/v1/m.room.message#m.image @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "ImageMessage", "description": "This message represents a single image and an optional thumbnail.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -31,7 +30,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] }, "info": { diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/v1/m.room.message#m.location index cd15f31e4..ef4b5e815 100644 --- a/event-schemas/schema/v1/m.room.message#m.location +++ b/event-schemas/schema/v1/m.room.message#m.location @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "LocationMessage", "description": "This message represents a real-world location.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -30,7 +29,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/v1/m.room.message#m.notice index 0f8044f59..195c56a3b 100644 --- a/event-schemas/schema/v1/m.room.message#m.notice +++ b/event-schemas/schema/v1/m.room.message#m.notice @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "NoticeMessage", "description": "A m.notice message should be considered similar to a plain m.text message except that clients should visually distinguish it in some way. It is intended to be used by automated clients, such as bots, bridges, and other entities, rather than humans. Additionally, such automated agents which watch a room for messages and respond to them ought to ignore m.notice messages. This helps to prevent infinite-loop situations where two automated clients continuously exchange messages, as each responds to the other.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/v1/m.room.message#m.text index 054d4a966..de59cf0ea 100644 --- a/event-schemas/schema/v1/m.room.message#m.text +++ b/event-schemas/schema/v1/m.room.message#m.text @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "TextMessage", "description": "This message is the most basic message and is used to represent text.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/v1/m.room.message#m.video index 222665060..ad68e72a1 100644 --- a/event-schemas/schema/v1/m.room.message#m.video +++ b/event-schemas/schema/v1/m.room.message#m.video @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "VideoMessage", "description": "This message represents a single video clip.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -55,7 +54,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core#/definitions/msgtype_infos/image_info" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } } diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/v1/m.room.message.feedback index 4cfd44d1e..2eaed9990 100644 --- a/event-schemas/schema/v1/m.room.message.feedback +++ b/event-schemas/schema/v1/m.room.message.feedback @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "MessageFeedback", - "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.", + "description": "**NB: Usage of this event is discouraged in favour of the** `receipts module`_. **Most clients will not recognise this event.** Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/v1/m.room.name index 5077774c4..0e0b25be2 100644 --- a/event-schemas/schema/v1/m.room.name +++ b/event-schemas/schema/v1/m.room.name @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "title": "RoomName", "description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.", "type": "object", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { @@ -12,7 +11,7 @@ "properties": { "name": { "type": "string", - "description": "The name of the room." + "description": "The name of the room. This MUST NOT exceed 255 bytes." } }, "required": ["name"] diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/v1/m.room.power_levels index 9e0cb81a6..2ef3bfa3b 100644 --- a/event-schemas/schema/v1/m.room.power_levels +++ b/event-schemas/schema/v1/m.room.power_levels @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Defines the power levels (privileges) of users in the room.", "description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/v1/m.room.redaction index 2896c0da4..5c3ef3916 100644 --- a/event-schemas/schema/v1/m.room.redaction +++ b/event-schemas/schema/v1/m.room.redaction @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Redaction", "description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data. There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.", "allOf": [{ - "$ref": "core#/definitions/room_event" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.third_party_invite b/event-schemas/schema/v1/m.room.third_party_invite new file mode 100644 index 000000000..ba66100f1 --- /dev/null +++ b/event-schemas/schema/v1/m.room.third_party_invite @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "title": "An invitation to a room issued to a third party identifier, rather than a matrix user ID.", + "description": "Acts as an ``m.room.member`` invite event, where there isn't a target user_id to invite. This event contains a token and a public key whose private key must be used to sign the token. Any user who can present that signature may use this invitation to join the target room.", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "display_name": { + "type": "string", + "description": "A user-readable string which represents the user who has been invited. This should not contain the user's third party ID, as otherwise when the invite is accepted it would leak the association between the matrix ID and the third party ID." + }, + "key_validity_url": { + "type": "string", + "description": "A URL which can be fetched, with querystring public_key=public_key, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." + }, + "public_key": { + "type": "string", + "description": "A base64-encoded ed25519 key with which token must be signed." + } + }, + "required": ["display_name", "key_validity_url", "public_key"] + }, + "state_key": { + "type": "string", + "description": "The token, of which a signature must be produced in order to join the room." + }, + "type": { + "type": "string", + "enum": ["m.room.third_party_invite"] + } + } +} diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/v1/m.room.topic index 266df8fac..d13f0c239 100644 --- a/event-schemas/schema/v1/m.room.topic +++ b/event-schemas/schema/v1/m.room.topic @@ -1,10 +1,9 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "title": "Topic", "description": "A topic is a short message detailing what is currently being discussed in the room. It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.", "allOf": [{ - "$ref": "core#/definitions/state_event" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/v1/m.typing new file mode 100644 index 000000000..b712f6ecb --- /dev/null +++ b/event-schemas/schema/v1/m.typing @@ -0,0 +1,28 @@ +{ + "type": "object", + "title": "Typing Event", + "description": "Informs the client of the list of users currently typing.", + "properties": { + "content": { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The list of user IDs typing in this room, if any." + } + }, + "required": ["user_ids"] + }, + "type": { + "type": "string", + "enum": ["m.typing"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["type", "room_id", "content"] +} diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/event-schemas/schema/v1/v1-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/jenkins.sh b/jenkins.sh new file mode 100755 index 000000000..0b217e58a --- /dev/null +++ b/jenkins.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -ex + +(cd event-schemas/ && ./check_examples.py) +(cd api && ./check_examples.py) +(cd scripts && ./gendoc.py -v) +(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") +(cd event-schemas/ && ./check.sh) diff --git a/scripts/continuserv/README b/scripts/continuserv/README new file mode 100644 index 000000000..8ce378501 --- /dev/null +++ b/scripts/continuserv/README @@ -0,0 +1,6 @@ +continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. + +To run it, you must install the `go` tool. You will also need to install fsnotify by running: + `go get gopkg.in/fsnotify.v1` +You can then run continuserv by running: + `go run main.go` diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go new file mode 100644 index 000000000..573c2c955 --- /dev/null +++ b/scripts/continuserv/main.go @@ -0,0 +1,161 @@ +// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. +// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished. +// It does not currently pre-empt stale generations, but will block until they are complete. +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + fsnotify "gopkg.in/fsnotify.v1" +) + +var ( + port = flag.Int("port", 8000, "Port on which to serve HTTP") + + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + wg sync.WaitGroup // Indicates how many updates are pending. + mu sync.Mutex // Prevent multiple updates in parallel. +) + +func main() { + flag.Parse() + + w, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("Error making watcher: %v", err) + } + + dir, err := os.Getwd() + if err != nil { + log.Fatalf("Error getting wd: %v", err) + } + for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) { + if dir == "/" { + log.Fatalf("Could not find git root") + } + } + + filepath.Walk(dir, makeWalker(w)) + + wg.Add(1) + populateOnce(dir) + + ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes + go doPopulate(ch, dir) + + go watchFS(ch, w) + fmt.Printf("Listening on port %d\n", *port) + http.HandleFunc("/", serve) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) + +} + +func watchFS(ch chan struct{}, w *fsnotify.Watcher) { + for { + select { + case e := <-w.Events: + if filter(e) { + fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) + ch <- struct{}{} + } + } + } +} + +func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { + return func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatalf("Error walking: %v", err) + } + if err := w.Add(path); err != nil { + log.Fatalf("Failed to add watch: %v", err) + } + return nil + } +} + +// Return true if event should trigger re-population +func filter(e fsnotify.Event) bool { + // vim is *really* noisy about how it writes files + if e.Op != fsnotify.Write { + return false + } + // Avoid some temp files that vim writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + return false + } + + // Ignore the .git directory - It's very noisy + if strings.Contains(e.Name, "/.git/") { + return false + } + + // Avoid infinite cycles being caused by writing actual output + if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") { + return false + } + return true +} + +func serve(w http.ResponseWriter, req *http.Request) { + wg.Wait() + b := toServe.Load().([]byte) + w.Write(b) +} + +func populateOnce(dir string) { + defer wg.Done() + mu.Lock() + defer mu.Unlock() + cmd := exec.Command("python", "gendoc.py") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + return + } + specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + if err != nil { + toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + return + } + toServe.Store(specBytes) +} + +func doPopulate(ch chan struct{}, dir string) { + var pending int + for { + select { + case <-ch: + if pending == 0 { + wg.Add(1) + } + pending++ + case <-time.After(10 * time.Millisecond): + if pending > 0 { + pending = 0 + populateOnce(dir) + } + } + } +} + +func exists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} diff --git a/scripts/basic.css b/scripts/css/basic.css similarity index 100% rename from scripts/basic.css rename to scripts/css/basic.css diff --git a/scripts/css/blockquote.css b/scripts/css/blockquote.css new file mode 100644 index 000000000..151d3bce0 --- /dev/null +++ b/scripts/css/blockquote.css @@ -0,0 +1,5 @@ +blockquote { + margin: 20px 0 30px; + border-left: 5px solid; + padding-left: 20px; +} diff --git a/scripts/css/codehighlight.css b/scripts/css/codehighlight.css new file mode 100644 index 000000000..fafc43f47 --- /dev/null +++ b/scripts/css/codehighlight.css @@ -0,0 +1,16 @@ +pre.code .comment, code .comment { color: green } +pre.code .keyword, code .keyword { color: darkred; font-weight: bold } +pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold } +pre.code .name.tag, code .name.tag { color: darkgreen } +pre.code .literal, code .literal { color: darkblue } +pre.code .literal.number, code .literal.number { color: blue } + + +/* HTTP Methods have class "name function" */ +pre.code.http .name.function, code.http .name.function { color: black; font-weight: bold } +/* HTTP Paths have class "name namespace" */ +pre.code.http .name.namespace, code.http .name.namespace { color: darkgreen } +/* HTTP "HTTP" strings have class "keyword reserved" */ +pre.code.http .keyword.reserved, code.http .keyword.reserved { color: black; font-weight: bold } +/* HTTP Header names have class "name attribute" */ +pre.code.http .name.attribute, code.http .name.attribute { color: black; font-weight: bold } diff --git a/scripts/nature.css b/scripts/css/nature.css similarity index 98% rename from scripts/nature.css rename to scripts/css/nature.css index 9d67f689b..0fdcc55ad 100644 --- a/scripts/nature.css +++ b/scripts/css/nature.css @@ -244,10 +244,6 @@ div.viewcode-block:target { border-bottom: 1px solid #ac9; } -p { - margin: 0; -} - ul li dd { margin-top: 0; } @@ -282,3 +278,9 @@ td[colspan]:not([colspan="1"]) { thead { background: #eeeeee; } + +div.admonition-rationale { + background-color: #efe; + border: 1px solid #ccc; +} + diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f8ec6316b..ed36a5a78 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,22 +1,228 @@ #! /usr/bin/env python +from argparse import ArgumentParser from docutils.core import publish_file +import copy import fileinput import glob import os +import re import shutil import subprocess import sys +import yaml + +os.chdir(os.path.dirname(os.path.abspath(__file__))) stylesheets = { - "stylesheet_path": ["basic.css", "nature.css"] + "stylesheet_path": glob.glob("css/*.css"), } -def glob_spec_to(out_file_name): - with open(out_file_name, "wb") as outfile: - for f in sorted(glob.glob("../specification/*.rst")): - with open(f, "rb") as infile: - outfile.write(infile.read()) +VERBOSE = False + +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" +def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): + rst_lines = [] + title_chars = "".join(title_styles) + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") + + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None + for i, line in enumerate(file_stream, 1): + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): + rst_lines.append(line) + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): + rst_lines.append(line) + prev_non_title_line = line + continue + + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) + + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + logv((" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0])) + + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) + ) + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): + raise Exception( + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) + ) + + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + logv( + "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + (filename, line_title_style, title_styles[adjusted_level], + file_offset, title_level) + ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + + return "".join(rst_lines) + + +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): + # string are file paths to RST blobs + if isinstance(file_info, basestring): + log("%s %s" % (">" * (1 + title_level), file_info)) + with open(os.path.join(spec_dir, file_info), "r") as f: + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + + rst += "\n\n" + return rst + # dicts look like {0: filepath, 1: filepath} where the key is the title level + elif isinstance(file_info, dict): + levels = sorted(file_info.keys()) + rst = [] + for l in levels: + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + # lists are multiple file paths e.g. [filepath, filepath] + elif isinstance(file_info, list): + rst = [] + for f in file_info: + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) + return "".join(rst) + raise Exception( + "The following 'file' entry in this target isn't a string, list or dict. " + + "It really really should be. Entry: %s" % (file_info,) + ) + + +def build_spec(target, out_filename): + with open(out_filename, "wb") as outfile: + for file_info in target["files"]: + section = get_rst( + file_info=file_info, + title_level=0, + title_styles=target["title_styles"], + spec_dir="../specification/", + adjust_titles=True + ) + outfile.write(section) + + +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] + relative_title_chars = [ + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + def rst2html(i, o): @@ -31,18 +237,127 @@ def rst2html(i, o): settings_overrides=stylesheets ) -def run_through_template(input): - null = open(os.devnull, 'w') - subprocess.check_output( - [ - 'python', 'build.py', - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ], - stderr=null, - cwd="../templating", - ) + +def run_through_template(input, set_verbose): + tmpfile = './tmp/output' + try: + with open(tmpfile, 'w') as out: + args = [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ] + if set_verbose: + args.insert(2, "-v") + log("EXEC: %s" % " ".join(args)) + log(" ==== build.py output ==== ") + print subprocess.check_output( + args, + stderr=out, + cwd="../templating" + ) + except subprocess.CalledProcessError as e: + print e.output + with open(tmpfile, 'r') as f: + sys.stderr.write(f.read() + "\n") + raise + + +""" +Extract and resolve groups for the given target in the given targets listing. +Args: + targets_listing (str): The path to a YAML file containing a list of targets + target_name (str): The name of the target to extract from the listings. +Returns: + dict: Containing "filees" (a list of file paths), "relative_title_styles" + (a dict of relative style keyword to title character) and "title_styles" + (a list of characters which represent the global title style to follow, + with the top section title first, the second section second, and so on.) +""" +def get_build_target(targets_listing, target_name): + build_target = { + "title_styles": [], + "relative_title_styles": {}, + "files": [] + } + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + + build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + + def get_group(group_id, depth): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '%s' but it doesn't exist." % group_name + ) + if not isinstance(group, list): + raise Exception( + "Expected group '%s' to be a list but it isn't." % group_name + ) + # deep copy so changes to depths don't contaminate multiple uses of this group + group = copy.deepcopy(group) + # swap relative depths for absolute ones + for i, entry in enumerate(group): + if isinstance(entry, dict): + group[i] = { + (rel_depth + depth): v for (rel_depth, v) in entry.items() + } + return group + + resolved_files = [] + for file_entry in target["files"]: + # file_entry is a group id + if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + group = get_group(file_entry, 0) + # The group may be resolved to a list of file entries, in which case + # we want to extend the array to insert each of them rather than + # insert the entire list as a single element (which is what append does) + if isinstance(group, list): + resolved_files.extend(group) + else: + resolved_files.append(group) + # file_entry is a dict which has more file entries as values + elif isinstance(file_entry, dict): + resolved_entry = {} + for (depth, entry) in file_entry.iteritems(): + if not isinstance(entry, basestring): + raise Exception( + "Double-nested depths are not supported. Entry: %s" % (file_entry,) + ) + if entry.startswith("group:"): + resolved_entry[depth] = get_group(entry, depth) + else: + # map across without editing (e.g. normal file path) + resolved_entry[depth] = entry + resolved_files.append(resolved_entry) + continue + # file_entry is just a plain ol' file path + else: + resolved_files.append(file_entry) + build_target["files"] = resolved_files + return build_target + +def log(line): + print "gendoc: %s" % line + +def logv(line): + if VERBOSE: + print "gendoc:V: %s" % line + def prepare_env(): try: @@ -53,30 +368,49 @@ def prepare_env(): os.makedirs("./tmp") except OSError: pass - + + def cleanup_env(): shutil.rmtree("./tmp") -def main(): + +def main(target_name, keep_intermediates): prepare_env() - glob_spec_to("tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + log("Building spec [target=%s]" % target_name) + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst", VERBOSE) + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") - cleanup_env() + if not keep_intermediates: + cleanup_env() + if __name__ == '__main__': - if len(sys.argv) > 1: - # we accept no args, so they don't know what they're doing! - print "gendoc.py - Generate the Matrix specification as HTML." - print "Usage:" - print " python gendoc.py" - print "" - print "The specification can then be found in the gen/ folder." - print "" - print "Requirements:" - print " - This script requires Jinja2 and rst2html (docutils)." - sys.exit(0) - main() + parser = ArgumentParser( + "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." + ) + parser.add_argument( + "--nodelete", "-n", action="store_true", + help="Do not delete intermediate files. They will be found in tmp/" + ) + parser.add_argument( + "--target", "-t", default="main", + help="Specify the build target to build from specification/targets.yaml" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Turn on verbose mode." + ) + args = parser.parse_args() + if not args.target: + parser.print_help() + sys.exit(1) + VERBOSE = args.verbose + main(args.target, args.nodelete) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/matrix-org-gendoc.sh index 18961046c..5716786d2 100755 --- a/scripts/matrix-org-gendoc.sh +++ b/scripts/matrix-org-gendoc.sh @@ -1,21 +1,63 @@ #! /bin/bash if [ -z "$1" ]; then - echo "Expected /includes/nav.html file as arg." + echo "Expected /includes/head.html file as 1st arg." exit 1 fi -NAV_BAR=$1 +if [ -z "$2" ]; then + echo "Expected /includes/nav.html file as 2nd arg." + exit 1 +fi + +if [ -z "$3" ]; then + echo "Expected /includes/footer.html file as 3rd arg." + exit 1 +fi + + +HEADER=$1 +NAV_BAR=$2 +FOOTER=$3 + +if [ ! -f $HEADER ]; then + echo $HEADER " does not exist" + exit 1 +fi if [ ! -f $NAV_BAR ]; then echo $NAV_BAR " does not exist" exit 1 fi +if [ ! -f $FOOTER ]; then + echo $FOOTER " does not exist" + exit 1 +fi + python gendoc.py -perl -pi -e 's###' gen/specification.html gen/howtos.html +perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header + +#' gen/specification.html gen/howtos.html -perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s##
[matrix]
#' gen/specification.html gen/howtos.html +perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s## +
+
+ $nav +
+
+
+#' gen/specification.html gen/howtos.html -perl -pi -e 's##
#' gen/specification.html gen/howtos.html +perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s## +
+
+
+
+
+
+ $footer +
+ + #' gen/specification.html gen/howtos.html diff --git a/scripts/speculator/README b/scripts/speculator/README new file mode 100644 index 000000000..82dc3d36e --- /dev/null +++ b/scripts/speculator/README @@ -0,0 +1,18 @@ +speculator allows you to preview pull requests to the matrix.org specification. + +It serves the following HTTP endpoints: + - / lists open pull requests + - /spec/123 which renders the spec as html at pull request 123. + - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. + - /diff/html/123 which gives a diff of the spec's HTML at pull request 123. + +The build or run, you need a working `go` installation. +Then fetch dependencies: + ` go get github.com/hashicorp/golang-lru` + +To run it, then run: + `go run main.go` + +To build the binary (which is necessary for deployment to the matrix.org +servers), you must again install `go` and dependencies, and then run: + `go build` diff --git a/scripts/speculator/htmldiff.pl b/scripts/speculator/htmldiff.pl new file mode 100755 index 000000000..b16890673 --- /dev/null +++ b/scripts/speculator/htmldiff.pl @@ -0,0 +1,564 @@ +#!/usr/bin/perl +# +# htmldiff - present a diff marked version of two html documents +# +# Copyright (c) 1998-2006 MACS, Inc. +# +# Copyright (c) 2007 SiSco, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# See http://www.themacs.com for more information. +# +# usage: htmldiff [[-c] [-l] [-o] oldversion newversion [output]] +# +# -c - disable metahtml comment processing +# -o - disable outputting of old text +# -l - use navindex to create sequence of diffs +# oldversion - the previous version of the document +# newversion - the newer version of the document +# output - a filename to place the output in. If omitted, the output goes to +# standard output. +# +# if invoked with no options or arguments, operates as a CGI script. It then +# takes the following parameters: +# +# oldfile - the URL of the original file +# newfile - the URL of the new file +# mhtml - a flag to indicate whether it should be aware of MetaHTML comments. +# +# requires GNU diff utility +# also requires the perl modules Getopt::Std +# +# NOTE: The markup created by htmldiff may not validate against the HTML 4.0 +# DTD. This is because the algorithm is realtively simple, and there are +# places in the markup content model where the span element is not allowed. +# Htmldiff is NOT aware of these places. +# +# $Source: /u/sources/public/2009/htmldiff/htmldiff.pl,v $ +# $Revision: 1.1 $ +# +# $Log: htmldiff.pl,v $ +# Revision 1.1 2014/01/06 08:04:51 dom +# added copy of htmldiff perl script since aptest.com repo no longer available +# +# Revision 1.5 2008/03/05 13:23:16 ahby +# Fixed a problem with leading whitespace before markup. +# +# Revision 1.4 2007/12/13 13:09:16 ahby +# Updated copyright and license. +# +# Revision 1.3 2007/12/13 12:53:34 ahby +# Changed use of span to ins and del +# +# Revision 1.2 2002/02/13 16:27:23 ahby +# Changed processing model. +# Improved handling of old text and changed styles. +# +# Revision 1.1 2000/07/12 12:20:04 ahby +# Updated to remove empty spans - this fixes validation problems under +# strict. +# +# Revision 1.11 1999/12/08 19:46:45 ahby +# Fixed validation errors introduced by placing markup where it didn't +# belong. +# +# Revision 1.10 1999/10/18 13:42:58 ahby +# Added -o to the usage message. +# +# Revision 1.9 1999/05/04 12:29:11 ahby +# Added an option to turn off the display of old text. +# +# Revision 1.8 1999/04/09 14:37:27 ahby +# Fixed a perl syntax error. +# +# Revision 1.7 1999/04/09 14:35:49 ahby +# Added reference to MACS homepage. +# +# Revision 1.6 1999/04/09 14:35:09 ahby +# Added comment about validity of generated markup. +# +# Revision 1.5 1999/02/22 22:17:54 ahby +# Changed to use stylesheets. +# Changed to rely upon span. +# Changed to work around content model problems. +# +# Revision 1.4 1999/02/08 02:32:22 ahby +# Added a copyright statement. +# +# Revision 1.3 1999/02/08 02:30:40 ahby +# Added header processing. +# +# Revision 1.2 1998/12/10 17:31:31 ahby +# Fixed to escape less-thans in change blocks and to not permit change +# markup within specific elements (like TITLE). +# +# Revision 1.1 1998/11/26 00:09:22 ahby +# Initial revision +# +# + +use Getopt::Std; + +sub usage { + print STDERR "htmldiff [-c] [-o] oldversion newversion [output]\n"; + exit; +} + +sub url_encode { + my $str = shift; + $str =~ s/([\x00-\x1f\x7F-\xFF])/ + sprintf ('%%%02x', ord ($1))/eg; + return $str; +} + +# markit - diff-mark the streams +# +# markit(file1, file2) +# +# markit relies upon GNUdiff to mark up the text. +# +# The markup is encoded using special control sequences: +# +# a block wrapped in control-a is deleted text +# a block wrapped in control-b is old text +# a block wrapped in control-c is new text +# +# The main processing loop attempts to wrap the text blocks in appropriate +# SPANs based upon the type of text that it is. +# +# When the loop encounters a < in the text, it stops the span. Then it outputs +# the element that is defined, then it restarts the span. + +sub markit { + my $retval = ""; + my($file1) = shift; + my($file2) = shift; +# my $old="deleted text: %c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $old="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $new="%c'\012'%c'\003'%c'\012'%>%c'\012'%c'\003'%c'\012'"; + my $unchanged="%="; + my $changed="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + if ($opt_o) { + $old = ""; + $changed = "%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + } +# my $old="%c'\002'deleted text:%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\012'%c'\002'"; +# my $new="%c'\002'%c'\012'%c'\002'%>%c'\002'%c'\002'%c'\012'"; +# my $unchanged="%="; +# my $changed="%c'\002'%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\002'%c'\012'%>%c'\012'%c'\002'%c'\002'%c'\012'"; + + my @span; + $span[0]=""; + $span[1]=""; + $span[2]=""; + $span[3]=""; + $span[4]=""; + + my @diffEnd ; + $diffEnd[1] = ''; + $diffEnd[2] = ''; + $diffEnd[3] = ''; + $diffEnd[4] = ''; + + my $diffcounter = 0; + + open(FILE, qq(diff -d --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 |)) || die("Diff failed: $!"); +# system (qq(diff --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 > /tmp/output)); + + my $state = 0; + my $inblock = 0; + my $temp = ""; + my $lineCount = 0; + +# strategy: +# +# process the output of diff... +# +# a link with control A-D means the start/end of the corresponding ordinal +# state (1-4). Resting state is state 0. +# +# While in a state, accumulate the contents for that state. When exiting the +# state, determine if it is appropriate to emit the contents with markup or +# not (basically, if the accumulated buffer contains only empty lines or lines +# with markup, then we don't want to emit the wrappers. We don't need them. +# +# Note that if there is markup in the "old" block, that markup is silently +# removed. It isn't really that interesting, and it messes up the output +# something fierce. + + while () { + my $anchor = $opt_l ? qq[] : "" ; + my $anchorEnd = $opt_l ? q[] : "" ; + $lineCount ++; + if ($state == 0) { # if we are resting and we find a marker, + # then we must be entering a block + if (m/^([\001-\004])/) { + $state = ord($1); + $_ = ""; + } +# if (m/^\001/) { +# $state = 1; +# s/^/$span[1]/; +# } elsif (m/^\002/) { +# $state = 2; +# s/^/$span[2]/; +# } elsif (m/^\003/) { +# $state = 3; +# s/^/$span[3]/; +# } elsif (m/^\004/) { +# $state = 4; +# s/^/$span[4]/; +# } + } else { + # if we are in "old" state, remove markup + if (($state == 1) || ($state == 2)) { + s/\<.*\>//; # get rid of any old markup + s/\/>/g; + } + # if we found another marker, we must be exiting the state + if (m/^([\001-\004])/) { + if ($temp ne "") { + $_ = $span[$state] . $anchor . $temp . $anchorEnd . $diffEnd[$state] . "\n"; + $temp = ""; + } else { + $_ = "" ; + } + $state = 0; + } elsif (m/^\s*\ +.diff-old-a { + font-size: smaller; + color: red; +} + +.diff-new { background-color: yellow; } +.diff-chg { background-color: lime; } +.diff-new:before, +.diff-new:after + { content: "\2191" } +.diff-chg:before, .diff-chg:after + { content: "\2195" } +.diff-old { text-decoration: line-through; background-color: #FBB; } +.diff-old:before, +.diff-old:after + { content: "\2193" } +:focus { border: thin red solid} + +); + if ($opt_t) { + $styles .= q( + +); + + } + + if ($stripheader) { + open(HEADER, ">$headertmp"); + } + + my $incomment = 0; + my $inhead = 1; + open(FILE, $filename) || die("File $filename cannot be opened: $!"); + while () { + if ($inhead == 1) { + if (m/\<\/head/i) { + print HEADER $styles; + } + if (m/\ +); + } + close HEADER; + } else { + print HEADER; + } + } else { + if ($incomment) { + if (m;-->;) { + $incomment = 0; + s/.*-->//; + } else { + next; + } + } + if (m;;) { + s///; + } + if (m; - m.call.candidate --------> - [more candidates events] - User answers call - <------ m.call.answer - [...] - <------ m.call.hangup - -Or a rejected call: - -:: - - Caller Callee - m.call.invite -----------> - m.call.candidate --------> - [more candidates events] - User rejects call - <------- m.call.hangup - -Calls are negotiated according to the WebRTC specification. - - -Glare -~~~~~ -This specification aims to address the problem of two users calling each other -at roughly the same time and their invites crossing on the wire. It is a far -better experience for the users if their calls are connected if it is clear -that their intention is to set up a call with one another. - -In Matrix, calls are to rooms rather than users (even if those rooms may only -contain one other user) so we consider calls which are to the same room. - -The rules for dealing with such a situation are as follows: - - - If an invite to a room is received whilst the client is preparing to send an - invite to the same room, the client should cancel its outgoing call and - instead automatically accept the incoming call on behalf of the user. - - If an invite to a room is received after the client has sent an invite to - the same room and is waiting for a response, the client should perform a - lexicographical comparison of the call IDs of the two calls and use the - lesser of the two calls, aborting the greater. If the incoming call is the - lesser, the client should accept this call on behalf of the user. - -The call setup should appear seamless to the user as if they had simply placed -a call and the other party had accepted. Thusly, any media stream that had been -setup for use on a call should be transferred and used for the call that -replaces it. - diff --git a/specification/40_content_repository.rst b/specification/40_content_repository.rst deleted file mode 100644 index df04605a1..000000000 --- a/specification/40_content_repository.rst +++ /dev/null @@ -1,76 +0,0 @@ -Content repository -================== - -HTTP API --------- - -Uploads are POSTed to a resource which returns a token which is used to GET -the download. Uploads are POSTed to the sender's local homeserver, but are -downloaded from the recipient's local homeserver, which must thus first transfer -the content from the origin homeserver using the same API (unless the origin -and destination homeservers are the same). The upload/download API is:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content-uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - -Clients can get thumbnails by supplying a desired width and height and -thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - -The thumbnail methods are "crop" and "scale". "scale" trys to return an -image where either the width or the height is smaller than the requested -size. The client should then scale and letterbox the image if it needs to -fit within a given rectangle. "crop" trys to return an image where the -width and height are close to the requested size and the aspect matches -the requested size. The client should scale the image if it needs to fit -within a given rectangle. - -Homeservers may generate thumbnails for content uploaded to remote -homeservers themselves or may rely on the remote homeserver to thumbnail -the content. Homeservers may return thumbnails of a different size to that -requested. However homeservers should provide exact matches where reasonable. -Homeservers must never upscale images. - -Security --------- - -Clients may try to upload very large files. Homeservers should not store files -that are too large and should not serve them to clients. - -Clients may try to upload very large images. Homeservers should not attempt to -generate thumbnails for images that are too large. - -Remote homeservers may host very large files or images. Homeserver should not -proxy or thumbnail large files or images from remote homeservers. - -Clients may try to upload a large number of files. Homeservers should limit the -number and total size of media that can be uploaded by clients. - -Clients may try to access a large number of remote files through a homeserver. -Homeservers should restrict the number and size of remote files that it caches. - -Clients or remote homeservers may try to upload malicious files targeting -vulnerabilities in either the homeserver thumbnailing or the client decoders. - diff --git a/specification/42_push_overview.rst b/specification/42_push_overview.rst deleted file mode 100644 index 972a8eea1..000000000 --- a/specification/42_push_overview.rst +++ /dev/null @@ -1,78 +0,0 @@ -Push Notifications -================== - -Overview --------- - -:: - - +--------------------+ +-------------------+ - Matrix HTTP | | | | - Notification Protocol | App Developer | | Device Vendor | - | | | | - +-------------------+ | +----------------+ | | +---------------+ | - | | | | | | | | | | - | Matrix Home Server+-----> Push Gateway | +---> Push Provider | | - | | | | | | | | | | - +-^-----------------+ | +----------------+ | | +----+----------+ | - | | | | | | - Matrix | | | | | | - Client/Server API + | | | | | - | | +--------------------+ +-------------------+ - | +--+-+ | - | | <------------------------------------------+ - +---+ | - | | Provider Push Protocol - +----+ - - Mobile Device or Client - - -Matrix supports push notifications as a first class citizen. Home Servers send -notifications of user events to user-configured HTTP endpoints. User may also -configure a number of rules that determine what events generate notifications. -These are all stored and managed by the users home server such that settings can -be reused between client apps as appropriate. - -The above diagram shows the flow of push notifications being sent to a handset -where push notifications are submitted via the handset vendor, such as Apple's -APNS or Google's GCM. This happens as follows: - - 1. The client app signs in to a Matrix Home Server - 2. The client app registers with its vendor's Push Notification provider and - obtains a routing token of some kind. - 3. The mobile app, uses the Matrix client/server API to add a 'pusher', - providing the URL of a specific Push Gateway which is configured for that - application. It also provides the routing token it has acquired from the - Push Notification Provider. - 4. The Home Server starts sending notification HTTP requests to the Push - Gateway using the supplied URL. The Push Gateway relays this notification to - the Push Notification Provider, passing the routing token along with any - necessary private credentials the provider requires to send push - notifications. - 5. The Push Notification provider sends the notification to the device. - -Nomenclature ------------- - -Pusher - A 'pusher' is an activity in the Home Server that manages the sending - of HTTP notifications for a single device of a single user. - -Push Rules - A push rule is a single rule, configured by a matrix user, that gives - instructions to the Home Server about whether an event should be notified - about and how given a set of conditions. Matrix clients allow the user to - configure these. They create and view them via the Client to Server REST API. - -Push Gateway - A push gateway is a server that receives HTTP event notifications from Home - Servers and passes them on to a different protocol such as APNS for iOS - devices or GCM for Android devices. Matrix.org provides a reference push - gateway, 'sygnal'. A client app tells a Home Server what push gateway - to send notifications to when it sets up a pusher. - -For information on the client-server API for setting pushers and push rules, see -the Client Server API section. For more information on the format of HTTP -notifications, see the HTTP Notification Protocol section. - diff --git a/specification/43_push_cs_api.rst b/specification/43_push_cs_api.rst deleted file mode 100644 index 022f50c3b..000000000 --- a/specification/43_push_cs_api.rst +++ /dev/null @@ -1,423 +0,0 @@ -Pushers HTTP API ----------------- - -To receive any notification pokes at all, it is necessary to configure a -'pusher' on the Home Server that you wish to receive notifications from. There -is a single API endpoint for this:: - - POST $PREFIX/pushers/set - -This takes a JSON object with the following keys: - -pushkey - This is a unique identifier for this pusher. The value you should use for this - is the routing or destination address information for the notification, for - example, the APNS token for APNS or the Registration ID for GCM. If your - notification client has no such concept, use any unique identifier. Max length, - 512 bytes. -kind - The kind of pusher to configure. 'http' makes a pusher that sends HTTP pokes. - null deletes the pusher. -profile_tag - This is a string that determines what set of device rules will be matched when - evaluating push rules for this pusher. It is an arbitrary string. Multiple - devices maybe use the same profile_tag. It is advised that when an app's - data is copied or restored to a different device, this value remain the same. - Client apps should offer ways to change the profile_tag, optionally copying - rules from the old profile tag. Max length, 32 bytes. -app_id - appId is a reverse-DNS style identifier for the application. It is recommended - that this end with the platform, such that different platform versions get - different app identifiers. Max length, 64 chars. -app_display_name - A string that will allow the user to identify what application owns this - pusher. -device_display_name - A string that will allow the user to identify what device owns this pusher. -lang - The preferred language for receiving notifications (eg, 'en' or 'en-US') -data - A dictionary of information for the pusher implementation itself. For HTTP - pushers, this must contain a 'url' key which is a string of the URL that - should be used to send notifications. -append - If this is set to boolean true, the Home Server should add another pusher - with the given pushkey and App ID in addition to any others with different - user IDs. Otherwise, the Home Server must remove any other pushers with the - same App ID and pushkey for different users. The default is false. - -If the pusher was created successfully, a JSON dictionary is returned (which may -be empty). - - -Push Rules ----------- -Home Servers have an interface to configure what events trigger notifications. -This behaviour is configured through 'Push Rules'. Push Rules come in a variety -of different kinds and each kind of rule has an associated priority. The -different kinds of rule, in descending order of priority, are: - -Override Rules - The highest priority rules are user-configured overrides. -Content Rules - These configure behaviour for (unencrypted) messages that match certain - patterns. Content rules take one parameter, 'pattern', that gives the pattern - to match against. This is treated in the same way as pattern for event_match - conditions, below. -Room Rules - These change the behaviour of all messages to a given room. The rule_id of a - room rule is always the ID of the room that it affects. -Sender - These rules configure notification behaviour for messages from a specific, - named Matrix user ID. The rule_id of Sender rules is always the Matrix user - ID of the user whose messages theyt apply to. -Underride - These are identical to override rules, but have a lower priority than content, - room and sender rules. - -In addition, each kind of rule may be either global or device-specific. Device -specific rules only affect delivery of notifications via pushers with a matching -profile_tag. All device-specific rules are higher priority than all global -rules. Thusly, the full list of rule kinds, in descending priority order, is as -follows: - - * Device-specific Override - * Device-specific Content - * Device-specific Room - * Device-specific Sender - * Device-specific Underride - * Global Override - * Global Content - * Global Room - * Global Sender - * Global Underride - -For some kinds of rule, rules of the same kind also have an ordering with -respect to one another. The kinds that do not are room and sender rules where -the rules are mutually exclusive by definition and therefore an ordering would -be redundant. Actions for the highest priority rule and only that rule apply -(for example, a set_tweak action in a lower priority rule will not apply if a -higher priority rule matches, even if that rule does not specify any tweaks). - -Rules also have an identifier, rule_id, which is a string. The rule_id is -unique within the kind of rule and scope: rule_ids need not be unique between -rules of the same kind on different devices. - -A home server may also have server default rules of each kind and in each scope. -Server default rules are lower priority than user-defined rules in each scope. -Server default rules (and only server default rules) begin with a dot ('.') -character. - -In addition, all rules may be enabled or disabled. Disabled rules never match. - -If no rules match an event, the Home Server should not notify for the message -(that is to say, the default action is "dont-notify"). Events that the user sent -themself are never alerted for. - -Predefined Rules ----------------- -Matrix specifies the following rule IDs for server default rules. Home Servers -may define rules as follows with the given IDs. If Home Servers provide rules -with these IDs, their semantics should match those given below: - -.m.rule.contains_user_name - Matches any message whose content is unencrypted and contains the local part - of the user's Matrix ID, separated by word boundaries. - - Definition (as a content rule):: - - { - "rule_id": ".m.rule.contains_user_name" - "pattern": "[the lcoal part of the user's Matrix ID]", - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ], - } - -.m.rule.contains_display_name - Matches any message whose content is unencrypted and contains the user's - current display name in the room in which it was sent. - - Definition (this rule can only be an override or underride rule):: - - { - "rule_id": ".m.rule.contains_display_name" - "conditions": [ - { - "kind": "contains_display_name" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ], - } - -.m.rule.room_one_to_one - Matches any message sent in a room with exactly two members. - - Definition (this rule can only be an override or underride rule):: - - { - "rule_id": ".m.rule.room_two_members" - "conditions": [ - { - "is": "2", - "kind": "room_member_count" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ], - } - -.m.rule.suppress_notices - Matches messages with 'msgtype' of 'notice'. This should be an override rule - such that, when enabled, it takes priority over content / sender / room rules. - - Definition:: - - { - 'rule_id': '.m.rule.suppress_notices', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.msgtype', - 'pattern': 'm.notice', - } - ], - 'actions': [ - 'dont-notify', - ] - } - -.m.rule.fallback - Matches any message. Used to define the behaviour of messages that match no - other rules. Therefore, if Home Servers define this, it should be the lowest - priority underride rule. - - Definition:: - - { - "rule_id": ".m.rule.fallback" - "conditions": [], - "actions": [ - "notify" - ], - } - -Push Rules: Actions: --------------------- -All rules have an associated list of 'actions'. An action affects if and how a -notification is delievered for a matching event. This standard defines the -following actions, although if Home servers wish to support more, they are free -to do so: - -notify - This causes each matching event to generate a notification. -dont_notify - Prevents this event from generating a notification -coalesce - This enables notifications for matching events but activates Home Server - specific behaviour to intelligently coalesce multiple events into a single - notification. Not all Home Servers may support this. Those that do not should - treat it as the 'notify' action. -set_tweak - Sets an entry in the 'tweaks' dictionary key that is sent in the notification - poke. This takes the form of a dictionary with a 'set_tweak' key whose value - is the name of the tweak to set. It may also have a 'value' key which is - the value to which it should be set. - -Actions that have no parameters are represented as a string. Otherwise, they are -represented as a dictionary with a key equal to their name and other keys as -their parameters, eg. { "set_tweak": "sound", "value": "default" } - -Push Rules: Actions: Tweaks ---------------------------- -The 'set_tweak' key action is used to add an entry to the 'tweaks' dictionary -that is sent in the notification poke. The following tweaks are defined: - -sound - A sound to be played when this notification arrives. 'default' means to - play a default sound. -highlight - Whether or not this message should be highlighted in the UI. This will - normally take the form of presenting the message in a different colour and/or - weight. The UI might also be adjusted to draw particular attention to the room - in which the event occurred. The value may be omitted from the highlight - tweak, in which case it should be read as if it had a value of true. - -Tweaks are passed transparently through the Home Server so client applications -and push gateways may agree on additional tweaks, for example, how to flash the -notification light on a mobile device. - -If a kind of tweak that a client understands is not specified in an action, the -client may choose a sensible behaviour for the tweak. - -Push Rules: Conditions ----------------------- -Override, Underride and Default rules have a list of 'conditions'. All -conditions must hold true for an event in order for a rule to be applied to an -event. A rule with no conditions always matches. Matrix specifies the following -conditions, although if Home Servers wish to support others, they are free to -do so: - -event_match - This is a glob pattern match on a field of the event. Parameters: - * 'key': The dot-separated field of the event to match, eg. content.body - * 'pattern': The glob-style pattern to match against. Patterns with no - special glob characters should be treated as having asterisks - prepended and appended when testing the condition. -profile_tag - Matches the profile_tag of the device that the notification would be - delivered to. Parameters: - - * 'profile_tag': The profile_tag to match with. -contains_display_name - This matches unencrypted messages where content.body contains the owner's - display name in that room. This is a separate rule because display names may - change and as such it would be hard to maintain a rule that matched the user's - display name. This condition has no parameters. -room_member_count - This matches the current number of members in the room. - * 'is': A decimal integer optionally prefixed by one of, '==', '<', '>', - '>=' or '<='. A prefix of '<' matches rooms where the member count is - strictly less than the given number and so forth. If no prefix is present, - this matches rooms where the member count is exactly equal to the given - number (ie. the same as '=='). - -Room, Sender, User and Content rules do not have conditions in the same way, -but instead have predefined conditions, the behaviour of which can be configured -using parameters named as described above. In the cases of room and sender -rules, the rule_id of the rule determines its behaviour. - -Push Rules: API ---------------- -Rules live under a hierarchy in the REST API that resembles:: - - $PREFIX/pushrules/// - -The component parts are as follows: - -scope - Either 'global' or 'device/' to specify global rules or - device rules for the given profile_tag. -kind - The kind of rule, ie. 'override', 'underride', 'sender', 'room', 'content'. -rule_id - The identifier for the rule. - -To add or change a rule, a client performs a PUT request to the appropriate URL. -When adding rules of a type that has an ordering, the client can add parameters -that define the priority of the rule: - -before - Use 'before' with a rule_id as its value to make the new rule the next-more - important rule with respect to the given rule. -after - This makes the new rule the next-less important rule relative to the given - rule. - -All requests to the push rules API also require an access_token as a query -paraemter. - -The content of the PUT request is a JSON object with a list of actions under the -'actions' key and either conditions (under the 'conditions' key) or the -appropriate parameters for the rule (under the appropriate key name). - -Examples: - -To create a rule that suppresses notifications for the room with ID '!dj234r78wl45Gh4D:matrix.org':: - - curl -X PUT -H "Content-Type: application/json" -d '{ "actions" : ["dont_notify"] }' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" - -To suppress notifications for the user '@spambot:matrix.org':: - - curl -X PUT -H "Content-Type: application/json" -d '{ "actions" : ["dont_notify"] }' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" - -To always notify for messages that contain the work 'cake' and set a specific sound (with a rule_id of 'SSByZWFsbHkgbGlrZSBjYWtl'):: - - curl -X PUT -H "Content-Type: application/json" -d '{ "pattern": "cake", "actions" : ["notify", {"set_sound":"cakealarm.wav"}] }' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" - -To add a rule suppressing notifications for messages starting with 'cake' but ending with 'lie', superseeding the previous rule:: - - curl -X PUT -H "Content-Type: application/json" -d '{ "pattern": "cake*lie", "actions" : ["notify"] }' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" - -To add a custom sound for notifications messages containing the word 'beer' in any rooms with 10 members or fewer (with greater importance than the room, sender and content rules):: - - curl -X PUT -H "Content-Type: application/json" -d '{ "conditions": [{"kind": "event_match", "key": "content.body", "pattern": "beer" }, {"kind": "room_member_count", "is": "<=10"}], "actions" : ["notify", {"set_sound":"beeroclock.wav"}] }' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456 - - -To delete rules, a client would just make a DELETE request to the same URL:: - - curl -X DELETE "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%23spam%3Amatrix.org?access_token=123456" - - -Retrieving the current ruleset can be done either by fetching individual rules -using the scheme as specified above. This returns the rule in the same format as -would be given in the PUT API with the addition of a rule_id:: - - curl "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%23spam%3Amatrix.org?access_token=123456" - -Returns:: - - { - "actions": [ - "dont_notify" - ], - "rule_id": "#spam:matrix.org", - "enabled": true - } - -Clients can also fetch broader sets of rules by removing path components. -Requesting the root level returns a structure as follows:: - - { - "device": { - "exampledevice": { - "content": [], - "override": [], - "room": [ - { - "actions": [ - "dont_notify" - ], - "rule_id": "#spam:matrix.org", - "enabled", true - } - ], - "sender": [], - "underride": [] - } - }, - "global": { - "content": [], - "override": [], - "room": [], - "sender": [], - "underride": [] - } - } - -Adding patch components to the request drills down into this structure to filter -to only the requested set of rules. - -Enabling and Disabling Rules ----------------------------- -Rules can be enabled or disabled with a PUT operation to the 'enabled' component -beneath the rule's URI with a content of 'true' or 'false':: - - curl -X PUT -H "Content-Type: application/json" -d 'false' "http://localhost:8008/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org/enabled?access_token=123456" - - diff --git a/specification/44_push_push_gw_api.rst b/specification/44_push_push_gw_api.rst deleted file mode 100644 index b182503b5..000000000 --- a/specification/44_push_push_gw_api.rst +++ /dev/null @@ -1,144 +0,0 @@ -HTTP Notification Protocol --------------------------- - -This describes the format used by "http" pushers to send notifications of -events. - -Notifications are sent as HTTP POST requests to the URL configured when the -pusher is created, but Matrix strongly recommends that the path should be:: - - /_matrix/push/v1/notify - -The body of the POST request is a JSON dictionary. The format -is as follows:: - - { - "notification": { - "id": "$3957tyerfgewrf384", - "room_id": "!slw48wfj34rtnrf:example.com", - "type": "m.room.message", - "sender": "@exampleuser:matrix.org", - "sender_display_name": "Major Tom", - "room_name": "Mission Control", - "room_alias": "#exampleroom:matrix.org", - "prio": "high", - "content": { - "msgtype": "m.text", - "body": "I'm floating in a most peculiar way." - } - }, - "counts": { - "unread" : 2, - "missed_calls": 1 - } - "devices": [ - { - "app_id": "org.matrix.matrixConsole.ios", - "pushkey": "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/", - "pushkey_ts": 12345678, - "data" : { - }, - "tweaks": { - "sound": "bing" - } - } - ] - } - } - -The contents of this dictionary are defined as follows: - -id - An identifier for this notification that may be used to detect duplicate - notification requests. This is not necessarily the ID of the event that - triggered the notification. -room_id - The ID of the room in which this event occurred. -type - The type of the event as in the event's 'type' field. -sender - The sender of the event as in the corresponding event field. -sender_display_name - The current display name of the sender in the room in which the event - occurred. -room_name - The name of the room in which the event occurred. -room_alias - An alias to display for the room in which the event occurred. -prio - The priority of the notification. Acceptable values are 'high' or 'low. If - omitted, 'high' is assumed. This may be used by push gateways to deliver less - time-sensitive notifications in a way that will preserve battery power on - mobile devices. -content - The 'content' field from the event, if present. If the event had no content - field, this field is omitted. -counts - This is a dictionary of the current number of unacknowledged communications - for the recipient user. Counts whose value is zero are omitted. -unread - The number of unread messages a user has accross all of the rooms they are a - member of. -missed_calls - The number of unacknowledged missed calls a user has accross all rooms of - which they are a member. -device - This is an array of devices that the notification should be sent to. -app_id - The app_id given when the pusher was created. -pushkey - The pushkey given when the pusher was created. -pushkey_ts - The unix timestamp (in seconds) when the pushkey was last updated. -data - A dictionary of additional pusher-specific data. For 'http' pushers, this is - the data dictionary passed in at pusher creation minus the 'url' key. -tweaks - A dictionary of customisations made to the way this notification is to be - presented. These are added by push rules. -sound - Sets the sound file that should be played. 'default' means that a default - sound should be played. - -And additional key is defined but only present on member events: - -user_is_target - This is true if the user receiving the notification is the subject of a member - event (ie. the state_key of the member event is equal to the user's Matrix - ID). - -The recipient of an HTTP notification should respond with an HTTP 2xx response -when the notification has been processed. If the endpoint returns an HTTP error -code, the Home Server should retry for a reasonable amount of time with a -reasonable backoff scheme. - -The endpoint should return a JSON dictionary as follows:: - - { - "rejected": [ "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/" ] - } - -Whose keys are: - -rejected - A list of all pushkeys given in the notification request that are not valid. - These could have been rejected by an upstream gateway because they have - expired or have never been valid. Home Servers must cease sending notification - requests for these pushkeys and remove the associated pushers. It may not - necessarily be the notification in the request that failed: it could be that - a previous notification to the same pushkey failed. - -Push: Recommendations for APNS ------------------------------- -For sending APNS notifications, the exact format is flexible and up to the -client app and its push gateway to agree on (since APNS requires that the sender -have a private key owned by the app developer, each app must have its own push -gateway). However, Matrix strongly recommends: - - * That the APNS token be base64 encoded and used as the pushkey. - * That a different app_id be used for apps on the production and sandbox - APS environments. - * That APNS push gateways do not attempt to wait for errors from the APNS - gateway before returning and instead to store failures and return - 'rejected' responses next time that pushkey is used. - diff --git a/specification/45_typing_notifications.rst b/specification/45_typing_notifications.rst deleted file mode 100644 index 7df9a238e..000000000 --- a/specification/45_typing_notifications.rst +++ /dev/null @@ -1,58 +0,0 @@ -Typing Notifications -==================== - -Client APIs ------------ - -To set "I am typing for the next N msec":: - PUT .../rooms//typing/ - Content: { "typing": true, "timeout": N } - # timeout is in msec; I suggest no more than 20 or 30 seconds - -This should be re-sent by the client to continue informing the server the user -is still typing; I suggest a safety margin of 5 seconds before the expected -timeout runs out. Just keep declaring a new timeout, it will replace the old -one. - -To set "I am no longer typing":: - PUT ../rooms//typing/ - Content: { "typing": false } - -Client Events -------------- - -All room members will receive an event on the event stream:: - - { - "type": "m.typing", - "room_id": "!room-id-here:matrix.org", - "content": { - "user_ids": ["list of", "every user", "who is", "currently typing"] - } - } - -The client must use this list to *REPLACE* its knowledge of every user who is -currently typing. The reason for this is that the server DOES NOT remember -users who are not currently typing, as that list gets big quickly. The client -should mark as not typing, any user ID who is not in that list. - -Server APIs ------------ - -Servers will emit EDUs in the following form:: - - { - "type": "m.typing", - "content": { - "room_id": "!room-id-here:matrix.org", - "user_id": "@user-id-here:matrix.org", - "typing": true/false, - } - } - -Server EDUs don't (currently) contain timing information; it is up to -originating HSes to ensure they eventually send "stop" notifications. - -((This will eventually need addressing, as part of the wider typing/presence -timer addition work)) - diff --git a/specification/50_appendices.rst b/specification/appendices.rst similarity index 51% rename from specification/50_appendices.rst rename to specification/appendices.rst index 295c8f695..286827769 100644 --- a/specification/50_appendices.rst +++ b/specification/appendices.rst @@ -24,7 +24,7 @@ Threat: Unrecoverable Consistency Violations ++++++++++++++++++++++++++++++++++++++++++++ An attacker could send messages which created an unrecoverable "split-brain" -state in the cluster such that the victim's servers could no longer dervive a +state in the cluster such that the victim's servers could no longer derive a consistent view of the chatroom state. Threat: Bad History @@ -63,7 +63,7 @@ Spoofing An attacker could try to send a message claiming to be from the victim without the victim having sent the message in order to: -* Impersonate the victim while performing illict activity. +* Impersonate the victim while performing illicit activity. * Obtain privileges of the victim. Threat: Altering Message Contents @@ -81,7 +81,7 @@ with a phony "origin" field. Spamming ~~~~~~~~ -The attacker could try to send a high volume of solicicted or unsolicted +The attacker could try to send a high volume of solicited or unsolicited messages to the victim in order to: * Find victims for scams. @@ -129,10 +129,159 @@ An attacker could take control of a server within a chatroom to expose message contents or metadata for messages in that room. -Identity Servers -================ -.. NOTE:: - This section is a work in progress. +Cryptographic Test Vectors +-------------------------- -.. TODO-doc Dave - - 3PIDs and identity server, functions +To assist in the development of compatible implementations, the following test +values may be useful for verifying the cryptographic event signing code. + +Signing Key +~~~~~~~~~~~ + +The following test vectors all use the 32-byte value given by the following +Base64-encoded string as the seed for generating the ``ed25519`` signing key: + +.. code:: + + SIGNING_KEY_SEED = decode_base64( + "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" + ) + +In each case, the server name and key ID are as follows: + +.. code:: + + SERVER_NAME = "domain" + + KEY_ID = "ed25519:1" + +JSON Signing +~~~~~~~~~~~~ + +Given an empty JSON object: + +.. code:: json + + {} + +The JSON signing algorithm should emit the following signed data: + +.. code:: json + + { + "signatures": { + "domain": { + "ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ" + } + } + } + +Given the following JSON object with data values in it: + +.. code:: json + + { + "one": 1, + "two": "Two" + } + +The JSON signing algorithm should emit the following signed JSON: + +.. code:: json + + { + "one": 1, + "signatures": { + "domain": { + "ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw" + } + }, + "two": "Two" + } + +Event Signing +~~~~~~~~~~~~~ + +Given the following minimally-sized event: + +.. code:: json + + { + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": {}, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + +.. code:: json + + { + "event_id": "$0:domain", + "hashes": { + "sha256": "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": { + "domain": { + "ed25519:1": "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA" + } + }, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } + } + +Given the following event containing redactable content: + +.. code:: json + + { + "content": { + "body": "Here is the message content", + }, + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": {}, + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + +.. code:: json + + { + "content": { + "body": "Here is the message content", + }, + "event_id": "$0:domain", + "hashes": { + "sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": { + "domain": { + "ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" + } + }, + "unsigned": { + "age_ts": 1000000 + } + } diff --git a/specification/25_application_service_api.rst b/specification/application_service_api.rst similarity index 53% rename from specification/25_application_service_api.rst rename to specification/application_service_api.rst index 871849504..cf2f9d57f 100644 --- a/specification/25_application_service_api.rst +++ b/specification/application_service_api.rst @@ -4,26 +4,26 @@ Application Service API The Matrix client-server API and server-server APIs provide the means to implement a consistent self-contained federated messaging fabric. However, they provide limited means of implementing custom server-side behaviour in Matrix -(e.g. gateways, filters, extensible hooks etc). - -The Application Service API defines a standard API to allow such extensible -functionality to be implemented irrespective of the underlying homeserver -implementation. +(e.g. gateways, filters, extensible hooks etc). The Application Service API (AS API) +defines a standard API to allow such extensible functionality to be implemented +irrespective of the underlying homeserver implementation. .. TODO-spec Add in Client-Server services? Overview of bots? Seems weird to be in the spec given it is VERY implementation specific. -Passive Application Services ----------------------------- -"Passive" application services can only observe events from a given home server. +Application Services +-------------------- +Application services are passive and can only observe events from a given +homeserver. They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of -the event being sent. +the event being sent. In order to observe events from a homeserver, the +homeserver needs to be configured to pass certain types of traffic to the +application service. This is achieved by manually configuring the homeserver +with information about the application service (AS). -In order to observe events from a homeserver, the homeserver needs to be -configured to pass certain types of traffic to the application service. This -is achieved by manually configuring the homeserver with information about the -AS.. +Registration +~~~~~~~~~~~~ .. NOTE:: Previously, application services could register with a homeserver via HTTP @@ -41,7 +41,30 @@ AS.. A better solution would be to somehow mandate that the API done to avoid abuse. -An example HS configuration required to pass traffic to the AS is: +Application services register "namespaces" of user IDs, room aliases and room IDs. +These namespaces are represented as regular expressions. An application service +is said to be "interested" in a given event if one of the IDs in the event match +the regular expression provided by the application service. An application +service can also state whether they should be the only ones who +can manage a specified namespace. This is referred to as an "exclusive" +namespace. An exclusive namespace prevents humans and other application +services from creating/deleting entities in that namespace. Typically, +exclusive namespaces are used when the rooms represent real rooms on +another service (e.g. IRC). Non-exclusive namespaces are used when the +application service is merely augmenting the room itself (e.g. providing +logging or searching facilities). Namespaces are represented by POSIX extended +regular expressions and look like: + +.. code-block:: yaml + + users: + - exclusive: true + regex: @irc.freenode.net_.* + + +The registration is represented by a series of key-value pairs, which this +specification will present as YAML. An example HS configuration required to pass +traffic to the AS is: .. code-block:: yaml @@ -57,135 +80,20 @@ An example HS configuration required to pass traffic to the AS is: aliases: [] # Namespaces of room aliases which should be delegated to the AS rooms: [] # Namespaces of room ids which should be delegated to the AS -- An application service can state whether they should be the only ones who - can manage a specified namespace. This is referred to as an "exclusive" - namespace. An exclusive namespace prevents humans and other application - services from creating/deleting entities in that namespace. Typically, - exclusive namespaces are used when the rooms represent real rooms on - another service (e.g. IRC). Non-exclusive namespaces are used when the - application service is merely augmenting the room itself (e.g. providing - logging or searching facilities). -- Namespaces are represented by POSIX extended regular expressions, - e.g.: - -.. code-block:: yaml - - users: - - exclusive: true - - regex: @irc.freenode.net/.* +.. WARNING:: + If the homeserver in question has multiple application services, each + ``as_token`` MUST be unique per application service as this token is used to + identify the application service. The homeserver MUST enforce this. Home Server -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This contains application service APIs which are used by the home server. All -application services MUST implement these APIs. - -User Query -++++++++++ - -This API is called by the HS to query the existence of a user on the Application -Service's namespace. - -Inputs: - - User ID - - HS Credentials -Output: - - Whether the user exists. -Side effects: - - User is created on the HS by the AS via CS APIs during the processing of this request. -API called when: - - HS receives an event for an unknown user ID in the AS's namespace, e.g. an - invite event to a room. -Notes: - - When the AS receives this request, if the user exists, it must create the user via - the CS API. - - It can also set arbitrary information about the user (e.g. display name, join rooms, etc) - using the CS API. - - When this setup is complete, the AS should respond to the HS request. This means the AS - blocks the HS until the user is created. - - This is deemed more flexible than alternative methods (e.g. returning a JSON blob with the - user's display name and get the HS to provision the user). -Retry notes: - - The home server cannot respond to the client's request until the response to - this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a - 408 Request Timeout to the client. - -:: - - GET /users/$user_id?access_token=$hs_token - - Returns: - 200 : User is recognised. - 404 : User not found. - 401 : Credentials need to be supplied. - 403 : HS credentials rejected. - - - 200 OK response format - - {} - -Room Alias Query -++++++++++++++++ -This API is called by the HS to query the existence of a room alias on the -Application Service's namespace. - -Inputs: - - Room alias - - HS Credentials -Output: - - Whether the room exists. -Side effects: - - Room is created on the HS by the AS via CS APIs during the processing of - this request. -API called when: - - HS receives an event to join a room alias in the AS's namespace. -Notes: - - When the AS receives this request, if the room exists, it must create the room via - the CS API. - - It can also set arbitrary information about the room (e.g. name, topic, etc) - using the CS API. - - It can send messages as other users in order to populate scrollback. - - When this setup is complete, the AS should respond to the HS request. This means the AS - blocks the HS until the room is created and configured. - - This is deemed more flexible than alternative methods (e.g. returning an initial sync - style JSON blob and get the HS to provision the room). It also means that the AS knows - the room ID -> alias mapping. -Retry notes: - - The home server cannot respond to the client's request until the response to - this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a - 408 Request Timeout to the client. - -:: - - GET /rooms/$room_alias?access_token=$hs_token - - Returns: - 200 : Room is recognised. - 404 : Room not found. - 401 : Credentials need to be supplied. - 403 : HS credentials rejected. - - - 200 OK response format - - {} -Pushing -+++++++ -This API is called by the HS when the HS wants to push an event (or batch of -events) to the AS. - -Inputs: - - HS Credentials - - Event(s) to give to the AS - - HS-generated transaction ID -Output: - - None. +Pushing events +++++++++++++++ -Data flows: +The application service API provides a transaction API for sending a list of +events. Each list of events includes a transaction ID, which works as follows: :: @@ -199,40 +107,53 @@ Data flows: HS ---> AS : Home server retries with the same transaction ID of T. <--- : AS sends back 200 OK. If the AS had processed these events already, it can NO-OP this request (and it knows if it is the same - events based on the transacton ID). - - -Retry notes: - - If the HS fails to pass on the events to the AS, it must retry the request. - - Since ASes by definition cannot alter the traffic being passed to it (unlike - say, a Policy Server), these requests can be done in parallel to general HS - processing; the HS doesn't need to block whilst doing this. - - Home servers should use exponential backoff as their retry algorithm. - - Home servers MUST NOT alter (e.g. add more) events they were going to - send within that transaction ID on retries, as the AS may have already - processed the events. - -Ordering notes: - - The events sent to the AS should be linearised, as they are from the event - stream. - - The home server will need to maintain a queue of transactions to send to - the AS. + events based on the transaction ID). + +The events sent to the application service should be linearised, as if they were +from the event stream. The homeserver MUST maintain a queue of transactions to +send to the AS. If the application service cannot be reached, the homeserver +SHOULD backoff exponentially until the application service is reachable again. +As application services cannot *modify* the events in any way, these requests can +be made without blocking other aspects of the homeserver. Homeservers MUST NOT +alter (e.g. add more) events they were going to send within that transaction ID +on retries, as the AS may have already processed the events. + +Querying +++++++++ -:: +The application service API includes two querying APIs: for room aliases and for +user IDs. The application service SHOULD create the queried entity if it desires. +During this process, the application service is blocking the homeserver until the +entity is created and configured. If the homeserver does not receive a response +to this request, the homeserver should retry several times before timing out. This +should result in an HTTP status 408 "Request Timeout" on the client which initiated +this request (e.g. to join a room alias). - PUT /transactions/$transaction_id?access_token=$hs_token - - Request format - { - events: [ - ... - ] - } +.. admonition:: Rationale + + Blocking the homeserver and expecting the application service to create the entity + using the client-server API is simpler and more flexible than alternative methods + such as returning an initial sync style JSON blob and get the HS to provision + the room/user. This also meant that there didn't need to be a "backchannel" to inform + the application service about information about the entity such as room ID to + room alias mappings. + + +HTTP APIs ++++++++++ + +This contains application service APIs which are used by the home server. All +application services MUST implement these APIs. These APIs are defined below. + +{{application_service_http_api}} + + +.. _create the user: `sect:asapi-permissions`_ Client-Server v2 API Extensions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Passive application services can utilise a more powerful version of the +Application services can utilise a more powerful version of the client-server API by identifying itself as an application service to the home server. @@ -247,11 +168,8 @@ additional permissions granting the AS permission to masquerade as a matrix user Inputs: - Application service token (``access_token``) + - User ID in the AS namespace to act as. - Either: - - User ID in the AS namespace to act as. - Or: - - OAuth2 token of real user (which may end up being an access token) Notes: - This will apply on all aspects of the CS API, except for Account Management. - The ``as_token`` is inserted into ``access_token`` which is usually where the @@ -266,12 +184,6 @@ Notes: access_token: The application service token user_id: The desired user ID to act as. - /path?access_token=$token&user_token=$token - - Query Parameters: - access_token: The application service token - user_token: The token granted to the AS by the real user - Timestamp massaging +++++++++++++++++++ The application service may want to inject events at a certain time (reflecting @@ -294,6 +206,9 @@ Notes: Server admin style permissions ++++++++++++++++++++++++++++++ + +.. _sect:asapi-permissions: + The home server needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as @@ -320,13 +235,13 @@ including the AS token on a ``/register`` request, along with a login type of Application services which attempt to create users or aliases *outside* of their defined namespaces will receive an error code ``M_EXCLUSIVE``. Similarly, -normal users who attempt to create users or alises *inside* an application +normal users who attempt to create users or aliases *inside* an application service-defined namespace will receive the same ``M_EXCLUSIVE`` error code, but only if the application service has defined the namespace as ``exclusive``. ID conventions ~~~~~~~~~~~~~~ -.. NOTE:: +.. TODO-spec - Giving HSes the freedom to namespace still feels like the Right Thing here. - Exposing a public API provides the consistency which was the main complaint against namespacing. @@ -345,7 +260,7 @@ types, including: - MSISDNs (``tel``) - Email addresses (``mailto``) - IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04) -- XMPP (xep-0032) +- XMPP (XEP-0032) - SIP URIs (RFC 3261) As a result, virtual user IDs SHOULD relate to their URI counterpart. This @@ -373,9 +288,10 @@ an API is exposed. Room Aliases ++++++++++++ We may want to expose some 3P network rooms so Matrix users can join them directly, -e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. mailto, -tel. Rooms which are publicly accessible (e.g. IRC rooms) can be exposed as an alias by -the application service. Private rooms (e.g. sending an email to someone) should not +e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. +``mailto``, ``tel``. Rooms which are publicly accessible (e.g. IRC rooms) can be +exposed as an alias by the application service. Private rooms +(e.g. sending an email to someone) should not be exposed in this way, but should instead operate using normal invite/join semantics. Therefore, the ID conventions discussed below are only valid for public rooms which expose room aliases. @@ -395,27 +311,9 @@ SHOULD be mapped in the same way as "user" URIs. Event fields ++++++++++++ -We recommend that any gatewayed events should include an ``external_url`` field -in their content to provide a way for Matrix clients to link into the 'native' -client from which the event originated. For instance, this could contain the -message-ID for emails/nntp posts, or a link to a blog comment when gatewaying -blog comment traffic in & out of matrix - -Active Application Services ----------------------------- - -.. TODO-spec - API that provides hooks into the server so that you can intercept and - manipulate events, and/or insert virtual users & rooms into the server. - -Policy Servers -============== -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - We should mention them in the Architecture section at least: how they fit - into the picture. +We recommend that any events that originated from a remote network should +include an ``external_url`` field in their content to provide a way for Matrix +clients to link into the 'native' client from which the event originated. +For instance, this could contain the message-ID for emails/nntp posts, or a link +to a blog comment when bridging blog comment traffic in & out of Matrix. -Enforcing policies ------------------- diff --git a/specification/10_client_server_api.rst b/specification/client_server_api.rst similarity index 61% rename from specification/10_client_server_api.rst rename to specification/client_server_api.rst index df1bb9a60..b02dbf28c 100644 --- a/specification/10_client_server_api.rst +++ b/specification/client_server_api.rst @@ -1,18 +1,15 @@ -Client-Server API v1 -==================== - -Overview --------- +Client-Server API +================= The client-server API provides a simple lightweight API to let clients send messages, control rooms and synchronise conversation history. It is designed to support both lightweight clients which store no state and lazy-load data from the server as required - as well as heavyweight clients which maintain a full -local peristent copy of server state. +local persistent copy of server state. This mostly describes v1 of the Client-Server API as featured in the original September 2014 launch of Matrix, apart from user-interactive authentication where it is -encouraged to move to V2, therefore this is the version documented here. +encouraged to move to v2, therefore this is the version documented here. Version 2 is currently in development (as of Jan-March 2015) as an incremental but backwards-incompatible refinement of Version 1 and will be released shortly. @@ -31,7 +28,10 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or ``M_UNKNOWN_TOKEN`` respectively. User-Interactive Authentication API ------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _sect:auth-api: + This section refers to API Version 2. Some API endpoints such as ``login`` or ``register`` require authentication that @@ -59,7 +59,7 @@ authentication. A request to an endpoint that uses User-Interactive Authentication never succeeds without auth. Home Servers may allow requests that don't require auth by offering a stage with only the ``m.login.dummy`` auth type. The home server returns a response with HTTP status 401 and a JSON object -as folows:: +as follows:: { "flows": [ @@ -154,12 +154,12 @@ Matrix client, for example, an email confirmation may be completed when the user clicks on the link in the email. In this case, the client retries the request with an auth dict containing only the session key. The response to this will be the same as if the client were attempting to complete an auth state normally, -ie. the request will either complete or request auth, with the presence or +i.e. the request will either complete or request auth, with the presence or absence of that login stage type in the 'completed' array indicating whether that stage is complete. Example -~~~~~~~ ++++++++ At a high level, the requests made for an API call completing an auth flow with three stages will resemble the following diagram:: @@ -197,14 +197,15 @@ This specification defines the following login types: - ``m.login.recaptcha`` - ``m.login.oauth2`` - ``m.login.email.identity`` + - ``m.login.token`` - ``m.login.dummy`` Password-based -~~~~~~~~~~~~~~ +++++++++++++++ :Type: ``m.login.password`` :Description: - The client submits a username and secret password, both sent in plaintext. + The client submits a username and secret password, both sent in plain-text. To respond to this type, reply with an auth dict as follows:: @@ -214,8 +215,14 @@ To respond to this type, reply with an auth dict as follows:: "password": "" } +.. WARNING:: + Clients SHOULD enforce that the password provided is suitably complex. The + password SHOULD include a lower-case letter, an upper-case letter, a number + and a symbol and be at a minimum 8 characters in length. Servers MAY reject + weak passwords with an error code ``M_WEAK_PASSWORD``. + Google ReCaptcha -~~~~~~~~~~~~~~~~ +++++++++++++++++ :Type: ``m.login.recaptcha`` :Description: @@ -228,8 +235,39 @@ To respond to this type, reply with an auth dict as follows:: "response": "" } +Token-based ++++++++++++ +:Type: + ``m.login.token`` +:Description: + The client submits a username and token. + +To respond to this type, reply with an auth dict as follows:: + + { + "type": "m.login.token", + "user": "", + "token": "", + "txn_id": "" + } + +The ``nonce`` should be a random string generated by the client for the +request. The same ``nonce`` should be used if retrying the request. + +There are many ways a client may receive a ``token``, including via an email or +from an existing logged in device. + +The ``txn_id`` may be used by the server to disallow other devices from using +the token, thus providing "single use" tokens while still allowing the device +to retry the request. This would be done by tying the token to the ``txn_id`` +server side, as well as potentially invalidating the token completely once the +device has successfully logged in (e.g. when we receive a request from the +newly provisioned access_token). + +The ``token`` must be a macaroon. + OAuth2-based -~~~~~~~~~~~~ +++++++++++++ :Type: ``m.login.oauth2`` :Description: @@ -247,13 +285,13 @@ service which the home server accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the -OAuth2 Allow/Deny prompt. Hitting 'Allow' returns the [XXX: redirects to the?]``redirect URI`` with the -auth code. Home servers can choose any path for the ``redirect URI``. Once the -OAuth flow has completed, the client retries the request with the session only, -as above. +OAuth2 Allow/Deny prompt. Hitting 'Allow' redirects to the ``redirect URI`` with +the auth code. Home servers can choose any path for the ``redirect URI``. Once +the OAuth flow has completed, the client retries the request with the session +only, as above. Email-based (identity server) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++++++++++++++++++++++++++++++ :Type: ``m.login.email.identity`` :Description: @@ -278,7 +316,7 @@ To respond to this type, reply with an auth dict as follows:: } Dummy Auth -~~~~~~~~~~ +++++++++++ :Type: ``m.login.dummy`` :Description: @@ -295,7 +333,7 @@ if provided:: Fallback -~~~~~~~~ +++++++++ Clients cannot be expected to be able to know how to process every single login type. If a client does not know how to handle a given login type, it can direct the user to a web browser with the URL of a fallback page which will allow the @@ -308,19 +346,132 @@ Where ``stage type`` is the type name of the stage it is attempting and ``session id`` is the ID of the session given by the home server. This MUST return an HTML page which can perform this authentication stage. This -page must attempt to call the Javascript function ``window.onAuthDone`` when +page must attempt to call the JavaScript function ``window.onAuthDone`` when the authentication has been completed. +API calls using the User-Interactive Authentication mechanism +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This section refers to API Version 2. These API calls currently use the prefix +``/_matrix/client/v2_alpha``. + +.. _User-Interactive Authentication: `sect:auth-api`_ + +{{v2_registration_http_api}} + +Old V1 API docs: |register|_ + +{{login_http_api}} + +Login Fallback +++++++++++++++ + +If a client does not recognize any or all login flows it can use the fallback +login API:: + + GET /_matrix/static/client/login/ + +This returns an HTML and JavaScript page which can perform the entire login +process. The page will attempt to call the JavaScript function +``window.onLogin`` when login has been successfully completed. + +Changing Password ++++++++++++++++++ +Request:: + + POST $V2PREFIX/account/password + +This API endpoint uses the User-Interactive Authentication API. An access token +should be submitted to this endpoint if the client has an active session. The +Home Server may change the flows available depending on whether a valid access +token is provided. + +The body of the POST request is a JSON object containing: + +new_password + The new password for the account. + +On success, an empty JSON object is returned. + +The error code M_NOT_FOUND is returned if the user authenticated with a third +party identifier but the Home Server could not find a matching account in its +database. + +Adding Account Administrative Contact Information ++++++++++++++++++++++++++++++++++++++++++++++++++ +Request:: + + POST $V2PREFIX/account/3pid + +Used to add contact information to the user's account. + +The body of the POST request is a JSON object containing: + +threePidCreds + An object containing contact information. +bind + Optional. A boolean indicating whether the Home Server should also bind this + third party identifier to the account's matrix ID with the Identity Server. If + supplied and true, the Home Server must bind the 3pid accordingly. + +The contact information object comprises: + +id_server + The colon-separated hostname and port of the Identity Server used to + authenticate the third party identifier. If the port is the default, it and the + colon should be omitted. +sid + The session ID given by the Identity Server +client_secret + The client secret used in the session with the Identity Server. + +On success, the empty JSON object is returned. + +May also return error codes: + +M_THREEPID_AUTH_FAILED + If the credentials provided could not be verified with the ID Server. + +Fetching Currently Associated Contact Information ++++++++++++++++++++++++++++++++++++++++++++++++++ +Request:: + + GET $V2PREFIX/account/3pid + +This returns a list of third party identifiers that the Home Server has +associated with the user's account. This is *not* the same as the list of third +party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers +in this list may be used by the Home Server as, for example, identifiers that it +will accept to reset the user's account password. + +Returns a JSON object with the key ``threepids`` whose contents is an array of +objects with the following keys: + +medium + The medium of the 3pid (eg, ``email``) +address + The textual address of the 3pid, eg. the email address + Pagination ---------- -Querying large datasets in Matrix always uses the same pagination API pattern to +.. NOTE:: + The paths referred to in this section are not actual endpoints. They only + serve as examples to explain how pagination functions. + +Pagination is the process of dividing a dataset into multiple discrete pages. +Matrix makes use of pagination to allow clients to view extremely large datasets. +These datasets are not limited to events in a room (for example clients may want +to paginate a list of rooms in addition to events within those rooms). Regardless +of *what* is being paginated, there is a common underlying API which is used to to give clients a consistent way of selecting subsets of a potentially changing -dataset. Requests pass in ``from``, ``to`` and ``limit`` parameters which describe -where to read from the stream. ``from`` and ``to`` are opaque textual 'stream -tokens' which describe positions in the dataset. The response returns new -``start`` and ``end`` stream token values which can then be passed to subsequent -requests to continue pagination. +dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters +which describe where to read from the stream. ``from`` and ``to`` are opaque +textual 'stream tokens' which describe the current position in the dataset. +The ``dir`` parameter is an enum representing the direction of events to return: +either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and +``end`` stream token values which can then be passed to subsequent requests to +continue pagination. Not all endpoints will make use of all the parameters +outlined here: see the specific endpoint in question for more information. Pagination Request Query Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -335,24 +486,26 @@ Query parameters: limit: integer - An integer representing the maximum number of items to return. + dir: + f|b - The direction to return events in. Typically this is ``b`` to paginate + backwards in time. 'START' and 'END' are placeholder values used in these examples to describe the start and end of the dataset respectively. -Unless specified, the default pagination parameters are from=START, to=END, -without a limit set. This allows you to hit an API like -/events without any query parameters to get everything. +Unless specified, the default pagination parameters are ``from=START``, +``to=END``, without a limit set. -For example, the event stream has events E1 -> E15. The client wants the last 5 +For example, if an endpoint had events E1 -> E15. The client wants the last 5 events and doesn't know any previous events:: S E |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| | | | - | _____| | - |__________________ | ___________________| - | | | - GET /events?to=START&limit=5&from=END + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END Returns: E15,E14,E13,E12,E11 @@ -369,11 +522,11 @@ now show page 3 (rooms R11 -> 15):: Currently | viewing | | - GET /rooms/list?from=9&to=END&limit=5 + GET /roomslist?from=9&to=END&limit=5 Returns: R11,R12,R13,R14,R15 Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the intial request was '9' which corresponded to R10. When the 2nd +token from the initial request was '9' which corresponded to R10. When the 2nd request was made, R10 did not appear again, even though from=9 was specified. If you know the token, you already have the data. @@ -395,8 +548,7 @@ the complete dataset is provided in "chunk". Events ------ -Overview -~~~~~~~~ +.. _sect:events: The model of conversation history exposed by the client-server API can be considered as a list of events. The server 'linearises' the @@ -425,9 +577,9 @@ You can visualise the range of events being returned as:: | | start: '1-2-3' end: 'a-b-c' -Now, to receive future events in realtime on the eventstream, you simply GET +Now, to receive future events in real-time on the event stream, you simply GET $PREFIX/events with a ``from`` parameter of 'a-b-c': in other words passing in the -``end`` token returned by initialsync. The request blocks until new events are +``end`` token returned by initial sync. The request blocks until new events are available or until your specified timeout elapses, and then returns a new paginatable chunk of events alongside new start and end parameters:: @@ -455,70 +607,87 @@ To continue paginating backwards, one calls the /messages API again, supplying the new ``start`` value as the ``from`` parameter. -Receiving live updates on a client -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Types of room events +~~~~~~~~~~~~~~~~~~~~ -Clients receive new events by long-polling the home server via the -$PREFIX/events API, specifying a timeout in milliseconds in the timeout -parameter. This will hold open the HTTP connection for a short period of time -waiting for new events, returning early if an event occurs. This is called the -`Event Stream`_. All events which are visible to the client will appear in the -event stream. When the request returns, an ``end`` token is included in the -response. This token can be used in the next request to continue where the -last request left off. +Room events are split into two categories: -All events must be deduplicated based on their event ID. +:State Events: + These are events which update the metadata state of the room (e.g. room topic, + room membership etc). State is keyed by a tuple of event ``type`` and a + ``state_key``. State in the room with the same key-tuple will be overwritten. -.. TODO - is deduplication actually a hard requirement in CS v2? +:Message events: + These are events which describe transient "once-off" activity in a room: + typically communication such as sending an instant message or setting up a + VoIP call. + +This specification outlines several events, all with the event type prefix +``m.``. (See `Room Events`_ for the m. event specification.) However, +applications may wish to add their own type of event, and this can be achieved +using the REST API detailed in the following sections. If new events are added, +the event ``type`` key SHOULD follow the Java package naming convention, +e.g. ``com.example.myapp.event``. This ensures event types are suitably +namespaced for each application and reduces the risk of clashes. + + +Syncing +~~~~~~~ + +Clients receive new events by "long-polling" the home server via the events API. +This involves specifying a timeout in the request which will hold +open the HTTP connection for a short period of time waiting for new events, +returning early if an event occurs. Only the events API supports long-polling. +All events which are visible to the client will appear in the +events API. When the request returns, an ``end`` token is included in the +response. This token can be used in the next request to continue where the +last request left off. Multiple events can be returned per long-poll. + +.. Warning:: + Events are ordered in this API according to the arrival time of the event on + the homeserver. This can conflict with other APIs which order events based on + their partial ordering in the event graph. This can result in duplicate events + being received (once per distinct API called). Clients SHOULD de-duplicate + events based on the event ID when this happens. .. TODO-spec - Do we ever return multiple events in a single request? - Don't we get lots of request setup RTT latency if we only do one event per request? Do we ever support streaming requests? Why not websockets? When the client first logs in, they will need to initially synchronise with -their home server. This is achieved via the |initialSync|_ API. This API also -returns an ``end`` token which can be used with the event stream. See the 'Room Sync' section below. +their home server. This is achieved via the initial sync API described below. +This API also returns an ``end`` token which can be used with the event stream. -Events in a room -~~~~~~~~~~~~~~~~ +{{sync_http_api}} -Room events are split into two categories: +{{v2_sync_http_api}} -:State Events: - These are events which update the metadata state of the room (e.g. room topic, - room membership etc). State is keyed by a tuple of event ``type`` and a - ``state_key``. State in the room with the same key-tuple will be overwritten. -:Message events: - These are events which describe transient "once-off" activity in a room: - typically communication such as sending an instant messaage or setting up a - VoIP call. These used to be called 'non-state' events. +Getting events for a room +~~~~~~~~~~~~~~~~~~~~~~~~~ -This specification outlines several events, all with the event type prefix -``m.``. However, applications may wish to add their own type of event, and this -can be achieved using the REST API detailed in the following sections. If new -events are added, the event ``type`` key SHOULD follow the Java package naming -convention, e.g. ``com.example.myapp.event``. This ensures event types are -suitably namespaced for each application and reduces the risk of clashes. +There are several APIs provided to ``GET`` events for a room: -State events -~~~~~~~~~~~~ +{{rooms_http_api}} -State events can be sent by ``PUT`` ing to -|/rooms//state//|_. These events will be -overwritten if ````, ```` and ```` all match. -If the state event has no ``state_key``, it can be omitted from the path. These -requests **cannot use transaction IDs** like other ``PUT`` paths because they -cannot be differentiated from the ``state_key``. Furthermore, ``POST`` is -unsupported on state paths. Valid requests look like:: - PUT /rooms/!roomid:domain/state/m.example.event - { "key" : "without a state key" } +{{message_pagination_http_api}} - PUT /rooms/!roomid:domain/state/m.another.example.event/foo - { "key" : "with 'foo' as the state key" } + +Sending events to a room +~~~~~~~~~~~~~~~~~~~~~~~~ + +{{room_state_http_api}} + + +**Examples** + +Valid requests look like:: + + PUT /rooms/!roomid:domain/state/m.example.event + { "key" : "without a state key" } + + PUT /rooms/!roomid:domain/state/m.another.example.event/foo + { "key" : "with 'foo' as the state key" } In contrast, these requests are invalid:: @@ -544,136 +713,8 @@ In some cases, there may be no need for a ``state_key``, so it can be omitted:: PUT /rooms/!roomid:domain/state/m.room.bgd.color { "color": "red", "hex": "#ff0000" } -See `Room Events`_ for the ``m.`` event specification. - -Message events -~~~~~~~~~~~~~~ - -Message events can be sent by sending a request to -|/rooms//send/|_. These requests *can* use transaction -IDs and ``PUT``/``POST`` methods. Message events allow access to historical -events and pagination, making it best suited for sending messages. For -example:: - - POST /rooms/!roomid:domain/send/m.custom.example.message - { "text": "Hello world!" } - - PUT /rooms/!roomid:domain/send/m.custom.example.message/11 - { "text": "Goodbye world!" } - -See `Room Events`_ for the ``m.`` event specification. - -Syncing rooms -~~~~~~~~~~~~~ - -.. NOTE:: - This section is a work in progress. - -When a client logs in, they may have a list of rooms which they have already -joined. These rooms may also have a list of events associated with them. The -purpose of 'syncing' is to present the current room and event information in a -convenient, compact manner. The events returned are not limited to room events; -presence events will also be returned. A single syncing API is provided: - - - |initialSync|_ : A global sync which will present room and event information - for all rooms the user has joined. - -.. TODO-spec room-scoped initial sync - - |/rooms//initialSync|_ : A sync scoped to a single room. Presents - room and event information for this room only. - - Room-scoped initial sync is Very Tricky because typically people would - want to sync the room then listen for any new content from that point - onwards. The event stream cannot do this for a single room currently. - As a result, commenting room-scoped initial sync at this time. - -The |initialSync|_ API contains the following keys: - -``presence`` - Description: - Contains a list of presence information for users the client is interested - in. - Format: - A JSON array of ``m.presence`` events. - -``end`` - Description: - Contains an event stream token which can be used with the `Event Stream`_. - Format: - A string containing the event stream token. - -``rooms`` - Description: - Contains a list of room information for all rooms the client has joined, - and limited room information on rooms the client has been invited to. - Format: - A JSON array containing Room Information JSON objects. - -Room Information: - Description: - Contains all state events for the room, along with a limited amount of - the most recent events, configured via the ``limit`` query - parameter. Also contains additional keys with room metadata, such as the - ``room_id`` and the client's ``membership`` to the room. - Format: - A JSON object with the following keys: - ``room_id`` - A string containing the ID of the room being described. - ``membership`` - A string representing the client's membership status in this room. - ``messages`` - An event stream JSON object containing a ``chunk`` of recent - events (both state events and non-state events), along with an ``end`` token. - ``state`` - A JSON array containing all the current state events for this room. - -Getting events for a room -~~~~~~~~~~~~~~~~~~~~~~~~~ +{{room_send_http_api}} -There are several APIs provided to ``GET`` events for a room: - -``/rooms//state//`` - Description: - Get the state event identified. - Response format: - A JSON object representing the state event **content**. - Example: - ``/rooms/!room:domain.com/state/m.room.name`` returns ``{ "name": "Room name" }`` - -|/rooms//state|_ - Description: - Get all state events for a room. - Response format: - ``[ { state event }, { state event }, ... ]`` - Example: - TODO-doc - -|/rooms//members|_ - Description: - Get all ``m.room.member`` state events. - Response format: - ``{ "start": "", "end": "", "chunk": [ { m.room.member event }, ... ] }`` - Example: - TODO-doc - -|/rooms//messages|_ - Description: - Get all events from the room's timeline. This API supports - pagination using ``from`` and ``to`` query parameters, coupled with the - ``start`` and ``end`` tokens from an |initialSync|_ API. - - Response format: - ``{ "start": "", "end": "" }`` - Example: - TODO-doc - -|/rooms//initialSync|_ - Description: - Get all relevant events for a room. This includes state events, paginated - non-state events and presence events. - Response format: - `` { TODO-doc } `` - Example: - TODO-doc Redactions ~~~~~~~~~~ @@ -682,13 +723,10 @@ to add keys that are, for example offensive or illegal. Since some events cannot be simply deleted, e.g. membership events, we instead 'redact' events. This involves removing all keys from an event that are not required by the protocol. This stripped down event is thereafter returned anytime a client or -remote server requests it. - -Events that have been redacted include a ``redacted_because`` key whose value -is the event that caused it to be redacted, which may include a reason. - -Redacting an event cannot be undone, allowing server owners to delete the -offending content from the databases. +remote server requests it. Redacting an event cannot be undone, allowing server +owners to delete the offending content from the databases. Events that have been +redacted include a ``redacted_because`` key whose value is the event that caused +it to be redacted, which may include a reason. .. TODO Currently, only room admins can redact events by sending a ``m.room.redaction`` @@ -698,120 +736,50 @@ offending content from the databases. Upon receipt of a redaction event, the server should strip off any keys not in the following list: - - ``event_id`` - - ``type`` - - ``room_id`` - - ``user_id`` - - ``state_key`` - - ``prev_state`` - - ``content`` +- ``event_id`` +- ``type`` +- ``room_id`` +- ``user_id`` +- ``state_key`` +- ``prev_state`` +- ``content`` The content object should also be stripped of all keys, unless it is one of one of the following event types: - - ``m.room.member`` allows key ``membership`` - - ``m.room.create`` allows key ``creator`` - - ``m.room.join_rules`` allows key ``join_rule`` - - ``m.room.power_levels`` allows keys ``ban``, ``events``, ``events_default``, +- ``m.room.member`` allows key ``membership`` +- ``m.room.create`` allows key ``creator`` +- ``m.room.join_rules`` allows key ``join_rule`` +- ``m.room.power_levels`` allows keys ``ban``, ``events``, ``events_default``, ``kick``, ``redact``, ``state_default``, ``users``, ``users_default``. - - ``m.room.aliases`` allows key ``aliases`` +- ``m.room.aliases`` allows key ``aliases`` .. TODO Need to update m.room.power_levels to reflect new power levels formatting -The redaction event should be added under the key ``redacted_because``. - -When a client receives a redaction event it should change the redacted event +The redaction event should be added under the key ``redacted_because``. When a +client receives a redaction event it should change the redacted event in the same way a server does. - Rooms ----- Creation ~~~~~~~~ -To create a room, a client has to use the |createRoom|_ API. There are various -options which can be set when creating a room: - -``visibility`` - Type: - String - Optional: - Yes - Value: - Either ``public`` or ``private``. - Description: - A ``public`` visibility indicates that the room will be shown in the public - room list. A ``private`` visibility will hide the room from the public room - list. Rooms default to ``private`` visibility if this key is not included. - -``room_alias_name`` - Type: - String - Optional: - Yes - Value: - The room alias localpart. - Description: - If this is included, a room alias will be created and mapped to the newly - created room. The alias will belong on the same home server which created - the room, e.g. ``!qadnasoi:domain.com >>> #room_alias_name:domain.com`` - -``name`` - Type: - String - Optional: - Yes - Value: - The ``name`` value for the ``m.room.name`` state event. - Description: - If this is included, an ``m.room.name`` event will be sent into the room to - indicate the name of the room. See `Room Events`_ for more information on - ``m.room.name``. - -``topic`` - Type: - String - Optional: - Yes - Value: - The ``topic`` value for the ``m.room.topic`` state event. - Description: - If this is included, an ``m.room.topic`` event will be sent into the room - to indicate the topic for the room. See `Room Events`_ for more information - on ``m.room.topic``. - -``invite`` - Type: - List - Optional: - Yes - Value: - A list of user ids to invite. - Description: - This will tell the server to invite everyone in the list to the newly - created room. - -Example:: - - { - "visibility": "public", - "room_alias_name": "thepub", - "name": "The Grand Duke Pub", - "topic": "All about happy hour" - } - -The home server will create a ``m.room.create`` event when the room is created, -which serves as the root of the PDU graph for this room. This event also has a +The home server will create an ``m.room.create`` event when a room is created, +which serves as the root of the event graph for this room. This event also has a ``creator`` key which contains the user ID of the room creator. It will also generate several other events in order to manage permissions in this room. This includes: - - ``m.room.power_levels`` : Sets the power levels of users and required power - levels. - - ``m.room.join_rules`` : Whether the room is "invite-only" or not. +- ``m.room.power_levels`` : Sets the power levels of users and required power + levels for various actions within the room such as sending events. +- ``m.room.join_rules`` : Whether the room is "invite-only" or not. + +See `Room Events`_ for more information on these events. To create a room, a +client has to use the the following API. -See `Room Events`_ for more information on these events. +{{create_room_http_api}} Room aliases ~~~~~~~~~~~~ @@ -863,72 +831,38 @@ Permissions Permissions for rooms are done via the concept of power levels - to do any action in a room a user must have a suitable power level. Power levels are -stored as state events in a given room. - -The power levels required for operations and the power levels for users are -defined in ``m.room.power_levels``, where both a default and specific users' -power levels can be set. - +stored as state events in a given room. The power levels required for operations +and the power levels for users are defined in ``m.room.power_levels``, where +both a default and specific users' power levels can be set. By default all users have a power level of 0, other than the room creator whose power level defaults to 100. Users can grant other users increased power levels up to their own power level. For example, user A with a power level of 50 could increase the power level of user B to a maximum of level 50. Power levels for users are tracked per-room even if the user is not present in the room. - The keys contained in ``m.room.power_levels`` determine the levels required for certain operations such as kicking, banning and sending state events. See `m.room.power_levels`_ for more information. - Joining rooms ~~~~~~~~~~~~~ -.. TODO-doc What does the home server have to do to join a user to a room? - - See SPEC-30. +Users need to be a member of a room in order to send and receive events in that +room. There are several states in which a user may be, in relation to a room: -Users need to join a room in order to send and receive events in that room. A -user can join a room by making a request to |/join/|_ with:: +- Unrelated (the user cannot send or receive events in the room) +- Invited (the user has been invited to participate in the room, but is not + yet participating) +- Joined (the user can send and receive events in the room) +- Banned (the user is not allowed to join the room) - {} +There is an exception to the requirement that a user join a room before sending +events to it: users may send an ``m.room.member`` event to a room with +``content.membership`` set to ``leave`` to reject an invitation if they have +currently been invited to a room but have not joined it. -Alternatively, a user can make a request to |/rooms//join|_ with the -same request content. This is only provided for symmetry with the other -membership APIs: ``/rooms//invite`` and ``/rooms//leave``. If -a room alias was specified, it will be automatically resolved to a room ID, -which will then be joined. The room ID that was joined will be returned in -response:: - - { - "room_id": "!roomid:domain" - } - -The membership state for the joining user can also be modified directly to be -``join`` by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "join" - } - -See the `Room events`_ section for more information on ``m.room.member``. - -After the user has joined a room, they will receive subsequent events in that -room. This room will now appear as an entry in the |initialSync|_ API. - -Some rooms enforce that a user is *invited* to a room before they can join that -room. Other rooms will allow anyone to join the room even if they have not -received an invite. - -Inviting users -~~~~~~~~~~~~~~ -.. TODO-doc Invite-join dance - - Outline invite join dance. What is it? Why is it required? How does it work? - - What does the home server have to do? - -The purpose of inviting users to a room is to notify them that the room exists -so they can choose to become a member of that room. Some rooms require that all -users who join a room are previously invited to it (an "invite-only" room). -Whether a given room is an "invite-only" room is determined by the room config -key ``m.room.join_rules``. It can have one of the following values: +Some rooms require that users be invited to it before they can join; others +allow anyone to join. Whether a given room is an "invite-only" room is +determined by the room config key ``m.room.join_rules``. It can have one of the +following values: ``public`` This room is free for anyone to join without an invite. @@ -936,26 +870,7 @@ key ``m.room.join_rules``. It can have one of the following values: ``invite`` This room can only be joined if you were invited. -Only users who have a membership state of ``join`` in a room can invite new -users to said room. The person being invited must not be in the ``join`` state -in the room. The fully-qualified user ID must be specified when inviting a -user, as the user may reside on a different home server. To invite a user, send -the following request to |/rooms//invite|_, which will manage the -entire invitation process:: - - { - "user_id": "" - } - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "invite" - } - -See the `Room events`_ section for more information on ``m.room.member``. +{{membership_http_api}} Leaving rooms ~~~~~~~~~~~~~ @@ -970,8 +885,11 @@ Leaving rooms A user can leave a room to stop receiving events for that room. A user must -have joined the room before they are eligible to leave the room. If the room is -an "invite-only" room, they will need to be re-invited before they can re-join +have been invited to or have joined the room before they are eligible to leave +the room. Leaving a room to which the user has been invited rejects the invite. + +Whether or not they actually joined the room, if the room is +an "invite-only" room they will need to be re-invited before they can re-join the room. To leave a room, a request should be made to |/rooms//leave|_ with:: @@ -985,11 +903,8 @@ directly by sending the following request to "membership": "leave" } -See the `Room events`_ section for more information on ``m.room.member``. - -Once a user has left a room, that room will no longer appear on the -|initialSync|_ API. - +See the `Room events`_ section for more information on ``m.room.member``. Once a +user has left a room, that room will no longer appear on the |initialSync|_ API. If all members in a room leave, that room becomes eligible for deletion. Banning users in a room @@ -1016,178 +931,31 @@ member's state, by making a request to "membership": "ban" } +Profiles +-------- -Registration ------------- -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Registering for a user account is done using the request:: - - POST $V2PREFIX/register - -This API endpoint uses the User-Interactive Authentication API. -This API endoint does not require an access token. - -The body of the POST request is a JSON object containing: - -username - Optional. This is the local part of the desired Matrix ID. If omitted, the - Home Server must generate a Matrix ID local part. -password - Required. The desired password for the account. -bind_email - Optional. If ``true``, the server binds the email used for authentication to - the Matrix ID with the ID Server. - -On success, this returns a JSON object with keys: - -user_id - The fully-qualified Matrix ID that has been registered. -access_token - An access token for the new account. -home_server - The hostname of the Home Server on which the account has been registered. - -This endpoint may also return the following error codes: - -M_USER_IN_USE - If the Matrix ID is already in use -M_EXCLUSIVE - If the requested Matrix ID is in the exclusive namespace of an application - service. - -Home Servers MUST perform the relevant checks and return these codes before -performing User-Interactive Authentication, although they may also return -them after authentication is completed if, for example, the requested user ID -was registered whilst the client was performing authentication. - -Old V1 API docs: |register|_ - -Login -~~~~~ -This section refers to API Version 1. - -API docs: |login|_ - -Obtaining an access token for an existing user account is done using the -request:: - - POST $PREFIX/login - -The body of the POST request is a JSON object containing: - -username - The full qualified or local part of the Matrix ID to log in with. -password - The password for the account. - -On success, this returns a JSON object with keys: - -user_id - The fully-qualified Matrix ID that has been registered. -access_token - An access token for the new account. -home_server - The hostname of the Home Server on which the account has been registered. - -Changing Password -~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/password - -This API endpoint uses the User-Interactive Authentication API. An access token -should be submitted to this endpoint if the client has an active session. The -Home Server may change the flows available depending on whether a valid access -token is provided. - -The body of the POST request is a JSON object containing: - -new_password - The new password for the account. - -On success, an empty JSON object is returned. - -The error code M_NOT_FOUND is returned if the user authenticated with a third -party identifier but the Home Server could not find a matching account in its -database. - -Adding a Third Party Identifier -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - POST $V2PREFIX/account/3pid - -Used to add a third party identifier to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing third party identifier credentials. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The third party identifier credentials object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifer. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Third Party Identifiers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This section refers to API Version 2. These API calls currently use the prefix -``/_matrix/client/v2_alpha``. - -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. - -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: - -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address +{{profile_http_api}} -Presence --------- -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them +Events on Change of Profile Information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields cause an automatic +propagation event to occur, informing likely-interested parties of the new +values. This change is conveyed using two separate mechanisms: -{{presence_http_api}} +- a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. +- a ``m.presence`` presence status update is sent, again containing the new + values of the ``displayname`` and ``avatar_url`` keys, in addition to the + required ``presence`` key containing the current presence state of the user. -Profiles --------- +Both of these should be done automatically by the home server when a user +successfully changes their display name or avatar URL fields. -{{profile_http_api}} +Additionally, when home servers emit room membership events for their own +users, they should include the display name and avatar URL fields in these +events so that clients already have these details to hand, and do not have to +perform extra round trips to query it. Security -------- @@ -1240,12 +1008,6 @@ have to wait in milliseconds before they can try again. .. |/rooms//state| replace:: ``/rooms//state`` .. _/rooms//state: /docs/api/client-server/#!/-rooms/get_state_events -.. |/rooms//send/| replace:: ``/rooms//send/`` -.. _/rooms//send/: /docs/api/client-server/#!/-rooms/send_non_state_event - -.. |/rooms//state//| replace:: ``/rooms//state//`` -.. _/rooms//state//: /docs/api/client-server/#!/-rooms/send_state_event - .. |/rooms//invite| replace:: ``/rooms//invite`` .. _/rooms//invite: /docs/api/client-server/#!/-rooms/invite diff --git a/specification/31_event_signing.rst b/specification/event_signing.rst similarity index 86% rename from specification/31_event_signing.rst rename to specification/event_signing.rst index 9351cd57d..3ad3d8a7c 100644 --- a/specification/31_event_signing.rst +++ b/specification/event_signing.rst @@ -1,8 +1,8 @@ Signing Events -============== +-------------- Canonical JSON --------------- +~~~~~~~~~~~~~~ Matrix events are represented using JSON objects. If we want to sign JSON events we need to encode the JSON as a binary string. Unfortunately the same @@ -30,7 +30,7 @@ using this representation. value, # Encode code-points outside of ASCII as UTF-8 rather than \u escapes ensure_ascii=False, - # Remove unecessary white space. + # Remove unnecessary white space. separators=(',',':'), # Sort the keys of dictionaries. sort_keys=True, @@ -38,7 +38,7 @@ using this representation. ).encode("UTF-8") Grammar -~~~~~~~ ++++++++ Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing insignificant whitespace, fractions, exponents and redundant character escapes @@ -69,14 +69,14 @@ insignificant whitespace, fractions, exponents and redundant character escapes / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X Signing JSON ------------- +~~~~~~~~~~~~ We can now sign a JSON object by encoding it as a sequence of bytes, computing the signature for that sequence and then adding the signature to the original JSON object. Signing Details -~~~~~~~~~~~~~~~ ++++++++++++++++ JSON is signed by encoding the JSON object without ``signatures`` or keys grouped as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the @@ -133,7 +133,7 @@ and additional signatures. return json_object Checking for a Signature -~~~~~~~~~~~~~~~~~~~~~~~~ +++++++++++++++++++++++++ To check if an entity has signed a JSON object a server does the following @@ -151,7 +151,7 @@ To check if an entity has signed a JSON object a server does the following the check fails. Otherwise the check succeeds. Signing Events --------------- +~~~~~~~~~~~~~~ Signing events is a more complicated process since servers can choose to redact non-essential parts of an event. Before signing the event it is encoded as @@ -190,9 +190,64 @@ in the event JSON in a ``hash`` object under a ``sha256`` key. event_json_object["unsigned"] = unsigned return event_json_object -Then all non-essential keys are stripped from the event object, and the -resulting object which included the ``hash`` key is signed using the JSON -signing algorithm +The event is then stripped of all non-essential keys both at the top level and +within the ``content`` object. Any top-level keys not in the following list +MUST be removed: + +.. code:: + + auth_events + depth + event_id + hashes + membership + origin + origin_server_ts + prev_events + prev_state + room_id + sender + signatures + state_key + type + +A new ``content`` object is constructed for the resulting event that contains +only the essential keys of the original ``content`` object. If the original +event lacked a ``content`` object at all, a new empty JSON object is created +for it. + +The keys that are considered essential for the ``content`` object depend on the +the ``type`` of the event. These are: + +.. code:: + + type is "m.room.aliases": + aliases + + type is "m.room.create": + creator + + type is "m.room.history_visibility": + history_visibility + + type is "m.room.join_rules": + join_rule + + type is "m.room.member": + membership + + type is "m.room.power_levels": + ban + events + events_default + kick + redact + state_default + users + users_default + +The resulting stripped object with the new ``content`` object and the original +``hashes`` key is then signed using the JSON signing algorithm outlined below: .. code:: python diff --git a/specification/events.rst b/specification/events.rst new file mode 100644 index 000000000..a1aece1c4 --- /dev/null +++ b/specification/events.rst @@ -0,0 +1,88 @@ +Event Structure +=============== + +All communication in Matrix is expressed in the form of data objects called +Events. These are the fundamental building blocks common to the client-server, +server-server and application-service APIs, and are described below. + +{{common_event_fields}} + +{{common_room_event_fields}} + +{{common_state_event_fields}} + + +Differences between /v1 and /v2 events +-------------------------------------- + +There are a few differences between how events are formatted for sending +between servers over federation and how they are formatted for sending between +a server and its clients. + +Additionally there are a few differences between the format of events in the +responses to client APIs with a /v1 prefix and responses APIs with a /v2 +prefix. + +Events in responses for APIs with the /v2 prefix are generated from an event +formatted for federation by: + +* Removing the following keys: + ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in + milliseconds that has ellapsed since the event was sent. +* Adding a ``prev_content`` to the ``unsigned`` object if the event is + a ``state event`` which gives previous content of that state key. +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was + redacted which gives the event that redacted it. +* Adding a ``transaction_id`` if the event was sent by the client requesting it. + +Events in responses for APIs with the /v1 prefix are generated from an event +formatted for the /v2 prefix by: + +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. +* Rename the ``sender`` key to ``user_id``. +* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` + then adding a ``invite_room_state`` key to the top level event object. + + +Size limits +----------- + +The total size of any event MUST NOT exceed 65 KB. There are additional +restrictions on sizes per key: + +- ``user_id`` MUST NOT exceed 255 bytes (including domain). +- ``room_id`` MUST NOT exceed 255 bytes. +- ``state_key`` MUST NOT exceed 255 bytes. +- ``type`` MUST NOT exceed 255 bytes. +- ``event_id`` MUST NOT exceed 255 bytes. + +Some event types have additional size restrictions which are specified in +the description of the event. Additional keys have no limit other than that +implied by the total 65 KB limit on events. + +Room Events +----------- +.. NOTE:: + This section is a work in progress. + +This specification outlines several standard event types, all of which are +prefixed with ``m.`` + +{{m_room_aliases_event}} + +{{m_room_canonical_alias_event}} + +{{m_room_create_event}} + +{{m_room_join_rules_event}} + +{{m_room_member_event}} + +{{m_room_power_levels_event}} + +{{m_room_redaction_event}} + diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst new file mode 100644 index 000000000..f79b92c51 --- /dev/null +++ b/specification/feature_profiles.rst @@ -0,0 +1,95 @@ +Feature Profiles +================ + +.. _sect:feature-profiles: + +Matrix supports many different kinds of clients: from embedded IoT devices to +desktop clients. Not all clients can provide the same feature sets as other +clients e.g. due to lack of physical hardware such as not having a screen. +Clients can fall into one of several profiles and each profile contains a set +of features that the client MUST support. This section details a set of +"feature profiles". Clients are expected to implement a profile in its entirety +in order for it to be classified as that profile. + +Summary +------- + +===================================== ========== ========== ========== ========== ========== + Module / Profile Web Mobile Desktop CLI Embedded +===================================== ========== ========== ========== ========== ========== + `Instant Messaging`_ Required Required Required Required Optional + `Presence`_ Required Required Required Required Optional + `Push Notifications`_ Optional Required Optional Optional Optional + `Receipts`_ Required Required Required Required Optional + `Typing Notifications`_ Required Required Required Required Optional + `VoIP`_ Required Required Required Optional Optional + `Content Repository`_ Required Required Required Optional Optional + `Managing History Visibility`_ Required Required Required Required Optional + `End-to-End Encryption`_ Optional Optional Optional Optional Optional + `Server Side Search`_ Optional Optional Optional Optional Optional +===================================== ========== ========== ========== ========== ========== + +*Please see each module for more details on what clients need to implement.* + +.. _End-to-End Encryption: `module:e2e`_ +.. _Instant Messaging: `module:im`_ +.. _Presence: `module:presence`_ +.. _Push Notifications: `module:push`_ +.. _Receipts: `module:receipts`_ +.. _Typing Notifications: `module:typing`_ +.. _VoIP: `module:voip`_ +.. _Content Repository: `module:content`_ +.. _Managing History Visibility: `module:history-visibility`_ +.. _Server Side Search: `module:search`_ + +Clients +------- + +Stand-alone web (``Web``) +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a web page which heavily uses Matrix for communication. Single-page web +apps would be classified as a stand-alone web client, as would multi-page web +apps which use Matrix on nearly every page. + +Mobile (``Mobile``) +~~~~~~~~~~~~~~~~~~~ + +This is a Matrix client specifically designed for consumption on mobile devices. +This is typically a mobile app but need not be so provided the feature set can +be reached (e.g. if a mobile site could display push notifications it could be +classified as a mobile client). + +Desktop (``Desktop``) +~~~~~~~~~~~~~~~~~~~~~ + +This is a native GUI application which can run in its own environment outside a +browser. + +Command Line Interface (``CLI``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a client which is used via a text-based terminal. + +Embedded (``Embedded``) +~~~~~~~~~~~~~~~~~~~~~~~ + +This is a client which is embedded into another application or an embedded +device. + +Application ++++++++++++ + +This is a Matrix client which is embedded in another website, e.g. using +iframes. These embedded clients are typically for a single purpose +related to the website in question, and are not intended to be fully-fledged +communication apps. + +Device +++++++ + +This is a client which is typically running on an embedded device such as a +kettle, fridge or car. These clients tend to perform a few operations and run +in a resource constrained environment. Like embedded applications, they are +not intended to be fully-fledged communication systems. + diff --git a/specification/identity_servers.rst b/specification/identity_servers.rst new file mode 100644 index 000000000..6ec013bd7 --- /dev/null +++ b/specification/identity_servers.rst @@ -0,0 +1,8 @@ +Identity Servers +================ +.. NOTE:: + This section is a work in progress. + +.. TODO-doc Dave + - 3PIDs and identity server, functions + diff --git a/specification/00_basis.rst b/specification/intro.rst similarity index 75% rename from specification/00_basis.rst rename to specification/intro.rst index 10709f1d4..64a211083 100644 --- a/specification/00_basis.rst +++ b/specification/intro.rst @@ -24,7 +24,6 @@ Introduction The Matrix specification is still evolving: the APIs are not yet frozen and this document is in places a work in progress or stale. We have made every effort to clearly flag areas which are still being finalised. - We're publishing it at this point because it's complete enough to be more than useful and provide a canonical reference to how Matrix is evolving. Our end goal is to mirror WHATWG's `Living Standard @@ -34,10 +33,9 @@ Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice over IP (VoIP) and Internet of Things (IoT) communication, designed to create and support a new global real-time communication ecosystem. The intention is to provide an open decentralised pubsub layer for the internet for securely -persisting and publishing/subscribing JSON objects. - -This specification is the ongoing result of standardising the APIs used by the -various components of the Matrix ecosystem to communicate with one another. +persisting and publishing/subscribing JSON objects. This specification is the +ongoing result of standardising the APIs used by the various components of the +Matrix ecosystem to communicate with one another. The principles that Matrix attempts to follow are: @@ -79,7 +77,7 @@ The functionality that Matrix provides includes: - Extensible user management (inviting, joining, leaving, kicking, banning) mediated by a power-level based user privilege system. - Extensible room state management (room naming, aliasing, topics, bans) -- Extensible user profile management (avatars, displaynames, etc) +- Extensible user profile management (avatars, display names, etc) - Managing user accounts (registration, login, logout) - Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, Facebook accounts to authenticate, identify and discover users on Matrix. @@ -91,7 +89,7 @@ The functionality that Matrix provides includes: The end goal of Matrix is to be a ubiquitous messaging layer for synchronising arbitrary data between sets of people, devices and services - be that for instant messages, VoIP call setups, or any other objects that need to be -reliably and persistently pushed from A to B in an interoperable and federated +reliably and persistently pushed from A to B in an inter-operable and federated manner. Overview @@ -101,23 +99,23 @@ Architecture ------------ Matrix defines APIs for synchronising extensible JSON objects known as -``events`` between compatible clients, servers and services. Clients are +"events" between compatible clients, servers and services. Clients are typically messaging/VoIP applications or IoT devices/hubs and communicate by -synchronising communication history with their ``homeserver`` using the -``Client-Server API``. Each homeserver stores the communication history and +synchronising communication history with their "homeserver" using the +"Client-Server API". Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the -context of a virtual ``room``. Room data is replicated across *all of the +context of a virtual "room". Room data is replicated across *all of the homeservers* whose users are participating in a given room. As such, *no single homeserver has control or ownership over a given room*. Homeservers model communication history as a partially ordered graph of events known as -the room's ``event graph``, which is synchronised with eventual consistency -between the participating servers using the ``Server-Server API``. This process +the room's "event graph", which is synchronised with eventual consistency +between the participating servers using the "Server-Server API". This process of synchronising shared conversation history between homeservers run by -different parties is called ``Federation``. Matrix optimises for the the +different parties is called "Federation". Matrix optimises for the the Availability and Partitioned properties of CAP theorem at the expense of Consistency. @@ -153,13 +151,13 @@ Users ~~~~~ Each client is associated with a user account, which is identified in Matrix -using a unique "User ID". This ID is namespaced to the home server which +using a unique "User ID". This ID is namespaced to the homeserver which allocated the account and has the form:: @localpart:domain The ``localpart`` of a user ID may be a user name, or an opaque ID identifying -this user. They are case-insensitive. +this user. The ``domain`` of a user ID is the domain of the homeserver. .. TODO-spec - Need to specify precise grammar for Matrix IDs @@ -171,20 +169,23 @@ All data exchanged over Matrix is expressed as an "event". Typically each client action (e.g. sending a message) correlates with exactly one event. Each event has a ``type`` which is used to differentiate different kinds of data. ``type`` values MUST be uniquely globally namespaced following Java's `package naming -conventions -`, e.g. +conventions`_, e.g. ``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved for events defined in the Matrix specification - for instance ``m.room.message`` is the event type for instant messages. Events are usually sent in the context of a "Room". +.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions + Event Graphs ~~~~~~~~~~~~ +.. _sect:event-graph: + Events exchanged in the context of a room are stored in a directed acyclic graph -(DAG) called an ``event graph``. The partial ordering of this graph gives the +(DAG) called an "event graph". The partial ordering of this graph gives the chronological ordering of events within the room. Each event in the graph has a -list of zero or more ``parent`` events, which refer to any preceeding events +list of zero or more "parent" events, which refer to any preceding events which have no chronological successor from the perspective of the homeserver which created the event. @@ -213,10 +214,8 @@ which have the form:: There is exactly one room ID for each room. Whilst the room ID does contain a domain, it is simply for globally namespacing room IDs. The room does NOT reside on the domain specified. Room IDs are not meant to be human readable. -They are case-sensitive. - -The following conceptual diagram shows an ``m.room.message`` event being sent to -the room ``!qporfwt:matrix.org``:: +They are case-sensitive. The following conceptual diagram shows an +``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: { @alice:matrix.org } { @bob:domain.com } | ^ @@ -257,28 +256,28 @@ the room ``!qporfwt:matrix.org``:: Federation maintains *shared data structures* per-room between multiple home servers. The data is split into ``message events`` and ``state events``. -``Message events`` describe transient 'once-off' activity in a room such as an -instant messages, VoIP call setups, file transfers, etc. They generally describe -communication activity. +Message events: + These describe transient 'once-off' activity in a room such as an + instant messages, VoIP call setups, file transfers, etc. They generally + describe communication activity. -``State events`` describe updates to a given piece of persistent information -('state') related to a room, such as the room's name, topic, membership, -participating servers, etc. State is modelled as a lookup table of key/value -pairs per room, with each key being a tuple of ``state_key`` and ``event type``. -Each state event updates the value of a given key. +State events: + These describe updates to a given piece of persistent information + ('state') related to a room, such as the room's name, topic, membership, + participating servers, etc. State is modelled as a lookup table of key/value + pairs per room, with each key being a tuple of ``state_key`` and ``event type``. + Each state event updates the value of a given key. The state of the room at a given point is calculated by considering all events preceding and including a given event in the graph. Where events describe the same state, a merge conflict algorithm is applied. The state resolution algorithm is transitive and does not depend on server state, as it must consistently select the same event irrespective of the server or the order the -events were received in. - -Events are signed by the originating server (the signature includes the parent -relations, type, depth and payload hash) and are pushed over federation to the -participating servers in a room, currently using full mesh topology. Servers may -also request backfill of events over federation from the other servers -participating in a room. +events were received in. Events are signed by the originating server (the +signature includes the parent relations, type, depth and payload hash) and are +pushed over federation to the participating servers in a room, currently using +full mesh topology. Servers may also request backfill of events over federation +from the other servers participating in a room. Room Aliases @@ -293,11 +292,12 @@ Each room can also have multiple "Room Aliases", which look like:: A room alias "points" to a room ID and is the human-readable label by which rooms are publicised and discovered. The room ID the alias is pointing to can -be obtained by visiting the domain specified. They are case-insensitive. Note -that the mapping from a room alias to a room ID is not fixed, and may change -over time to point to a different room ID. For this reason, Clients SHOULD -resolve the room alias to a room ID once and then use that ID on subsequent -requests. +be obtained by visiting the domain specified. Note that the mapping from a room +alias to a room ID is not fixed, and may change over time to point to a +different room ID. For this reason, Clients SHOULD resolve the room alias to a +room ID once and then use that ID on subsequent requests. Room aliases MUST NOT +exceed 255 bytes (including the domain). + When resolving a room alias the server will also respond with a list of servers that are in the room that can be used to join via. @@ -323,12 +323,10 @@ Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix users. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces *linked* to their account. - Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social network accounts and phone numbers to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts. - In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. @@ -337,58 +335,13 @@ Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. -Presence -~~~~~~~~ - -Each user has the concept of presence information. This encodes: - - * Whether the user is currently online - * How recently the user was last active (as seen by the server) - * Whether a given client considers the user to be currently idle - * Arbitrary information about the user's current status (e.g. "in a meeting"). - -This information is collated from both per-device (online; idle; last_active) and -per-user (status) data, aggregated by the user's homeserver and transmitted as -an ``m.presence`` event. This is one of the few events which are sent *outside -the context of a room*. Presence events are sent to all users who subscribe to -this user's presence through a presence list or by sharing membership of a room. - -.. TODO - How do we let users hide their presence information? - -.. TODO - The last_active specifics should be moved to the detailed presence event section - -Last activity is tracked by the server maintaining a timestamp of the last time -it saw a pro-active event from the user. Any event which could be triggered by a -human using the application is considered pro-active (e.g. sending an event to a -room). An example of a non-proactive client activity would be a client setting -'idle' presence status, or polling for events. This timestamp is presented via a -key called ``last_active_ago``, which gives the relative number of milliseconds -since the message is generated/emitted that the user was last seen active. - -N.B. in v1 API, status/online/idle state are muxed into a single 'presence' field on the m.presence event. - -Presence Lists -~~~~~~~~~~~~~~ - -Each user's home server stores a "presence list". This stores a list of user IDs -whose presence the user wants to follow. - -To be added to this list, the user being added must be invited by the list owner -and accept the invitation. Once accepted, both user's HSes track the -subscription. - Profiles ~~~~~~~~ Users may publish arbitrary key/value data associated with their account - such -as a human readable ``display name``, a profile photo URL, contact information -(email address, phone nubers, website URLs etc). - -In Client-Server API v2, profile data is typed using namespaced keys for -interoperability, much like events - e.g. ``m.profile.display_name``. +as a human readable display name, a profile photo URL, contact information +(email address, phone numbers, website URLs etc). .. TODO Actually specify the different types of data - e.g. what format are display @@ -408,6 +361,10 @@ dedicated API. The API is symmetrical to managing Profile data. API Standards ------------- +.. TODO + Need to specify any HMAC or access_token lifetime/ratcheting tricks + We need to specify capability negotiation for extensible transports + The mandatory baseline for communication in Matrix is exchanging JSON objects over HTTP APIs. HTTPS is mandated as the baseline for server-server (federation) communication. HTTPS is recommended for client-server @@ -415,20 +372,11 @@ communication, although HTTP may be supported as a fallback to support basic HTTP clients. More efficient optional transports for client-server communication will in future be supported as optional extensions - e.g. a packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. - -.. TODO - We need to specify capability negotiation for extensible transports - -For the default HTTP transport, all API calls use a Content-Type of -``application/json``. In addition, all strings MUST be encoded as UTF-8. - -Clients are authenticated using opaque ``access_token`` strings (see -`Client Authentication`_ for details), passed as a query string parameter on -all requests. - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks +low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all +API calls use a Content-Type of ``application/json``. In addition, all strings +MUST be encoded as UTF-8. Clients are authenticated using opaque +``access_token`` strings (see `Client Authentication`_ for details), passed as a +query string parameter on all requests. Any errors which occur at the Matrix API level MUST return a "standard error response". This is a JSON object which looks like:: @@ -442,7 +390,7 @@ The ``error`` string will be a human-readable error message, usually a sentence explaining what went wrong. The ``errcode`` string will be a unique string which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error codes should have their namespace first in ALL CAPS, followed by a single _ to -ease seperating the namespace from the error code.. For example, if there was a +ease separating the namespace from the error code. For example, if there was a custom namespace ``com.mydomain.here``, and a ``FORBIDDEN`` code, the error code should look like ``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the @@ -481,13 +429,12 @@ Some requests have unique error codes: :``M_BAD_PAGINATION``: Encountered when specifying bad pagination query parameters. -:``M_LOGIN_EMAIL_URL_NOT_YET``: - Encountered when polling for an email link which has not been clicked yet. +.. _sect:txn_ids: -The C-S API typically uses ``HTTP POST`` to submit requests. This means these -requests are not idempotent. The C-S API also allows ``HTTP PUT`` to make -requests idempotent. In order to use a ``PUT``, paths should be suffixed with -``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which +The Client-Server API typically uses ``HTTP POST`` to submit requests. This +means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to +make requests idempotent. In order to use a ``PUT``, paths should be suffixed +with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which identifies the request, and is scoped to a given Client (identified by that client's ``access_token``). Crucially, it **only** serves to identify new requests from retransmits. After the request has finished, the ``{txnId}`` diff --git a/specification/modules.rst b/specification/modules.rst new file mode 100644 index 000000000..0aad77e1c --- /dev/null +++ b/specification/modules.rst @@ -0,0 +1,3 @@ +Modules +======= + diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst new file mode 100644 index 000000000..9eee98439 --- /dev/null +++ b/specification/modules/_template.rst @@ -0,0 +1,56 @@ +Module Heading +============== + +.. _module:short-name: + +A short summary of the module. What features does this module provide? An anchor +should be specified at the top of the module using the format ``module:name``. + +Complicated modules may wish to have architecture diagrams or event flows +(e.g. VoIP call flows) here. Custom subsections can be included but they should +be used *sparingly* to reduce the risk of putting client or server behaviour +information in these custom sections. + +Events +------ +List the new event types introduced by this module, if any. If there are no +new events, this section can be omitted. Event types should be done as +subsections. This section is intended to document the "common shared event +structure" between client and server. Deviations from this shared structure +should be documented in the relevant behaviour section. + +``m.example.event.type`` +~~~~~~~~~~~~~~~~~~~~~~~~ +There should be JSON Schema docs for this event. Once there is JSON schema, +there will be a template variable with dots in the event type replaced with +underscores and the suffix ``_event``. You can insert a template like so: + +{{m_example_event_type_event}} + +Client behaviour +---------------- +List any new HTTP endpoints. These endpoints should be documented using Swagger. +Once there is Swagger, there will be a template variable based on the name of +the YAML file with the suffix ``_http_api``. You can insert a template for +swagger docs like so: + +{{name-of-yaml-file-without-file-ext_http_api}} + +List the steps the client needs to take to +correctly process this module. List what data structures the client should be +storing in order to aid implementation. + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. Federation-specific logic should be included +here. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst new file mode 100644 index 000000000..e07cf58cb --- /dev/null +++ b/specification/modules/content_repo.rst @@ -0,0 +1,87 @@ +Content repository +================== + +.. _module:content: + +This module allows users to upload content to their homeserver which is +retrievable from other homeservers. Its' purpose is to allow users to share +attachments in a room. Content locations are represented as Matrix Content (MXC) +URIs. They look like:: + + mxc:/// + + : The name of the homeserver where this content originated, e.g. matrix.org + : An opaque ID which identifies the content. + +Uploads are POSTed to a resource on the user's local homeserver which returns a +token which is used to GET the download. Content is downloaded from the +recipient's local homeserver, which must first transfer the content from the +origin homeserver using the same API (unless the origin and destination +homeservers are the same). + +Client behaviour +---------------- + +Clients can upload and download content using the following HTTP APIs. + +{{content_repo_http_api}} + +Thumbnails +~~~~~~~~~~ +The thumbnail methods are "crop" and "scale". "scale" tries to return an +image where either the width or the height is smaller than the requested +size. The client should then scale and letterbox the image if it needs to +fit within a given rectangle. "crop" tries to return an image where the +width and height are close to the requested size and the aspect matches +the requested size. The client should scale the image if it needs to fit +within a given rectangle. + +In summary: + * "scale" maintains the original aspect ratio of the image + * "crop" provides an image in the aspect ratio of the sizes given in the request + +Server behaviour +---------------- + +Homeservers may generate thumbnails for content uploaded to remote +homeservers themselves or may rely on the remote homeserver to thumbnail +the content. Homeservers may return thumbnails of a different size to that +requested. However homeservers should provide exact matches where reasonable. +Homeservers must never upscale images. + +Security considerations +----------------------- + +The HTTP GET endpoint does not require any authentication. Knowing the URL of +the content is sufficient to retrieve the content, even if the entity isn't in +the room. + +MXC URIs are vulnerable to directory traversal attacks such as +``mxc://127.0.0.1/../../../some_service/etc/passwd``. This would cause the target +homeserver to try to access and return this file. As such, homeservers MUST +sanitise MXC URIs by allowing only alphanumeric (``A-Za-z0-9``), ``_`` +and ``-`` characters in the ``server-name`` and ``media-id`` values. This set +of whitelisted characters allows URL-safe base64 encodings specified in RFC 4648. +Applying this character whitelist is preferable to blacklisting ``.`` and ``/`` +as there are techniques around blacklisted characters (percent-encoded characters, +UTF-8 encoded traversals, etc). + +Homeservers have additional content-specific concerns: + +- Clients may try to upload very large files. Homeservers should not store files + that are too large and should not serve them to clients. + +- Clients may try to upload very large images. Homeservers should not attempt to + generate thumbnails for images that are too large. + +- Remote homeservers may host very large files or images. Homeservers should not + proxy or thumbnail large files or images from remote homeservers. + +- Clients may try to upload a large number of files. Homeservers should limit the + number and total size of media that can be uploaded by clients. + +- Clients may try to access a large number of remote files through a homeserver. + Homeservers should restrict the number and size of remote files that it caches. + +- Clients or remote homeservers may try to upload malicious files targeting + vulnerabilities in either the homeserver thumbnailing or the client decoders. diff --git a/specification/41_end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst similarity index 96% rename from specification/41_end_to_end_encryption.rst rename to specification/modules/end_to_end_encryption.rst index 023881527..e3a526136 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -1,6 +1,8 @@ End-to-End Encryption ===================== +.. _module:e2e: + .. TODO-doc - Why is this needed. - Overview of process diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst new file mode 100644 index 000000000..112e2e219 --- /dev/null +++ b/specification/modules/history_visibility.rst @@ -0,0 +1,72 @@ +Room History Visibility +======================= + +.. _module:history-visibility: + +This module adds support for controlling the visibility of previous events in a +room. + +In all cases except ``world_readable``, a user needs to join a room to view events in that room. Once they +have joined a room, they will gain access to a subset of events in the room. How +this subset is chosen is controlled by the ``m.room.history_visibility`` event +outlined below. After a user has left a room, they may seen any events which they +were allowed to see before they left the room, but no events received after they +left. + +The four options for the ``m.room.history_visibility`` event are: + +- ``shared`` - Previous events are always accessible to newly joined members. All + events in the room are accessible, even those sent when the member was not a part + of the room. +- ``invited`` - Previous events are accessible to newly joined members from the point + they were invited onwards. Events stop being accessible when the member's state + changes to something other than ``invite`` or ``join``. +- ``joined`` - Previous events are accessible to newly joined members from the point + they joined the room onwards. Events stop being accessible when the member's state + changes to something other than ``join``. +- ``world_readable`` - All events while this is the ``m.room.history_visibility`` value + may be shared by any participating homeserver with anyone, regardless of whether + they have ever joined the room. + +.. WARNING:: + These options are applied at the point an event is *sent*. Checks are + performed with the state of the ``m.room.history_visibility`` event when the + event in question is added to the DAG. This means clients cannot + retrospectively choose to show or hide history to new users if the setting at + that time was more restrictive. + +Events +------ + +{{m_room_history_visibility_event}} + +Client behaviour +---------------- + +Clients that implement this module MUST present to the user the possible options +for setting history visibility when creating a room. + +Clients may want to display a notice that their events may be read by non-joined +people if the value is set to ``world_readable``. + +Server behaviour +---------------- + +By default if no ``history_visibility`` is set, or if the value is not understood, the visibility is assumed to be +``shared``. The rules governing whether a user is allowed to see an event depend +solely on the state of the room *at that event*: + +1. If the user was joined, allow. +2. If the user was invited and the ``history_visibility`` was set to + ``invited`` or ``shared``, allow. +3. If the user was neither invited nor joined but the ``history_visibility`` + was set to ``shared``, allow. +4. Otherwise, deny. + +Security considerations +----------------------- + +The default value for ``history_visibility`` is ``shared`` for +backwards-compatibility reasons. Clients need to be aware that by not setting +this event they are exposing all of their room history to anyone in the room. + diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst new file mode 100644 index 000000000..09fcb843d --- /dev/null +++ b/specification/modules/instant_messaging.rst @@ -0,0 +1,153 @@ +Instant Messaging +================= + +.. _module:im: + +This module adds support for sending human-readable messages to a room. It also +adds support for associating human-readable information with the room itself +such as a room name and topic. + +Events +------ + +{{m_room_message_event}} + +{{m_room_message_feedback_event}} + +Usage of this event is discouraged for several reasons: + - The number of feedback events will grow very quickly with the number of users + in the room. This event provides no way to "batch" feedback, unlike the + `receipts module`_. + - Pairing feedback to messages gets complicated when paginating as feedback + arrives before the message it is acknowledging. + - There are no guarantees that the client has seen the event ID being + acknowledged. + + +.. _`receipts module`: `module:receipts`_ + +{{m_room_name_event}} + +{{m_room_topic_event}} + +{{m_room_avatar_event}} + +m.room.message msgtypes +~~~~~~~~~~~~~~~~~~~~~~~ + +Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type +of message being sent. Each type has their own required and optional keys, as +outlined below. If a client cannot display the given ``msgtype`` then it SHOULD +display the fallback plain text ``body`` key instead. + +{{msgtype_events}} + + +Client behaviour +---------------- + +Clients SHOULD verify the structure of incoming events to ensure that the +expected keys exist and that they are of the right type. Clients can discard +malformed events or display a placeholder message to the user. Redacted +``m.room.message`` events MUST be removed from the client. This can either be +replaced with placeholder text (e.g. "[REDACTED]") or the redacted message can +be removed entirely from the messages view. + +Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be +uploaded using the `content repository module`_ where available. The +resulting ``mxc://`` URI can then be used in the ``url`` key. + +.. _`content repository module`: `module:content`_ + +Recommendations when sending messages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use +``PUT`` requests with `transaction IDs`_ to make requests idempotent. This +ensures that messages are sent exactly once even under poor network conditions. +Clients SHOULD retry requests using an exponential-backoff algorithm for a +certain amount of time T. It is recommended that T is no longer than 5 minutes. +After this time, the client should stop retrying and mark the message as "unsent". +Users should be able to manually resend unsent messages. + +Users may type several messages at once and send them all in quick succession. +Clients SHOULD preserve the order in which they were sent by the user. This +means that clients should wait for the response to the previous request before +sending the next request. This can lead to head-of-line blocking. In order to +reduce the impact of head-of-line blocking, clients should use a queue per room +rather than a global queue, as ordering is only relevant within a single room +rather than between rooms. + +.. _`transaction IDs`: `sect:txn_ids`_ + +Local echo +~~~~~~~~~~ + +Messages SHOULD appear immediately in the message view when a user presses the +"send" button. This should occur even if the message is still sending. This is +referred to as "local echo". Clients SHOULD implement "local echo" of messages. +Clients MAY display messages in a different format to indicate that the server +has not processed the message. This format should be removed when the server +responds. + +Clients need to be able to match the message they are sending with the same +message which they receive from the event stream. The echo of the same message +from the event stream is referred to as "remote echo". Both echoes need to be +identified as the same message in order to prevent duplicate messages being +displayed. Ideally this pairing would occur transparently to the user: the UI +would not flicker as it transitions from local to remote. Flickering cannot be +fully avoided in the current client-server API. Two scenarios need to be +considered: + +- The client sends a message and the remote echo arrives on the event stream + *after* the request to send the message completes. +- The client sends a message and the remote echo arrives on the event stream + *before* the request to send the message completes. + +In the first scenario, the client will receive an event ID when the request to +send the message completes. This ID can be used to identify the duplicate event +when it arrives on the event stream. However, in the second scenario, the event +arrives before the client has obtained an event ID. This makes it impossible to +identify it as a duplicate event. This results in the client displaying the +message twice for a fraction of a second before the the original request to send +the message completes. Once it completes, the client can take remedial actions +to remove the duplicate event by looking for duplicate event IDs. A future version +of the client-server API will resolve this by attaching the transaction ID of the +sending request to the event itself. + + +Displaying membership information with messages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show the display name and avatar URL of the room member who +sent a message. This can be achieved by inspecting the ``m.room.member`` state +event for that user ID. + +When a user paginates the message history, clients may wish to show the +**historical** display name and avatar URL for a room member. This is possible +because older ``m.room.member`` events are returned when paginating. This can +be implemented efficiently by keeping two sets of room state: old and current. +As new events arrive and/or the user paginates back in time, these two sets of +state diverge from each other. New events update the current state and paginated +events update the old state. When paginated events are processed sequentially, +the old state represents the state of the room *at the time the event was sent*. +This can then be used to set the historical display name and avatar URL. + +Server behaviour +---------------- + +Homeservers SHOULD reject ``m.room.message`` events which don't have a +``msgtype`` key, or which don't have a textual ``body`` key, with an HTTP status +code of 400. + +Security considerations +----------------------- + +Messages sent using this module are not encrypted. Messages can be encrypted +using the `E2E module`_. + +Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross-Site +Scripting (XSS) attacks. This includes room names and topics. + +.. _`E2E module`: `module:e2e`_ + diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst new file mode 100644 index 000000000..3602d105b --- /dev/null +++ b/specification/modules/presence.rst @@ -0,0 +1,112 @@ +Presence +======== + +.. _module:presence: + +Each user has the concept of presence information. This encodes: + +* Whether the user is currently online +* How recently the user was last active (as seen by the server) +* Whether a given client considers the user to be currently idle +* Arbitrary information about the user's current status (e.g. "in a meeting"). + +This information is collated from both per-device (``online``, ``idle``, +``last_active``) and per-user (status) data, aggregated by the user's homeserver +and transmitted as an ``m.presence`` event. This is one of the few events which +are sent *outside the context of a room*. Presence events are sent to all users +who subscribe to this user's presence through a presence list or by sharing +membership of a room. + +A presence list is a list of user IDs whose presence the user wants to follow. +To be added to this list, the user being added must be invited by the list owner +who must accept the invitation. + +User's presence state is represented by the ``presence`` key, which is an enum +of one of the following: + +- ``online`` : The default state when the user is connected to an event + stream. +- ``unavailable`` : The user is not reachable at this time e.g. they are + idle. +- ``offline`` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. +- ``free_for_chat`` : The user is generally willing to receive messages + moreso than default. + +Events +------ + +{{presence_events}} + +Client behaviour +---------------- + +Clients can manually set/get their presence/presence list using the HTTP APIs +listed below. + +{{presence_http_api}} + +Idle timeout +~~~~~~~~~~~~ + +Clients SHOULD implement an "idle timeout". This is a timer which fires after +a period of inactivity on the client. The definition of inactivity varies +depending on the client. For example, web implementations may determine +inactivity to be not moving the mouse for a certain period of time. When this +timer fires it should set the presence state to ``unavailable``. When the user +becomes active again (e.g. by moving the mouse) the client should set the +presence state to ``online``. A timeout value between 1 and 5 minutes is +recommended. + +Server behaviour +---------------- + +Each user's home server stores a "presence list" per user. Once a user accepts +a presence list, both user's HSes must track the subscription. + +Propagating profile information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields SHOULD cause an +automatic propagation event to occur, informing likely-interested parties of the +new values. One of these change mechanisms SHOULD be via ``m.presence`` events. +These events should set ``displayname`` and ``avatar_url`` to the new values +along with the presence-specific keys. This SHOULD be done automatically by the +home server when a user successfully changes their display name or avatar URL. + +.. admonition:: Rationale + + The intention for sending this information in ``m.presence`` is so that any + "user list" can display the *current* name/presence for a user ID outside the + scope of a room e.g. for a user page. This is bundled into a single event for + several reasons. The user's display name can change per room. This + event provides the "canonical" name for the user. In addition, the name is + bundled into a single event for the ease of client implementations. If this + was not done, the client would need to search all rooms for their own + membership event to pull out the display name. + + +Last active ago +~~~~~~~~~~~~~~~ +The server maintains a timestamp of the last time it saw a +pro-active event from the user. A pro-active event may be sending a message to a +room or changing presence state to a higher level of availability. Levels of +availability are defined from low to high as follows: + +- ``offline`` +- ``unavailable`` +- ``online`` +- ``free_for_chat`` + +Based on this list, changing state from ``unavailable`` to ``online`` counts as +a pro-active event, whereas ``online`` to ``unavailable`` does not. This +timestamp is presented via a key called ``last_active_ago`` which gives the +relative number of milliseconds since the pro-active event. + +Security considerations +----------------------- + +Presence information is shared with all users who share a room with the target +user. In large public rooms this could be undesirable. + diff --git a/specification/modules/push.rst b/specification/modules/push.rst new file mode 100644 index 000000000..893e94815 --- /dev/null +++ b/specification/modules/push.rst @@ -0,0 +1,440 @@ +Push Notifications +================== + +.. _module:push: + +:: + + +--------------------+ +-------------------+ + Matrix HTTP | | | | + Notification Protocol | App Developer | | Device Vendor | + | | | | + +-------------------+ | +----------------+ | | +---------------+ | + | | | | | | | | | | + | Matrix Home Server+-----> Push Gateway +------> Push Provider | | + | | | | | | | | | | + +-^-----------------+ | +----------------+ | | +----+----------+ | + | | | | | | + Matrix | | | | | | + Client/Server API + | | | | | + | | +--------------------+ +-------------------+ + | +--+-+ | + | | <-------------------------------------------+ + +---+ | + | | Provider Push Protocol + +----+ + + Mobile Device or Client + + +This module adds support for push notifications. Homeservers send notifications +of events to user-configured HTTP endpoints. Users may also configure a +number of rules that determine which events generate notifications. These are +all stored and managed by the user's homeserver. This allows user-specific push +settings to be reused between client applications. + +The above diagram shows the flow of push notifications being sent to a handset +where push notifications are submitted via the handset vendor, such as Apple's +APNS or Google's GCM. This happens as follows: + +1. The client app signs in to a homeserver. +2. The client app registers with its vendor's Push Provider and + obtains a routing token of some kind. +3. The mobile app uses the Client/Server API to add a 'pusher', providing the + URL of a specific Push Gateway which is configured for that + application. It also provides the routing token it has acquired from the + Push Provider. +4. The homeserver starts sending HTTP requests to the Push Gateway using the + supplied URL. The Push Gateway relays this notification to + the Push Provider, passing the routing token along with any + necessary private credentials the provider requires to send push + notifications. +5. The Push Provider sends the notification to the device. + +Definitions for terms used in this section are below: + +Push Provider + A push provider is a service managed by the device vendor which can send + notifications directly to the device. Google Cloud Messaging (GCM) and Apple + Push Notification Service (APNS) are two examples of push providers. + +Push Gateway + A push gateway is a server that receives HTTP event notifications from + homeservers and passes them on to a different protocol such as APNS for iOS + devices or GCM for Android devices. Clients inform the homeserver which + Push Gateway to send notifications to when it sets up a Pusher. + +.. _def:pushers: + +Pusher + A pusher is a worker on the homeserver that manages the sending + of HTTP notifications for a user. A user can have multiple pushers: one per + device. + +Push Rule + A push rule is a single rule that states under what *conditions* an event should + be passed onto a push gateway and *how* the notification should be presented. + These rules are stored on the user's homeserver. They are manually configured + by the user, who can create and view them via the Client/Server API. + +Push Ruleset + A push ruleset *scopes a set of rules according to some criteria*. For example, + some rules may only be applied for messages from a particular sender, + a particular room, or by default. The push ruleset contains the entire set + of scopes and rules. + +Client behaviour +---------------- + +Clients MUST configure a Pusher before they will receive push notifications. +There is a single API endpoint for this, as described below. + +{{pusher_http_api}} + +.. _pushers: `def:pushers`_ + +Push Rules +~~~~~~~~~~ +A push rule is a single rule that states under what *conditions* an event should +be passed onto a push gateway and *how* the notification should be presented. +There are different "kinds" of push rules and each rule has an associated +priority. Every push rule MUST have a ``kind`` and ``rule_id``. The ``rule_id`` +is a unique string within the kind of rule and its' scope: ``rule_ids`` do not +need to be unique between rules of the same kind on different devices. Rules may +have extra keys depending on the value of ``kind``.The different kinds of rule +in descending order of priority are: + +Override Rules ``override`` + The highest priority rules are user-configured overrides. +Content-specific Rules ``content`` + These configure behaviour for (unencrypted) messages that match certain + patterns. Content rules take one parameter: ``pattern``, that gives the glob + pattern to match against. This is treated in the same way as ``pattern`` for + ``event_match``. +Room-specific Rules ``room`` + These rules change the behaviour of all messages for a given room. The + ``rule_id`` of a room rule is always the ID of the room that it affects. +Sender-specific rules ``sender`` + These rules configure notification behaviour for messages from a specific + Matrix user ID. The ``rule_id`` of Sender rules is always the Matrix user + ID of the user whose messages they'd apply to. +Underride rules ``underride`` + These are identical to ``override`` rules, but have a lower priority than + ``content``, ``room`` and ``sender`` rules. + +Push rules may be either global or device-specific. Device specific rules only +affect delivery of notifications via pushers with a matching ``profile_tag``. +All device-specific rules have a higher priority than global rules. This means +that the full list of rule kinds, in descending priority order, is as follows: + +* Device-specific Override +* Device-specific Content +* Device-specific Room +* Device-specific Sender +* Device-specific Underride +* Global Override +* Global Content +* Global Room +* Global Sender +* Global Underride + +Rules with the same ``kind`` can specify an ordering priority. This determines +which rule is selected in the event of multiple matches. For example, a rule +matching "tea" and a separate rule matching "time" would both match the sentence +"It's time for tea". The ordering of the rules would then resolve the tiebreak +to determine which rule is executed. Only ``actions`` for highest priority rule +will be sent to the Push Gateway. + +Each rule can be enabled or disabled. Disabled rules never match. If no rules +match an event, the homeserver MUST NOT notify the Push Gateway for that event. +Homeservers MUST NOT notify the Push Gateway for events that the user has sent +themselves. + +Actions ++++++++ +All rules have an associated list of ``actions``. An action affects if and how a +notification is delivered for a matching event. The following actions are defined: + +``notify`` + This causes each matching event to generate a notification. +``dont_notify`` + This prevents each matching event from generating a notification +``coalesce`` + This enables notifications for matching events but activates homeserver + specific behaviour to intelligently coalesce multiple events into a single + notification. Not all homeservers may support this. Those that do not support + it should treat it as the ``notify`` action. +``set_tweak`` + Sets an entry in the ``tweaks`` dictionary key that is sent in the notification + request to the Push Gateway. This takes the form of a dictionary with a + ``set_tweak`` key whose value is the name of the tweak to set. It may also + have a ``value`` key which is the value to which it should be set. + +Actions that have no parameters are represented as a string. Otherwise, they are +represented as a dictionary with a key equal to their name and other keys as +their parameters, e.g. ``{ "set_tweak": "sound", "value": "default" }`` + +Tweaks +^^^^^^ +The ``set_tweak`` action is used to add an entry to the 'tweaks' dictionary +that is sent in the notification request to the Push Gateway. The following +tweaks are defined: + +``sound`` + A string representing the sound to be played when this notification arrives. + A value of ``default`` means to play a default sound. +``highlight`` + A boolean representing whether or not this message should be highlighted in + the UI. This will normally take the form of presenting the message in a + different colour and/or style. The UI might also be adjusted to draw + particular attention to the room in which the event occurred. The ``value`` + may be omitted from the highlight tweak, in which case it should default to + ``true``. + +Tweaks are passed transparently through the homeserver so client applications +and Push Gateways may agree on additional tweaks. For example, a tweak may be +added to specify how to flash the notification light on a mobile device. + +Predefined Rules +++++++++++++++++ +Homeservers can specify "server-default rules" which operate at a lower priority +than "user-defined rules". The ``rule_id`` for all server-default rules MUST +start with a dot (".") to identify them as "server-default". The following +server-default rules are specified: + +``.m.rule.contains_user_name`` + Matches any message whose content is unencrypted and contains the local part + of the user's Matrix ID, separated by word boundaries. + + Definition (as a ``content`` rule):: + + { + "rule_id": ".m.rule.contains_user_name" + "pattern": "[the local part of the user's Matrix ID]", + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ], + } + +``.m.rule.contains_display_name`` + Matches any message whose content is unencrypted and contains the user's + current display name in the room in which it was sent. + + Definition (this rule can only be an ``override`` or ``underride`` rule):: + + { + "rule_id": ".m.rule.contains_display_name" + "conditions": [ + { + "kind": "contains_display_name" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ], + } + +``.m.rule.room_one_to_one`` + Matches any message sent in a room with exactly two members. + + Definition (this rule can only be an ``override`` or ``underride`` rule):: + + { + "rule_id": ".m.rule.room_two_members" + "conditions": [ + { + "is": "2", + "kind": "room_member_count" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ], + } + +``.m.rule.suppress_notices`` + Matches messages with a ``msgtype`` of ``notice``. This should be an + ``override`` rule so that it takes priority over ``content`` / ``sender`` / + ``room`` rules. + + Definition:: + + { + 'rule_id': '.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + } + ], + 'actions': [ + 'dont-notify', + ] + } + +``.m.rule.fallback`` + Matches any message. Used to define the behaviour of messages that match no + other rules. If homeservers define this it should be the lowest priority + ``underride`` rule. + + Definition:: + + { + "rule_id": ".m.rule.fallback" + "conditions": [], + "actions": [ + "notify" + ], + } + + + +Conditions +++++++++++ + +Override, Underride and Default Rules MAY have a list of 'conditions'. +All conditions must hold true for an event in order to apply the ``action`` for +the event. A rule with no conditions always matches. Room, Sender, User and +Content rules do not have conditions in the same way, but instead have +predefined conditions. These conditions can be configured using the parameters +outlined below. In the cases of room and sender rules, the ``rule_id`` of the +rule determines its behaviour. The following conditions are defined: + +``event_match`` + This is a glob pattern match on a field of the event. Parameters: + + * ``key``: The dot-separated field of the event to match, e.g. ``content.body`` + * ``pattern``: The glob-style pattern to match against. Patterns with no + special glob characters should be treated as having asterisks + prepended and appended when testing the condition. + +``profile_tag`` + Matches the ``profile_tag`` of the device that the notification would be + delivered to. Parameters: + + * ``profile_tag``: The profile_tag to match with. + +``contains_display_name`` + This matches unencrypted messages where ``content.body`` contains the owner's + display name in that room. This is a separate rule because display names may + change and as such it would be hard to maintain a rule that matched the user's + display name. This condition has no parameters. + +``room_member_count`` + This matches the current number of members in the room. Parameters: + + * ``is``: A decimal integer optionally prefixed by one of, ``==``, ``<``, + ``>``, ``>=`` or ``<=``. A prefix of ``<`` matches rooms where the member + count is strictly less than the given number and so forth. If no prefix is + present, this parameter defaults to ``==``. + +Push Rules: API +~~~~~~~~~~~~~~~ + +Clients can retrieve, add, modify and remove push rules globally or per-device +using the APIs below. + +{{pushrules_http_api}} + +Examples +++++++++ + +To create a rule that suppresses notifications for the room with ID +``!dj234r78wl45Gh4D:matrix.org``:: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ + '{ + "actions" : ["dont_notify"] + }' + +To suppress notifications for the user ``@spambot:matrix.org``:: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ + '{ + "actions" : ["dont_notify"] + }' + +To always notify for messages that contain the work 'cake' and set a specific +sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``):: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ + '{ + "pattern": "cake", + "actions" : ["notify", {"set_sound":"cakealarm.wav"}] + }' + +To add a rule suppressing notifications for messages starting with 'cake' but +ending with 'lie', superseding the previous rule:: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ + '{ + "pattern": "cake*lie", + "actions" : ["notify"] + }' + +To add a custom sound for notifications messages containing the word 'beer' in +any rooms with 10 members or fewer (with greater importance than the room, +sender and content rules):: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ + '{ + "conditions": [ + {"kind": "event_match", "key": "content.body", "pattern": "beer" }, + {"kind": "room_member_count", "is": "<=10"} + ], + "actions" : [ + "notify", + {"set_sound":"beeroclock.wav"} + ] + }' + +Server behaviour +---------------- + +This describes the format used by "HTTP" pushers to send notifications of +events to Push Gateways. If the endpoint returns an HTTP error code, the +homeserver SHOULD retry for a reasonable amount of time using exponential-backoff. + +{{push_notifier_http_api}} + +Push Gateway behaviour +---------------------- + +Recommendations for APNS +~~~~~~~~~~~~~~~~~~~~~~~~ +The exact format for sending APNS notifications is flexible and up to the +client app and its' push gateway to agree on. As APNS requires that the sender +has a private key owned by the app developer, each app must have its own push +gateway. It is recommended that: + +* The APNS token be base64 encoded and used as the pushkey. +* A different app_id be used for apps on the production and sandbox + APS environments. +* APNS push gateways do not attempt to wait for errors from the APNS + gateway before returning and instead to store failures and return + 'rejected' responses next time that pushkey is used. + +Security considerations +----------------------- + +Clients specify the Push Gateway URL to use to send event notifications to. This +URL should be over HTTPS and *never* over HTTP. + +As push notifications will pass through a Push Provider, message content +shouldn't be sent in the push itself where possible. Instead, Push Gateways +should send a "sync" command to instruct the client to get new events from the +homeserver directly. + diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst new file mode 100644 index 000000000..a8ad3cd34 --- /dev/null +++ b/specification/modules/receipts.rst @@ -0,0 +1,84 @@ +Receipts +======== + +.. _module:receipts: + +This module adds in support for receipts. These receipts are a form of +acknowledgement of an event. This module defines a single acknowledgement: +``m.read`` which indicates that the user has read up to a given event. + +Sending a receipt for each event can result in sending large amounts of traffic +to a homeserver. To prevent this from becoming a problem, receipts are implemented +using "up to" markers. This marker indicates that the acknowledgement applies +to all events "up to and including" the event specified. For example, marking +an event as "read" would indicate that the user had read all events *up to* the +referenced event. + +Events +------ +Each ``user_id``, ``receipt_type`` pair must be associated with only a +single ``event_id``. + +{{m_receipt_event}} + +Client behaviour +---------------- + +In v1 ``/initialSync``, receipts are listed in a separate top level ``receipts`` +key. In v2 ``/sync``, receipts are contained in the ``ephemeral`` block for a +room. New receipts that come down the event streams are deltas which update +existing mappings. Clients should replace older receipt acknowledgements based +on ``user_id`` and ``receipt_type`` pairs. For example:: + + Client receives m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $aaa:example.com + + Client receives another m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $bbb:example.com + + The client should replace the older acknowledgement for $aaa:example.com with + this one for $bbb:example.com + +Clients should send read receipts when there is some certainty that the event in +question has been **displayed** to the user. Simply receiving an event does not +provide enough certainty that the user has seen the event. The user SHOULD need +to *take some action* such as viewing the room that the event was sent to or +dismissing a notification in order for the event to count as "read". + +A client can update the markers for its user by interacting with the following +HTTP APIs. + +{{v2_receipts_http_api}} + +Server behaviour +---------------- + +For efficiency, receipts SHOULD be batched into one event per room before +delivering them to clients. + +Receipts are sent across federation as EDUs with type ``m.receipt``. The +format of the EDUs are:: + + { + : { + : { + : { } + }, + ... + }, + ... + } + +These are always sent as deltas to previously sent receipts. Currently only a +single ```` should be used: ``m.read``. + +Security considerations +----------------------- + +As receipts are sent outside the context of the event graph, there are no +integrity checks performed on the contents of ``m.receipt`` events. + diff --git a/specification/modules/search.rst b/specification/modules/search.rst new file mode 100644 index 000000000..089953d22 --- /dev/null +++ b/specification/modules/search.rst @@ -0,0 +1,46 @@ +Server Side Search +================== + +.. _module:search: + +The search API allows clients to perform full text search across events in all +rooms that the user has been in, including those that they have left. Only +events that the user is allowed to see will be searched, e.g. it won't include +events in rooms that happened after you left. + +Client behaviour +---------------- +{{search_http_api}} + +Search Categories +----------------- + +The search API allows clients to search in different categories of items. +Currently the only specified category is ``room_events``. + +``room_events`` +~~~~~~~~~~~~~~~ + +This category covers all events that the user is allowed to see, including +events in rooms that they have left. The search is performed on certain keys of +certain event types. + +The supported keys to search over are: + +- ``content.body`` in ``m.room.message`` +- ``content.name`` in ``m.room.name`` +- ``content.topic`` in ``m.room.topic`` + +The search will *not* include rooms that are end to end encrypted. + +The results include a ``rank`` key that can be used to sort the results by +revelancy. The higher the ``rank`` the more relevant the result is. + +The value of ``count`` may not match the number of results. For example due to +the search query matching 1000s of results and the server truncating the +response. + +Security considerations +----------------------- +The server must only return results that the user has permission to see. + diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst new file mode 100644 index 000000000..85538c314 --- /dev/null +++ b/specification/modules/third_party_invites.rst @@ -0,0 +1,114 @@ +Third party invites +=================== + +.. _module:third_party_invites: + +This module adds in support for inviting new members to a room where their +Matrix user ID is not known, instead addressing them by a third party identifier +such as an email address. + +There are two flows here; one if a Matrix user ID is known for the third party +identifier, and one if not. Either way, the client calls ``/invite`` with the +details of the third party identifier. + +The homeserver asks the identity server whether a Matrix user ID is known for +that identifier. If it is, an invite is simply issued for that user. + +If it is not, the homeserver asks the identity server to record the details of +the invitation, and to notify the client of this pending invitation if it gets +a binding for this identifier in the future. The identity server returns a token +and public key to the homeserver. + +If a client then tries to join the room in the future, it will be allowed to if +it presents both the token, and a signature of that token from the identity +server which can be verified with the public key. + +Events +------ + +{{m_room_third_party_invite_event}} + +Client behaviour +---------------- + +A client asks a server to invite a user by their third party identifier. + +Server behaviour +---------------- + +All homeservers MUST verify the signature in the event's +``content.third_party_invite.signed`` object. + +If a client of the current homeserver is joining by an +``m.room.third_party_invite``, that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url``. It does +this by making an HTTP GET request to ``key_validity_url``: + +.. TODO: Link to identity server spec when it exists + +Schema:: + + => GET $key_validity_url?public_key=$public_key + <= HTTP/1.1 200 OK + { + "valid": true|false + } + + +Example:: + + key_validity_url = https://identity.server/is_valid + public_key = ALJWLAFQfqffQHFqFfeqFUOEHf4AIHfefh4 + => GET https://identity.server/is_valid?public_key=ALJWLAFQfqffQHFqFfeqFUOEHf4AIHfefh4 + <= HTTP/1.1 200 OK + { + "valid": true + } + +with the querystring +?public_key=``public_key``. A JSON object will be returned. +The invitation is valid if the object contains a key named ``valid`` which is +``true``. Otherwise, the invitation MUST be rejected. This request is +idempotent and may be retried by the homeserver. + +If a homeserver is joining a room for the first time because of an +``m.room.third_party_invite``, the server which is already participating in the +room (which is chosen as per the standard server-server specification) MUST +validate that the public key used for signing is still valid, by checking +``key_validity_url`` in the above described way. + +No other homeservers may reject the joining of the room on the basis of +``key_validity_url``, this is so that all homeservers have a consistent view of +the room. They may, however, indicate to their clients that a member's' +membership is questionable. + +For example: + + If room R has two participating homeservers, H1, H2 + + And user A on H1 invites a third party identifier to room R + + H1 asks the identity server for a binding to a Matrix user ID, and has none, + so issues an ``m.room.third_party_invite`` event to the room. + + When the third party user validates their identity, they are told about the + invite, and ask their homeserver, H3, to join the room. + + H3 validates the signature in the event's + ``content.third_party_invite.signed`` object. + + H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` + property *and* check ``key_validity_url``. + + Having validated these things, H1 writes the join event to the room, and H3 + begins participating in the room. H2 *must* accept this event. + +The reason that no other homeserver may reject the event based on checking +``key_validity_url`` is that we must ensure event acceptance is deterministic. +If some other participating server doesn't have a network path to the keyserver, +or if the keyserver were to go offline, or revoke its keys, that other server +would reject the event and cause the participating servers' graphs to diverge. +This relies on participating servers trusting each other, but that trust is +already implied by the server-server protocol. Also, the public key signature +verification must still be performed, so the attack surface here is minimized. + diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst new file mode 100644 index 000000000..d22536327 --- /dev/null +++ b/specification/modules/typing_notifications.rst @@ -0,0 +1,66 @@ +Typing Notifications +==================== + +.. _module:typing: + +Users may wish to be informed when another user is typing in a room. This can be +achieved using typing notifications. These are ephemeral events scoped to a +``room_id``. This means they do not form part of the `Event Graph`_ but still +have a ``room_id`` key. + +.. _Event Graph: `sect:event-graph`_ + +Events +------ + +{{m_typing_event}} + +Client behaviour +---------------- + +When a client receives an ``m.typing`` event, it MUST use the user ID list to +**REPLACE** its knowledge of every user who is currently typing. The reason for +this is that the server *does not remember* users who are not currently typing +as that list gets big quickly. The client should mark as not typing any user ID +who is not in that list. + +It is recommended that clients store a ``boolean`` indicating whether the user +is typing or not. Whilst this value is ``true`` a timer should fire periodically +every N seconds to send a typing HTTP request. The value of N is recommended to +be no more than 20-30 seconds. This request should be re-sent by the client to +continue informing the server the user is still typing. As subsequent +requests will replace older requests, a safety margin of 5 seconds before the +expected timeout runs out is recommended. When the user stops typing, the +state change of the ``boolean`` to ``false`` should trigger another HTTP request +to inform the server that the user has stopped typing. + +{{typing_http_api}} + +Server behaviour +---------------- + +Servers MUST emit typing EDUs in a different form to ``m.typing`` events which +are shown to clients. This form looks like:: + + { + "type": "m.typing", + "content": { + "room_id": "!room-id-here:matrix.org", + "user_id": "@user-id-here:matrix.org", + "typing": true/false + } + } + +This does not contain timing information so it is up to originating homeservers +to ensure they eventually send "stop" notifications. + +.. TODO + ((This will eventually need addressing, as part of the wider typing/presence + timer addition work)) + +Security considerations +----------------------- + +Clients may not wish to inform everyone in a room that they are typing and +instead only specific users in the room. + diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst new file mode 100644 index 000000000..6945a09e2 --- /dev/null +++ b/specification/modules/voip_events.rst @@ -0,0 +1,101 @@ +Voice over IP +============= + +.. _module:voip: + +This module outlines how two users in a room can set up a Voice over IP (VoIP) +call to each other. Voice and video calls are built upon the WebRTC 1.0 standard. +Call signalling is achieved by sending `message events`_ to the room. As a result, +this means that clients MUST only send call events to rooms with exactly two +participants as currently the WebRTC standard is based around two-party +communication. + +.. _message events: `sect:events`_ + +Events +------ + +{{voip_events}} + +Client behaviour +---------------- + +A call is set up with message events exchanged as follows: + +:: + + Caller Callee + [Place Call] + m.call.invite -----------> + m.call.candidate --------> + [..candidates..] --------> + [Answers call] + <--------------- m.call.answer + [Call is active and ongoing] + <--------------- m.call.hangup + +Or a rejected call: + +:: + + Caller Callee + m.call.invite ------------> + m.call.candidate ---------> + [..candidates..] ---------> + [Rejects call] + <-------------- m.call.hangup + +Calls are negotiated according to the WebRTC specification. + +Glare +~~~~~ + +"Glare" is a problem which occurs when two users call each other at roughly the +same time. This results in the call failing to set up as there already is an +incoming/outgoing call. A glare resolution algorithm can be used to determine +which call to hangup and which call to answer. If both clients implement the +same algorithm then they will both select the same call and the call will be +successfully connected. + + +As calls are "placed" to rooms rather than users, the glare resolution algorithm +outlined below is only considered for calls which are to the same room. The +algorithm is as follows: + +- If an ``m.call.invite`` to a room is received whilst the client is + **preparing to send** an ``m.call.invite`` to the same room: + + * the client should cancel its outgoing call and instead + automatically accept the incoming call on behalf of the user. + +- If an ``m.call.invite`` to a room is received **after the client has sent** + an ``m.call.invite`` to the same room and is waiting for a response: + + * the client should perform a lexicographical comparison of the call IDs of + the two calls and use the *lesser* of the two calls, aborting the + greater. If the incoming call is the lesser, the client should accept + this call on behalf of the user. + + +The call setup should appear seamless to the user as if they had simply placed +a call and the other party had accepted. This means any media stream that had been +setup for use on a call should be transferred and used for the call that +replaces it. + +Server behaviour +---------------- + +The homeserver MAY provide a TURN server which clients can use to contact the +remote party. The following HTTP API endpoints will be used by clients in order +to get information about the TURN server. + +{{voip_http_api}} + + +Security considerations +----------------------- + +Calls should only be placed to rooms with one other user in them. If they are +placed to group chat rooms it is possible that another user will intercept and +answer the call. + diff --git a/specification/30_server_server_api.rst b/specification/server_server_api.rst similarity index 84% rename from specification/30_server_server_api.rst rename to specification/server_server_api.rst index 484340754..26e040ca2 100644 --- a/specification/30_server_server_api.rst +++ b/specification/server_server_api.rst @@ -2,10 +2,9 @@ Federation API ============== Matrix home servers use the Federation APIs (also known as server-server APIs) -to communicate with each other. -Home servers use these APIs to push messages to each other in real-time, to -request historic messages from each other, and to query profile and presence -information about users on each other's servers. +to communicate with each other. Home servers use these APIs to push messages to +each other in real-time, to request historic messages from each other, and to +query profile and presence information about users on each other's servers. The APIs are implemented using HTTPS GETs and PUTs between each of the servers. These HTTPS requests are strongly authenticated using public key @@ -16,18 +15,18 @@ There are three main kinds of communication that occur between home servers: Persisted Data Units (PDUs): These events are broadcast from one home server to any others that have - joined the same "context" (namely, a Room ID). They are persisted in + joined the same room (identified by Room ID). They are persisted in long-term storage and record the history of messages and state for a - context. + room. Like email, it is the responsibility of the originating server of a PDU - to deliver that event to its recepient servers. However PDUs are signed + to deliver that event to its recipient servers. However PDUs are signed using the originating server's public key so that it is possible to deliver them through third-party servers. Ephemeral Data Units (EDUs): These events are pushed between pairs of home servers. They are not - persisted and are not part of the history of a "context", nor does the + persisted and are not part of the history of a room, nor does the receiving home server have to reply to them. Queries: @@ -59,11 +58,12 @@ and an optional TLS port. .. ** -If the port is present then the server is discovered by looking up an A record -for the DNS name and connecting to the specified TLS port. If the port is -absent then the server is discovered by looking up a ``_matrix._tcp`` -SRV record for the DNS name. - +If the port is present then the server is discovered by looking up an AAAA or +A record for the DNS name and connecting to the specified TLS port. If the port +is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV +record for the DNS name. If this record does not exist then the server is +discovered by looking up an AAAA or A record on the DNS name and taking the +default fallback port number of 8448. Home servers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. @@ -75,24 +75,25 @@ Version 2 Each home server publishes its public keys under ``/_matrix/key/v2/server/``. Home servers query for keys by either getting ``/_matrix/key/v2/server/`` -directly or by querying an intermediate perspective server using a -``/_matrix/key/v2/query`` API. Intermediate perspective servers query the +directly or by querying an intermediate notary server using a +``/_matrix/key/v2/query`` API. Intermediate notary servers query the ``/_matrix/key/v2/server/`` API on behalf of another server and sign the -response with their own key. A server may query multiple perspective servers -to ensure that they all report the same public keys. +response with their own key. A server may query multiple notary servers to +ensure that they all report the same public keys. + +This approach is borrowed from the `Perspectives Project`_, but modified to +include the NACL keys and to use JSON instead of XML. It has the advantage of +avoiding a single trust-root since each server is free to pick which notary +servers they trust and can corroborate the keys returned by a given notary +server by querying other servers. -This approach is borrowed from the Perspectives Project -(http://perspectives-project.org/), but modified to include the NACL keys and to -use JSON instead of XML. It has the advantage of avoiding a single trust-root -since each server is free to pick which perspective servers they trust and can -corroborate the keys returned by a given perspective server by querying other -servers. +.. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON -object at ``/_matrix/key/v2/server/${key_id}``. The response contains a list of +object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the server and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list @@ -119,15 +120,15 @@ The ``old_verify_keys`` can be used to sign events with an ``origin_server_ts`` before the ``expired_ts``. The ``expired_ts`` is a millisecond POSIX timestamp of when the originating server stopped using that key. -Intermediate perspective servers should cache a response for half of its -remaining life time to avoid serving a stale response. Originating servers should -avoid returning responses that expire in less than an hour to avoid repeated -requests for an about to expire certificate. Requesting servers should limit how +Intermediate notary servers should cache a response for half of its remaining +life time to avoid serving a stale response. Originating servers should avoid +returning responses that expire in less than an hour to avoid repeated requests +for an about to expire certificate. Requesting servers should limit how frequently they query for certificates to avoid flooding a server with requests. -If a server goes offline intermediate perspective servers should continue to -return the last response they received from that server so that the signatures -of old events sent by that server can still be checked. +If a server goes offline intermediate notary servers should continue to return +the last response they received from that server so that the signatures of old +events sent by that server can still be checked. ==================== =================== ====================================== Key Type Description @@ -175,7 +176,7 @@ of old events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a @@ -188,8 +189,8 @@ The ``minimum_valid_until_ts`` is a millisecond POSIX timestamp indicating when the returned certificate will need to be valid until to be useful to the requesting server. This can be set using the maximum ``origin_server_ts`` of an batch of events that a requesting server is trying to validate. This allows -an intermediate perspectives server to give a prompt cached response even if -the originating server is offline. +an intermediate notary server to give a prompt cached response even if the +originating server is offline. This API can return keys for servers that are offline be using cached responses taken from when the server was online. Keys can be queried from multiple @@ -337,11 +338,11 @@ PDUs All PDUs have: -- An ID -- A context +- An ID to identify the PDU itself +- A room ID that it relates to - A declaration of their type -- A list of other PDU IDs that have been seen recently on that context - (regardless of which origin sent them) +- A list of other PDU IDs that have been seen recently in that room (regardless + of which origin sent them) Required PDU Fields @@ -350,7 +351,7 @@ Required PDU Fields ==================== ================== ======================================= Key Type Description ==================== ================== ======================================= -``context`` String Event context identifier +``context`` String Room identifier ``user_id`` String The ID of the user sending the PDU ``origin`` String DNS name of homeserver that created this PDU @@ -362,7 +363,7 @@ Required PDU Fields ``content`` Object The content of the PDU. ``prev_pdus`` List of (String, The originating homeserver, PDU ids and String, Object) hashes of the most recent PDUs the - Triplets homeserver was aware of for the context + Triplets homeserver was aware of for the room when it made this PDU ``depth`` Integer The maximum depth of the previous PDUs plus one @@ -439,7 +440,7 @@ keys exist to support this: EDUs ---- -EDUs, by comparison to PDUs, do not have an ID, a context, or a list of +EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and destination home server names, and the actual nested content. @@ -490,24 +491,24 @@ Retrieves a given PDU from the server. The response will contain a single new Transaction, inside which will be the requested PDU. -To fetch all the state of a given context:: +To fetch all the state of a given room:: - GET .../state// + GET .../state// Response: JSON encoding of a single Transaction containing multiple PDUs -Retrieves a snapshot of the entire current state of the given context. The +Retrieves a snapshot of the entire current state of the given room. The response will contain a single Transaction, inside which will be a list of PDUs that encode the state. -To backfill events on a given context:: +To backfill events on a given room:: - GET .../backfill// + GET .../backfill// Query args: v, limit Response: JSON encoding of a single Transaction containing multiple PDUs Retrieves a sliding-window history of previous PDUs that occurred on the given -context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that -preceeded it are retrieved, up to a total number given by the "limit" argument. +room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that +preceded it are retrieved, up to a total number given by the "limit" argument. These are then returned in a new Transaction containing all of the PDUs. @@ -551,7 +552,10 @@ Every HTTP request made by a homeserver is authenticated using public key digital signatures. The request method, target and body are signed by wrapping them in a JSON object and signing it using the JSON signing algorithm. The resulting signatures are added as an Authorization header with an auth scheme -of X-Matrix. +of X-Matrix. Note that the target field should include the full path starting with +``/_matrix/...``, including the ``?`` and any query parameters if present, but +should not include the leading ``https:``, nor the destination server's +hostname. Step 1 sign JSON: @@ -626,7 +630,7 @@ because HTTP services like Matrix are often deployed behind load balancers that handle the TLS and these load balancers make it difficult to check TLS client certificates. -A home server may provide a TLS client certficate and the receiving home server +A home server may provide a TLS client certificate and the receiving home server may check that the client certificate matches the certificate of the origin home server. @@ -648,12 +652,12 @@ State Conflict Resolution - How does this work with deleting current state - How do we reject invalid federation traffic? -[[TODO(paul): At this point we should probably have a long description of how -State management works, with descriptions of clobbering rules, power levels, etc -etc... But some of that detail is rather up-in-the-air, on the whiteboard, and -so on. This part needs refining. And writing in its own document as the details -relate to the server/system as a whole, not specifically to server-server -federation.]] + [[TODO(paul): At this point we should probably have a long description of how + State management works, with descriptions of clobbering rules, power levels, etc + etc... But some of that detail is rather up-in-the-air, on the whiteboard, and + so on. This part needs refining. And writing in its own document as the details + relate to the server/system as a whole, not specifically to server-server + federation.]] Presence -------- @@ -669,8 +673,8 @@ Performing a presence update and poll subscription request:: Each should be an object with the following keys: user_id: string containing a User ID presence: "offline"|"unavailable"|"online"|"free_for_chat" - status_msg: (optional) string of freeform text - last_active_ago: miliseconds since the last activity by the user + status_msg: (optional) string of free-form text + last_active_ago: milliseconds since the last activity by the user poll: (optional): list of strings giving User IDs @@ -688,7 +692,7 @@ removed until explicitly requested by a later ``unpoll``. On receipt of a message containing a non-empty ``poll`` list, the receiving server should immediately send the sending server a presence update EDU of its own, containing in a ``push`` list the current state of every user that was in -the orginal EDU's ``poll`` list. +the original EDU's ``poll`` list. Sending a presence invite:: @@ -713,7 +717,7 @@ Rejecting a presence invite:: Content keys - as for m.presence_invite .. TODO-doc - - Explain the timing-based roundtrip reduction mechanism for presence + - Explain the timing-based round-trip reduction mechanism for presence messages - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence @@ -734,11 +738,31 @@ Querying profile information:: field: (optional) string giving a field name Returns: JSON object containing the following keys: - displayname: string of freeform text - avatar_url: string containing an http-scheme URL + displayname: string of free-form text + avatar_url: string containing an HTTP-scheme URL If the query contains the optional ``field`` key, it should give the name of a result field. If such is present, then the result should contain only a field of that name, with no others present. If not, the result should contain as much of the user's profile as the home server has available and can make public. +Directory +--------- + +The server API for directory queries is also based on Federation Queries. + +Querying directory information:: + + Query type: directory + + Arguments: + room_alias: the room alias to query + + Returns: JSON object containing the following keys: + room_id: string giving the underlying room ID the alias maps to + servers: list of strings giving the join candidates + +The list of join candidates is a list of server names that are likely to hold +the given room; these are servers that the requesting server may wish to try +joining with. This list may or may not include the server answering the query. + diff --git a/specification/targets.yaml b/specification/targets.yaml new file mode 100644 index 000000000..2482dcfda --- /dev/null +++ b/specification/targets.yaml @@ -0,0 +1,41 @@ +targets: + main: # arbitrary name to identify this build target + files: # the sort order of files to cat + - intro.rst + - client_server_api.rst + - { 1: events.rst } + - { 1: event_signing.rst } + - modules.rst + - { 1: feature_profiles.rst } + - { 1: "group:modules" } # reference a group of files + - application_service_api.rst + - server_server_api.rst + - identity_servers.rst + - appendices.rst +groups: # reusable blobs of files when prefixed with 'group:' + modules: + - modules/instant_messaging.rst + - modules/voip_events.rst + - modules/typing_notifications.rst + - modules/receipts.rst + - modules/presence.rst + - modules/content_repo.rst + - modules/end_to_end_encryption.rst + - modules/history_visibility.rst + - modules/push.rst + - modules/third_party_invites.rst + - modules/search.rst + + +title_styles: ["=", "-", "~", "+", "^", "`"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/supporting-docs/guides/2015-08-10-client-server.rst b/supporting-docs/guides/2015-08-10-client-server.rst new file mode 100644 index 000000000..2800584ca --- /dev/null +++ b/supporting-docs/guides/2015-08-10-client-server.rst @@ -0,0 +1,652 @@ +--- +layout: post +title: Client-Server API +categories: guides +--- +| + +.. figure:: http://matrix.org/matrix.png + :width: 305px + :height: 130px + :alt: [matrix] + :align: center + + +How to use the client-server API +================================ + +.. NOTE:: + The git version of this document is {% project_version %} + +This guide focuses on how the client-server APIs *provided by the reference +home server* can be used. Since this is specific to a home server +implementation, there may be variations in relation to registering/logging in +which are not covered in extensive detail in this guide. + +If you haven't already, get a home server up and running on +``http://localhost:8008``. + + +Accounts +======== +Before you can send and receive messages, you must **register** for an account. +If you already have an account, you must **login** into it. + +.. NOTE:: + `Try out the fiddle`__ + + .. __: http://jsfiddle.net/gh/get/jquery/1.8.3/matrix-org/matrix-doc/tree/master/supporting-docs/howtos/jsfiddles/register_login + +Registration +------------ +The aim of registration is to get a user ID and access token which you will need +when accessing other APIs:: + + curl -XPOST -d '{"user":"example", "password":"wordpass", "type":"m.login.password"}' "http://localhost:8008/_matrix/client/api/v1/register" + + { + "access_token": "QGV4YW1wbGU6bG9jYWxob3N0.AqdSzFmFYrLrTmteXc", + "home_server": "localhost", + "user_id": "@example:localhost" + } + +NB: If a ``user`` is not specified, one will be randomly generated for you. +If you do not specify a ``password``, you will be unable to login to the account +if you forget the ``access_token``. + +Implementation note: The matrix specification does not enforce how users +register with a server. It just specifies the URL path and absolute minimum +keys. The reference home server uses a username/password to authenticate user, +but other home servers may use different methods. This is why you need to +specify the ``type`` of method. + +Login +----- +The aim when logging in is to get an access token for your existing user ID:: + + curl -XGET "http://localhost:8008/_matrix/client/api/v1/login" + + { + "flows": [ + { + "type": "m.login.password" + } + ] + } + + curl -XPOST -d '{"type":"m.login.password", "user":"example", "password":"wordpass"}' "http://localhost:8008/_matrix/client/api/v1/login" + + { + "access_token": "QGV4YW1wbGU6bG9jYWxob3N0.vRDLTgxefmKWQEtgGd", + "home_server": "localhost", + "user_id": "@example:localhost" + } + +Implementation note: Different home servers may implement different methods for +logging in to an existing account. In order to check that you know how to login +to this home server, you must perform a ``GET`` first and make sure you +recognise the login type. If you do not know how to login, you can +``GET /login/fallback`` which will return a basic webpage which you can use to +login. The reference home server implementation support username/password login, +but other home servers may support different login methods (e.g. OAuth2). + + +Communicating +============= + +In order to communicate with another user, you must **create a room** with that +user and **send a message** to that room. + +.. NOTE:: + `Try out the fiddle`__ + + .. __: http://jsfiddle.net/gh/get/jquery/1.8.3/matrix-org/matrix-doc/tree/master/supporting-docs/howtos/jsfiddles/create_room_send_msg + +Creating a room +--------------- +If you want to send a message to someone, you have to be in a room with them. To +create a room:: + + curl -XPOST -d '{"room_alias_name":"tutorial"}' "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token=YOUR_ACCESS_TOKEN" + + { + "room_alias": "#tutorial:localhost", + "room_id": "!CvcvRuDYDzTOzfKKgh:localhost" + } + +The "room alias" is a human-readable string which can be shared with other users +so they can join a room, rather than the room ID which is a randomly generated +string. You can have multiple room aliases per room. + +.. TODO(kegan) + How to add/remove aliases from an existing room. + + +Sending messages +---------------- +You can now send messages to this room:: + + curl -XPOST -d '{"msgtype":"m.text", "body":"hello"}' "http://localhost:8008/_matrix/client/api/v1/rooms/%21CvcvRuDYDzTOzfKKgh%3Alocalhost/send/m.room.message?access_token=YOUR_ACCESS_TOKEN" + + { + "event_id": "YUwRidLecu" + } + +The event ID returned is a unique ID which identifies this message. + +NB: There are no limitations to the types of messages which can be exchanged. +The only requirement is that ``"msgtype"`` is specified. The Matrix +specification outlines the following standard types: ``m.text``, ``m.image``, +``m.audio``, ``m.video``, ``m.location``, ``m.emote``. See the specification for +more information on these types. + +Users and rooms +=============== + +Each room can be configured to allow or disallow certain rules. In particular, +these rules may specify if you require an **invitation** from someone already in +the room in order to **join the room**. In addition, you may also be able to +join a room **via a room alias** if one was set up. + +.. NOTE:: + `Try out the fiddle`__ + + .. __: http://jsfiddle.net/gh/get/jquery/1.8.3/matrix-org/matrix-doc/tree/master/supporting-docs/howtos/jsfiddles/room_memberships + +Inviting a user to a room +------------------------- +You can directly invite a user to a room like so:: + + curl -XPOST -d '{"user_id":"@myfriend:localhost"}' "http://localhost:8008/_matrix/client/api/v1/rooms/%21CvcvRuDYDzTOzfKKgh%3Alocalhost/invite?access_token=YOUR_ACCESS_TOKEN" + +This informs ``@myfriend:localhost`` of the room ID +``!CvcvRuDYDzTOzfKKgh:localhost`` and allows them to join the room. + +Joining a room via an invite +---------------------------- +If you receive an invite, you can join the room:: + + curl -XPOST -d '{}' "http://localhost:8008/_matrix/client/api/v1/rooms/%21CvcvRuDYDzTOzfKKgh%3Alocalhost/join?access_token=YOUR_ACCESS_TOKEN" + +NB: Only the person invited (``@myfriend:localhost``) can change the membership +state to ``"join"``. Repeatedly joining a room does nothing. + +Joining a room via an alias +--------------------------- +Alternatively, if you know the room alias for this room and the room config +allows it, you can directly join a room via the alias:: + + curl -XPOST -d '{}' "http://localhost:8008/_matrix/client/api/v1/join/%23tutorial%3Alocalhost?access_token=YOUR_ACCESS_TOKEN" + + { + "room_id": "!CvcvRuDYDzTOzfKKgh:localhost" + } + +You will need to use the room ID when sending messages, not the room alias. + +NB: If the room is configured to be an invite-only room, you will still require +an invite in order to join the room even though you know the room alias. As a +result, it is more common to see a room alias in relation to a public room, +which do not require invitations. + +Getting events +============== +An event is some interesting piece of data that a client may be interested in. +It can be a message in a room, a room invite, etc. There are many different ways +of getting events, depending on what the client already knows. + +.. NOTE:: + `Try out the fiddle`__ + + .. __: http://jsfiddle.net/gh/get/jquery/1.8.3/matrix-org/matrix-doc/tree/master/supporting-docs/howtos/jsfiddles/event_stream + +Getting all state +----------------- +If the client doesn't know any information on the rooms the user is +invited/joined on, they can get all the user's state for all rooms:: + + curl -XGET "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=YOUR_ACCESS_TOKEN" + + { + "end": "s39_18_0", + "presence": [ + { + "content": { + "last_active_ago": 1061436, + "user_id": "@example:localhost" + }, + "type": "m.presence" + } + ], + "rooms": [ + { + "membership": "join", + "messages": { + "chunk": [ + { + "content": { + "@example:localhost": 10, + "default": 0 + }, + "event_id": "wAumPSTsWF", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.power_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "join_rule": "public" + }, + "event_id": "jrLVqKHKiI", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.join_rules", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 10 + }, + "event_id": "WpmTgsNWUZ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.add_state_level", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 0 + }, + "event_id": "qUMBJyKsTQ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.send_event_level", + "user_id": "@example:localhost" + }, + { + "content": { + "ban_level": 5, + "kick_level": 5 + }, + "event_id": "YAaDmKvoUW", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.ops_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join" + }, + "event_id": "RJbPMtCutf", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409665586730, + "type": "m.room.member", + "user_id": "@example:localhost" + }, + { + "content": { + "body": "hello", + "hsob_ts": 1409665660439, + "msgtype": "m.text" + }, + "event_id": "YUwRidLecu", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "ts": 1409665660439, + "type": "m.room.message", + "user_id": "@example:localhost" + }, + { + "content": { + "membership": "invite" + }, + "event_id": "YjNuBKnPsb", + "membership": "invite", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@myfriend:localhost", + "ts": 1409666426819, + "type": "m.room.member", + "user_id": "@example:localhost" + }, + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join", + "prev": "join" + }, + "event_id": "KWwdDjNZnm", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409666551582, + "type": "m.room.member", + "user_id": "@example:localhost" + }, + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join" + }, + "event_id": "JFLVteSvQc", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409666587265, + "type": "m.room.member", + "user_id": "@example:localhost" + } + ], + "end": "s39_18_0", + "start": "t1-11_18_0" + }, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state": [ + { + "content": { + "creator": "@example:localhost" + }, + "event_id": "dMUoqVTZca", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.create", + "user_id": "@example:localhost" + }, + { + "content": { + "@example:localhost": 10, + "default": 0 + }, + "event_id": "wAumPSTsWF", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.power_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "join_rule": "public" + }, + "event_id": "jrLVqKHKiI", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.join_rules", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 10 + }, + "event_id": "WpmTgsNWUZ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.add_state_level", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 0 + }, + "event_id": "qUMBJyKsTQ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.send_event_level", + "user_id": "@example:localhost" + }, + { + "content": { + "ban_level": 5, + "kick_level": 5 + }, + "event_id": "YAaDmKvoUW", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.ops_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "membership": "invite" + }, + "event_id": "YjNuBKnPsb", + "membership": "invite", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@myfriend:localhost", + "ts": 1409666426819, + "type": "m.room.member", + "user_id": "@example:localhost" + }, + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join" + }, + "event_id": "JFLVteSvQc", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409666587265, + "type": "m.room.member", + "user_id": "@example:localhost" + } + ] + } + ] + } + +This returns all the room information the user is invited/joined on, as well as +all of the presences relevant for these rooms. This can be a LOT of data. You +may just want the most recent event for each room. This can be achieved by +applying query parameters to ``limit`` this request:: + + curl -XGET "http://localhost:8008/_matrix/client/api/v1/initialSync?limit=1&access_token=YOUR_ACCESS_TOKEN" + + { + "end": "s39_18_0", + "presence": [ + { + "content": { + "last_active_ago": 1279484, + "user_id": "@example:localhost" + }, + "type": "m.presence" + } + ], + "rooms": [ + { + "membership": "join", + "messages": { + "chunk": [ + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join" + }, + "event_id": "JFLVteSvQc", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409666587265, + "type": "m.room.member", + "user_id": "@example:localhost" + } + ], + "end": "s39_18_0", + "start": "t10-30_18_0" + }, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state": [ + { + "content": { + "creator": "@example:localhost" + }, + "event_id": "dMUoqVTZca", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.create", + "user_id": "@example:localhost" + }, + { + "content": { + "@example:localhost": 10, + "default": 0 + }, + "event_id": "wAumPSTsWF", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.power_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "join_rule": "public" + }, + "event_id": "jrLVqKHKiI", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.join_rules", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 10 + }, + "event_id": "WpmTgsNWUZ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.add_state_level", + "user_id": "@example:localhost" + }, + { + "content": { + "level": 0 + }, + "event_id": "qUMBJyKsTQ", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.send_event_level", + "user_id": "@example:localhost" + }, + { + "content": { + "ban_level": 5, + "kick_level": 5 + }, + "event_id": "YAaDmKvoUW", + "required_power_level": 10, + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "", + "ts": 1409665585188, + "type": "m.room.ops_levels", + "user_id": "@example:localhost" + }, + { + "content": { + "membership": "invite" + }, + "event_id": "YjNuBKnPsb", + "membership": "invite", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@myfriend:localhost", + "ts": 1409666426819, + "type": "m.room.member", + "user_id": "@example:localhost" + }, + { + "content": { + "avatar_url": null, + "displayname": null, + "membership": "join" + }, + "event_id": "JFLVteSvQc", + "membership": "join", + "room_id": "!MkDbyRqnvTYnoxjLYx:localhost", + "state_key": "@example:localhost", + "ts": 1409666587265, + "type": "m.room.member", + "user_id": "@example:localhost" + } + ] + } + ] + } + +Getting live state +------------------ +Once you know which rooms the client has previously interacted with, you need to +listen for incoming events. This can be done like so:: + + curl -XGET "http://localhost:8008/_matrix/client/api/v1/events?access_token=YOUR_ACCESS_TOKEN" + + { + "chunk": [], + "end": "s39_18_0", + "start": "s39_18_0" + } + +This will block waiting for an incoming event, timing out after several seconds. +Even if there are no new events (as in the example above), there will be some +pagination stream response keys. The client should make subsequent requests +using the value of the ``"end"`` key (in this case ``s39_18_0``) as the ``from`` +query parameter e.g. ``http://localhost:8008/_matrix/client/api/v1/events?access +_token=YOUR_ACCESS_TOKEN&from=s39_18_0``. This value should be stored so when the +client reopens your app after a period of inactivity, you can resume from where +you got up to in the event stream. If it has been a long period of inactivity, +there may be LOTS of events waiting for the user. In this case, you may wish to +get all state instead and then resume getting live state from a newer end token. + +NB: The timeout can be changed by adding a ``timeout`` query parameter, which is +in milliseconds. A timeout of 0 will not block. + + +Example application +------------------- +The following example demonstrates registration and login, live event streaming, +creating and joining rooms, sending messages, getting member lists and getting +historical messages for a room. This covers most functionality of a messaging +application. + +.. NOTE:: + `Try out the fiddle`__ + + .. __: http://jsfiddle.net/gh/get/jquery/1.8.3/matrix-org/matrix-doc/tree/master/supporting-docs/howtos/jsfiddles/example_app diff --git a/supporting-docs/guides/2015-08-14-getting_involved.md b/supporting-docs/guides/2015-08-14-getting_involved.md new file mode 100644 index 000000000..162ee7b86 --- /dev/null +++ b/supporting-docs/guides/2015-08-14-getting_involved.md @@ -0,0 +1,125 @@ +--- +layout: post +title: Getting involved +categories: guides +--- + +# How can I get involved? +Matrix is an ecosystem consisting of several apps written by lots of people. We at Matrix.org have written one server and a few clients, and people in the community have also written several clients, servers, and Application Services. We are collecting [a list of all known Matrix-apps](https://matrix.org/blog/try-matrix-now/). + +| + +You have a few options when it comes to getting involved: if you just want to use Matrix, you can [register an account on a public server using a public webclient](#reg). If you have a virtual private server (VPS) or similar, you might want to [run a server and/or client yourself](#run). If you want to look under the hood, you can [checkout the code and modify it - or write your own client or server](#checkout). Or you can write an [Application Service](#as), for example a bridge to an existing ecosystem. + +| + + + +## Access Matrix via a public webclient/server + +The easiest thing to do if you want to just have a play, is to use our reference webclient and create a user on the matrix.org homeserver. You do that by visiting http://matrix.org/beta/, selecting "Create account" and choosing your userID and password on the next page. You can also add your email, but this is optional (adding it will make it easier for your friends to find your Matrix user in the future). + +| + +At the bottom of the account creation page, you can pick the homeserver and identity server you want to use. In this case, we are using matrix.org's homeserver, so just leave it as-is. Your full Matrix-userID will be formed partly from the hostname your server is running as, and partly from an userID you specify when you create the account. For example, if you put bob as your userID, your full Matrix-userID will be @bob:matrix.org ("bob on matrix.org"). + +| + +You can use multiple clients with the same user, so you might want to also look at our [Android](https://github.com/matrix-org/matrix-android-console) or [iOS](https://github.com/matrix-org/matrix-ios-console) clients for your mobile phone! + +| + + + +## Run a server and/or client yourself + +You can clone our open source projects and run clients and servers yourself. Here is how: + +### Running your own client: + +You can run your own Matrix client; there are [numerous clients available](https://matrix.org/blog/try-matrix-now/). You can take Matrix.org's [reference client](https://github.com/matrix-org/matrix-angular-sdk) and use it as-is - or modify it any way you want! Since it's written in JavaScript, running a client is [really easy](https://github.com/matrix-org/matrix-angular-sdk#running)! + +| + +### Running your own homeserver: + +One of the core features of Matrix is that anyone can run a homeserver and join the federated network on equal terms (there is no hierarchy). If you want to set up your own homeserver, please see the relevant docs of the server you want to run. If you want to run Matrix.org's reference homeserver, please consult the [readme of the Synapse project](https://github.com/matrix-org/synapse/blob/master/README.rst). + +| + +Note that Synapse comes with a bundled Matrix.org webclient - but you can tell it to use your [development checkout snapshot instead](https://github.com/matrix-org/matrix-angular-sdk#matrix-angular-sdk) (or to not run a webclient at all via the "web_client: false" config option). + +| + + + +## Checkout our code - or write your own + +As described above, you can clone our code and [run a server and/or client yourself](#run). Infact, all the code that we at Matrix.org write is available from [our github](http://github.com/matrix-org) - and other servers and clients may also be open sourced - see [our list of all known Matrix-apps](https://matrix.org/blog/try-matrix-now/). + +| + +You can also implement your own client or server - after all, Matrix is at its core "just" a specification of a protocol. + +| + +### Write your own client: + +The [client-server API spec](http://matrix.org/docs/howtos/client-server.html) describes what API calls are available to clients, but a quick step-by-step guide would include: + +| + +1. Get a user either by registering your user in an existing client or running the [new-user script](https://github.com/matrix-org/synapse/blob/master/scripts/register_new_matrix_user) if you are running your own Synapse homeserver. + +2. Assuming the homeserver you are using allows logins by password, log in via the login API: +``` +curl -XPOST -d '{"type":"m.login.password", "user":"example", "password":"wordpass"}' "http://localhost:8008/_matrix/client/api/v1/login" +``` +3. If successful, this returns the following, including an `access_token`: + + { + "access_token": "QGV4YW1wbGU6bG9jYWxob3N0.vRDLTgxefmKWQEtgGd", + "home_server": "localhost", + "user_id": "@example:localhost" + } + +4. This ``access_token`` will be used for authentication for the rest of your API calls. Potentially the next step you want is to make a call to the initialSync API and get the last x events from each room your user is in (via the limit parameter): +``` +curl -XGET "http://localhost:8008/_matrix/client/api/v1/initialSync?limit=1&access_token=YOUR_ACCESS_TOKEN" +``` + +5. In addition to the last x events for all the rooms the user is interested in, this returns all the presences relevant for these rooms. Once you know which rooms the client has previously interacted with, you need to listen for incoming events. This can be done like so: +``` +curl -XGET "http://localhost:8008/_matrix/client/api/v1/events?access_token=YOUR_ACCESS_TOKEN" +``` + +6. This request will block waiting for an incoming event, timing out after several seconds if there is no event, returning something like this: + + { + "chunk": [], + "end": "s39_18_0", + "start": "s39_18_0" + } + +7. Even if there are no new events (as in the example above), there will be some pagination stream response keys. The client should make subsequent requests using the value of the "end" key (in this case s39_18_0) as the from query parameter e.g. +``` +http://localhost:8008/_matrix/client/api/v1/events?access _token=YOUR_ACCESS_TOKEN&from=s39_18_0 +``` + +8. This ensures that you only get new events. Now you have initial rooms and presence, and a stream of events - a good client should be able to process all these events and present them to the user. And potentially you might want to add functionality to generate events as well (such as messages from the user, for example) - again please consult the [client-server API spec](http://matrix.org/docs/howtos/client-server.html)! + +| + +### Write your own server: + +We are still working on the server-server spec, so the best thing to do if you are interested in writing a server, is to come talk to us in [#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org). + +If you are interested in how federation works, please see the [federation API chapter in the spec](http://matrix.org/docs/spec/#federation-api). + +| + + + +## Write an Application Service: + +Information about Application services and how they can be used can be found in the [Application services](./application_services.html) document! (This is based on Kegan's excellent blog posts, but now lives here so it can be kept up-to-date!) diff --git a/supporting-docs/guides/2015-08-19-faq.md b/supporting-docs/guides/2015-08-19-faq.md new file mode 100644 index 000000000..ff30827c7 --- /dev/null +++ b/supporting-docs/guides/2015-08-19-faq.md @@ -0,0 +1,593 @@ +--- +layout: post +title: FAQ +date: 2015-08-19 16:58:07 +categories: guides +--- + + +# FAQ +{:.no_toc} + +Categories +---------- +{:.no_toc} + +[General](#general) + +[Quick Start](#quick-start) + +[Standard](#standard) + +[Implementations](#implementations) + +| + +FAQ Content +----------- +{:.no_toc} + + +* TOC +{:toc .toc} + +### General + +##### What is Matrix? + +Matrix is an open standard for interoperable, decentralised, +real-time communication over IP. It can be used to power Instant +Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere +you need a standard HTTP API for publishing and subscribing to +data whilst tracking the conversation history. + +| + +Matrix defines the standard, and provides open source reference implementations +of Matrix-compatible Servers, Clients, Client SDKs and Application Services +to help you create new communication solutions or extend the capabilities +and reach of existing ones. + +##### What is Matrix's Mission? + +Matrix's initial goal is to fix the problem of fragmented IP communications: +letting users message and call each other without having to care what app +the other user is on - making it as easy as sending an email. + +| + +The longer term goal is for Matrix to act as a generic HTTP messaging and data +synchronisation system for the whole web - allowing people, services and devices +to easily communicate with each other, empowering users to own and control their +data and select the services and vendors they want to use. + +##### What does Matrix provide? + +Matrix provides: + +- [Open Standard](/docs/spec) HTTP APIs for transferring JSON messages (e.g. instant messages, WebRTC signalling), including: + - [Client\<-\>Server API](/docs/spec#client-server-api-v1) - defines how Matrix compatible clients communicate with Matrix homeservers. + - [Server\<-\>Server API](/docs/spec#federation-api) - defines how Matrix homeservers exchange messages and synchronise history with each other. + - [Application Service API](/docs/spec/#application-service-api) - defines how to extend the functionality of Matrix with 'integrations' and bridge to other networks. + - [Modules](/docs/spec/#modules) - specifies features that must be implemented by particular classes of clients. +- Open source reference implementations of: + - Clients (Web (React), iOS, Android) + - Client SDKs (Javascript, Web (React), iOS, Android) + - Homeservers (Synapse) + - Application Services (bridges to IRC, Slack, Skype, Lync and more...) +- The actual ecosystem and community of everyone running Matrix servers and services +- Loads of 3rd party contributions of clients, SDKs, servers and services. + +You can find the full list of Matrix enabled projects at https://matrix.org/blog/try-matrix-now. + +##### What does this mean for users? + +The aim is to provide an analogous ecosystem to email - one where you +can communicate with pretty much anyone, without caring what app or +server they are using, using whichever app & server you chose to use, +and use a neutral identity system like an e-mail address or phone +number to discover people to talk to. + +##### What kind of company is Matrix.org? + +Matrix is an open initiative which acts as a neutral custodian of the +Matrix standard. It's not actually incorporated anywhere at the moment +but we are looking at the best legal structure for the future (and as +of October 2015 we have hopefully found one). Whatever the legal +structure, we are committed to keeping the Matrix project open. + +##### Who is funding Matrix.org? + +Most of the current core contributors to Matrix work at +[Amdocs](http://amdocs.com), who have kindly given us permission to work +on Matrix as an independent non-profit initiative. Other contributors +are funded by their own employers or donate their own time to the project. + +##### Who is building Matrix? + +The core team is ~10 people with extensive experience in building custom +VoIP and Messaging apps for mobile network operators. Most of us have +day jobs at [Amdocs](http://amdocs.com) or [OpenMarket](http://openmarket.com), +but there are an increasing number of contributors from other companies and +folks all over the internet. + +##### Why are you called Matrix? + +We are called Matrix because we provide a structure in which all +communication can be matrixed together. + +| + +No, it's nothing to do with the film (although you could go and build virtual +worlds on top of Matrix if you wanted :) + +##### Why have you released this as open source? + +We believe that any open standard defining interoperable communication +needs to be justified, demonstrated and validated with transparent open +source implementations. For Matrix to achieve its mission of making all +communications services interoperable we believe it needs to be truly +open, giving people access to take all the code we produce and to use +and build on top of it. + +##### What do you mean by open? + +Matrix is an open standard, meaning that we have freely published the +details for how to communicate interoperably using the Matrix set of +HTTP APIs. We encourage anyone and everyone to use the APIs and build +their own projects which implement them and so benefit from +interoperability with the rest of the Matrix ecosystem. We also +ensure the standard is not encumbered by any known patent licensing +requirements. + +| + +Matrix is also open source, meaning that we have released the source +code of the reference servers, clients and services to the public domain +under the [Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to +encourage anyone and everyone to run their own servers and clients, and +enhance them and contribute their enhancements as they see fit. + +##### What does federated mean? + +Federation allows separate deployments of a communication service to +communicate with each other - for instance a mail server run by Google +federates with a mail server run by Microsoft when you send email from +@gmail.com to @hotmail.com. + +| + +Federation is different to interoperability, as interoperable clients +may simply be running on the same deployment - whereas in federation the +deployments themselves are exchanging data in a compatible manner. + +| + +Matrix provides open federation - meaning that anyone on the internet +can join into the Matrix ecosystem by deploying their own server. + +##### How is this like e-mail? + +When email was first set up in the early ‘80s, companies like Compuserve +and AT&T and Sprint set up isolated email communities which only allowed +you to exchange mail with users on the same system.  If you got your +email from one service and your friend from another, then you couldn't +message each other.  This is basically the situation we're in today with +VoIP and IM. + +##### Why has no-one done this before? + +There have been several attempts before including SIP, XMPP and RCS. + All of these have had some level of success, but many different +technological/usability/economic factors have ended up limiting their +success. Unfortunately, we've not ended up in a world where everyone +has a SIP URI or Jabber ID on their business card, or a phone that +actually uses RCS. + +##### What is the difference between Matrix and IRC? + +We love IRC.  In fact, as of today the core Matrix team still uses it as +our primary communication tool. Between us we've written IRCds, IRC bots +and admined dreamforge, UnrealIRCd, epona, ircservices and several +others. That said, it has some limitations that Matrix seeks to improve +on: + +- Text only +- No history +- No multiple-device support +- No presence support +- Fragmented identity model +- No open federation +- No standard APIs, just a rather limited TCP line protocol +- Non-standardised federation protocol +- No built-in end-to-end encryption +- Disruptive net-splits +- Non-extensible + +[IRCv3](http://ircv3.net) exists and is addressing some of these issues; +this is great news and we wish them well. It's almost a contradiction +in terms to get competitive between openly interoperable communication +projects - we look forward to increasing the richness of Matrix\<-\>IRC +bridges as the project progresses. + +##### What is the difference between Matrix and XMPP? + +The Matrix team used XMPP (Openfire, ejabberd, spectrum, asmack, +XMPPFramework) for IM before starting to experiment with open HTTP APIs +as an alternative in around 2012. The main issues with XMPP that +drove us in this direction were: + +- Not particularly web-friendly - you can't easily speak XMPP from a + web browser. (N.B. Nowadays you have options like XMPP-FTW and + Stanza.io that help loads with letting browsers talk XMPP). +- Single logical server per MUC is a single point of control and + availability. (MUCs can be distributed over multiple physical + servers, but they still sit behind a single logical JID and domain. + FMUC improves this with a similar approach to Matrix, but as of Oct + 2015 there are no open source implementations.) +- History synchronisation is very much a second class citizen feature +- Stanzas aren't framed or reliably delivered without extensions. (See + [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Four:_XMPP_is_unreliable_without_a_bunch_of_extensions.) + for an XMPP take on this) +- Multiple device support is limited. (Apparently Carbons and MAM help + with this) +- Baseline feature set is so minimal that fragmentation of features + between clients and servers is common, especially as interoperability + profiles for features have fallen behind (as of July 2015) +- No strong identity system (i.e. no standard E2E PKI, unless you + count X.509 certs, which [are + questionable](http://www.thoughtcrime.org/blog/ssl-and-the-future-of-authenticity/)) +- Not particularly well designed for mobile use cases: push; + bandwidth-efficient transports. (Since the time of writing a [Push + XEP has appeared](http://xmpp.org/extensions/xep-0357.html), and + [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Three:_It.27s_too_bandwidth-inefficient_for_mobile.) + claims that XMPP runs "fine" over a 9600bps + 30s latency link.) + +The whole subject of XMPP vs Matrix seems to bring out the worst in +people. Rather than fighting over which open interoperable communication +standard works the best, we should just collaborate and bridge everything +together. The more federation and interoperability the better. + +| + +We think of Matrix and XMPP as being quite different; at its core +Matrix can be thought of as an eventually consistent global JSON db with +an HTTP API and pubsub semantics - whilst XMPP can be thought of as a +message passing protocol. You can use them both to build chat systems; +you can use them both to build pubsub systems; each comes with different +tradeoffs. Matrix has a deliberately extensive 'kitchen sink' baseline +of functionality; XMPP has a deliberately minimal baseline set of +functionality. If XMPP does what you need it to do, then we're genuinely +happy for you :) Meanwhile, rather than competing, an XMPP Bridge like +[Skaverat's xmpptrix beta](https://github.com/SkaveRat/xmpptrix) or +[jfred's matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge) +or Matrix.org's own [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) +has potential to let both environments coexist and make the most of each +other's benefits. + +##### What is the difference between Matrix and PSYC? + +PSYC is a open federated messaging protocol loosely inspired by IRC.  In +version 1 it was a standalone protocol, and in version 2 it is being +reutilised as a messaging layer on top of GNUnet.  We honestly don't +know that much about it, beyond trying to use psycd as an XMPP\<-\>IRC +bridge in 2010. Matrix differentiates primarily by providing simple HTTP +APIs rather than the more exotic compact line protocol in PSYC v1 or the +comprehensive GNUnet stack in v2, and Matrix focuses more on decentralised +conversation history rather than just decentralised chat servers. +On the other hand, Matrix doesn't provide the metadata protection +guarantees that GNUnet/PSYC aims for. + +| + +See [http://about.psyc.eu/Matrix](http://about.psyc.eu/Matrix) for +PSYC's views on Matrix. + +##### What is the difference between Matrix and Tox? + +Tox.im looks to be a very cool clone of Skype - a fully decentralised +peer-to-peer network.  Matrix is deliberately not a 'pure' peer-to-peer +system; instead each user has a well-defined homeserver which stores +his data and that he can depend upon.  Matrix provides HTTP APIs; +Tox.im provides C APIs. As of October 2015 Tox doesn't seem to have an +answer yet for decentralised conversation history storage. + +##### How does Matrix compare with something like Trillian or Pidgin? + +Trillian and Pidgin and similar aggregating IM clients merge all your IM +activity into a single app.  However, your history and +identity is still fragmented across the networks.  People can't find you +easily, and your history is fragmented (other than on the device +where the client runs).   And rather than being able to chose the right +app for the job when communicating with people, you are pushed towards +relying on a specific aggregation app. + +Matrix lets you get the best of both worlds by linking to all the +different networks (XMPP, AIM, ICQ, Lync, Skype etc) on the serverside, +using bridges which can be run by anyone. Matrix then provides a simple +standard HTTP API to access any of these networks, and lets you choose +whichever client you prefer (either as a 'native' Matrix client or using +a non-Matrix client from one of the networks which has been bridged in). + +##### What Matrix compliant apps are there? + +Quite a few, ranging from the glossy mass-market to the geeky command-line. There's even an emacs macro. Check out [https://matrix.org/blog/try-matrix-now] for the current +list of Matrix enabled projects. + +##### What bridges to other networks are available? + +The number of 'bridges' which integrate existing communication networks into +Matrix are growing on a daily basis - both written by the Matrix core team +and contributed by the wider community. The full list can be seen at +https://matrix.org/blog/try-matrix-now, but the core ones as of Oct 2015 include: + + * [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge + * [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol + * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack + * [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) - lets you access any of the 20+ protocols supported by + [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including + Skype, Lync, XMPP, etc) + * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges + +Writing new bridges is incredibly fun and easy - see the [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) +for an example of how to write a fully functional Slack bridge in less than 100 lines of code! + +##### Why do you think existing apps will ever join this officially? + +We firmly believe it is what is right for the consumer. As people begin +to use interoperable communications tools, service providers will see the +benefit and compete on quality of service, security and features rather +than relying on locking people into their walled garden. We believe as +soon as users see the availability and benefits of interoperable +services they will demand it. + +##### Why aren't you doing this through the IETF? or W3C? or 3GPP? + +We do recognise the advantages of working with existing standards +bodies. We have been focused on writing code and getting it out, and the standard has been evolving rapidly since initial release in September 2014. +Once the standard has matured sufficiently it may well be appropriate to work with an official +standard body to maintain it going forwards. + +### Quick Start + +##### How do I get an account and get started? + +The quickest way is to pick a client from https://matrix.org/blog/try-matrix-now and sign up. +Please note that you can point clients to access any homeserver - you don't have to use matrix.org, +although as of day 1, matrix.org is the only communal homeserver +available. + +##### What can I actually do with this? + +A typical client provides a simple chatroom interface to Matrix - +letting the user interact with users and rooms anywhere within the +Matrix federation.  Text and image messages are supported, and basic +voice-only VoIP calling via WebRTC is supported in one-to-one rooms. +(As of October 2015, experimental multi-way calling is also available +on Vector.im). + +##### How do I connect my homeserver to the public Matrix network? + +See +[http://github.com/matrix-org/synapse](http://github.com/matrix-org/synapse) +for details + +##### How do I Matrix-enable my existing app? + +If your app doesn't have any communication capability already, you'll want +to use one of the Matrix client SDKs to add it in. These come in different +levels of sophistication - ranging from a simple HTTP API wrapper (like matrix-js-sdk, matrix-ios-sdk or matrix-android-sdk) +through to reusable UI components (like matrix-react-sdk and matrix-ios-kit). Pick +the one for your platform, or a 3rd party one if none of the above work for you, +and get plugging it in. You'll probably also want to read the [Client-Server API +HOWTO](http://matrix.org/docs/howtos/client-server.html) too. + +If you already have communication infrastructure set up (XMPP, custom HTTP, or whatever), +then you'll want to run a bridge to expose it to the wider Matrix ecosystem. +See [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) for a +guide of how to write bridges using the matrix-appservice-bridge framework, or co-opt one +from the list at https://matrix.org/blog/try-matrix-now. +[Application Service API](/docs/spec/#application-service-api) gives the details of the API +that bridges have to implement. + +##### How can I write a client on Matrix? + +See the [Client-Server API +HOWTO](http://matrix.org/docs/howtos/client-server.html) and the [API +docs](/docs/api) and [the Spec](/docs/spec) for all the details you need +to write a client. + +##### How can I help out with this? + +Come say hi on #matrix:matrix.org! Install synapse and tell us how you get on. Critique the spec.  Write +clients. Write bridges! Run bridges! Nose around in [Jira](https://matrix.org/jira) and +send us some pull requests on github to fix some bugs or add some features! You could even +try to write a homeserver (but be warned, Matrix's architecture makes homeservers orders of +magnitude harder than clients or bridges.) + +See [CONTRIBUTING.rst](http://github.com/matrix-org/synapse/tree/master/CONTRIBUTING.rst) for +full details on how to contribute to the project. All are welcome! + +##### Where can I get support? + +\#matrix:matrix.org aka \#matrix on irc.freenode.is your best bet. + +##### How do I register custom matrix event types? + +We're not yet managing a registry of custom matrix event types.  If you +have any particularly good ones you want to tell the world about, please +use the [mailing list](/mailman/listinfo/matrix-users) for now. + +##### How mature is this? + +We started working on Matrix in July 2014, and opened it to the +public in September 2014. We got all the core features in place in December 2014 +and entered beta, and since then have been iterating away on the beta refining the +architecture and APIs, fixing bugs and scalability, and adding new features, clients, +bridges etc. + +As of October 2015 (synapse 0.10) it's good for serious experimentation and +non-production services and can absolutely be used in the real world. However, we're +still in beta and we'll want to freeze the spec and implement clustering and other +nice features before we really declare it ready for production. + +### Standard + +##### What is a client? + +Users in Matrix use one or more clients to communicate. This could be any combination of a web client, a command line client, a mobile client - or embedded clients built into existing apps. It could even be a piece of hardware (e.g. a drone) that is Matrix enabled. + +##### Can I use Matrix without installing a Matrix client? + +Sure. An ever increasing number of protocols are being bridged into Matrix, so if you use something like IRC on Freenode you may well be indirectly benefiting from Matrix, as others may be connected into the IRC channel via Matrix. + +##### What is a home server? + +A user's clients connect to a single homeserver, which stores the communication history and account information for that user, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers. + +##### What is an identity server? + +Users in Matrix are identified internally via their matrix user ID (MXID). However, existing 3rd party ID (3PID) namespaces such as email addresses or phone numbers should be used publically to identify Matrix users, at least for invitation purposes. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces linked to their account. + +| + +Matrix users can link third-party IDs (3PIDs) to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts. + +| + +In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. +Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. + +| + +The precise architecture of identity servers is currently in flux and subject to change as we work to fully decentralise them. + +##### Where do my conversations get stored? + +Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the context of a virtual room. Room data is replicated across all of the homeservers *whose users are participating in a given room*. + +##### What is a 3PID? + +Third-party IDs (3PIDs) are IDs from other systems or contexts, such as email addresses, social network accounts and phone numbers. + +##### How do you do VoIP calls on Matrix? + +Voice (and video) over Matrix uses the WebRTC 1.0 standard to transfer call media (i.e. the actual voice and video traffic). Matrix is used to signal the establishment and termination of the call by sending call events, like any other event. Currently calls are only supported in rooms with exactly two participants - however, one of those participants may be a conferencing bridge. We're looking at better ways to do group calling. + +##### Can I log into other homeservers with my username and password? + +Currently, no. We are looking at options for decentralising or migrating user accounts between multiple servers, and might add this feature at a later stage. + +##### Why Apache Licence? + +The Apache Licence is a permissive licence. We want the Matrix protocol itself to be free and open, but people are free to create both free and commercial apps and services that uses the protocol. In our opinion, any Matrix-service only enhances the Matrix ecosystem. + +##### Can I write a Matrix homeserver? + +Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that as of October 2015, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in #matrix:matrix.org first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst). + +##### How secure is this? + +Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Servers maintain a public/private key pair, and sign the integrity of all messages in the context of the historical conversation, preventing tampering. Server keys are distributed using a PERSPECTIVES-style system. + +End-to-end encryption is coming shortly to clients for both 1:1 and group chats to protect user data stored on servers, using the [Olm](https://matrix.org/git/olm) cryptographic ratchet implementation. As of October 2015 this is blocked on implementing the necessary key distribution and fingerprint management. + +Privacy of metadata is not currently protected from server administrators - a malicious homeserver administrator can see who is talking to who and when, but not what is being said (once E2E encryption is enabled). See [this presentation from Jardin Entropique](http://matrix.org/~matthew/2015-06-26%20Matrix%20Jardin%20Entropique.pdf) for a more comprehensive discussion of privacy in Matrix. + +### Implementations + +##### What is Synapse? + +Synapse is a reference "homeserver" implementation of Matrix from the core development team at matrix.org, written in Python 2/Twisted. It is intended to showcase the concept of Matrix and let folks see the spec in the context of a codebase and let you run your own homeserver and generally help bootstrap the ecosystem. + +##### How do I join the global Matrix federation? + +You can download and run one of the available Matrix servers - please see [this guide](http://matrix.org/docs/guides/getting_involved.html#run) for details! + +##### What ports do I have to open up to join the global Matrix federation? + +We recommend servers use port 8448 for server\<-\>server HTTPS traffic. Look at ["Setting up Federation"](https://github.com/matrix-org/synapse#setting-up-federation) in the Synapse readme file for details. + +Client\<-\>Server traffic can talk directly to Synapse via port 8448, but as by default Synapse creates a self-signed TLS certificate this can cause problems for clients which can't easily trust self-signed certificates (e.g. most web browsers). Instead, you can proxy access to Synapse's HTTP listener on port 8008 via an existing HTTPS proxy with a valid certificate (e.g. an nginx listening on port 443), or you can point Synapse at a valid X.509 signed TLS certificate. In future, Synapse will probably use letsencrypt to autogenerate valid certificates rather than self-signed ones during installation, simplifying this process enormously. + +You can also put Synapse entirely behind an existing TLS load balancer and not expose port 8448 at all. In this situation, Synapse will need to be configured to share the same *public* TLS certificate as the load balancer (as Synapse uses the public certificate for identity in other areas too, and it has to match the certificate that other servers see when they connect). + +##### How do I run my own homeserver? + +Follow the instructions for the homeserver you want to run. If you want to run Synapse, the reference homeserver from Matrix.org, follow [these instructions](https://github.com/matrix-org/synapse#synapse-installation). + +##### Can I run my own identity server? + +Yes - the reference implementation is +[sydent](https://github.com/matrix-org/sydent) and you can run your own ID server cluster that tracks 3rd party to Matrix ID mappings. This won't be very useful right now, though, and we don't recommend it. + +If you want your server to participate in the global replicated Matrix ID +service then please get in touch with us. Meanwhile, we are looking at +ways of decentralising the 'official' Matrix identity service so that +identity servers are 100% decentralised and can openly federate with +each other. **N.B. that you can use Matrix without ever using the +identity service - it exists only to map 3rd party IDs (e.g. email +addresses) to matrix IDs to aid user discovery**. + +##### What are Synapse's platform requirements? + +Synapse will use as much RAM as you give it in order to cache conversations in RAM to avoid hitting the database. For small deployments (<50 active users) around 512MB of RAM is probably okay. You can configure the amount of RAM used by synapse with the event_cache_size config parameter - the more events in the cache, the more RAM required. Synapse itself requires relatively little diskspace other than for logging (which as of October 2015 is quite verbose for debugging purposes), but as it caches the content of all the file attachments (images, videos etc) viewed by its users, you may need to size storage appropriately. Synapse is currently effectively single threaded, and will never use more than 1 core. + +| + +For better performance, one should back Synapse with a Postgres database rather than the default SQLite - see https://github.com/matrix-org/synapse/tree/master/README.rst#using-postgresql for details. + +##### Why is Synapse in Python/Twisted? + +This is because both provide a mature and well known event-driven async IO framework for writing serverside code. Whilst this has been okay for our initial experimentation and proof of concept, it's likely that future homeserver work will be written in a more strongly typed language (e.g. Go). + +##### Why aren't you using an ORM layer like SqlAlchemy in Synapse? + +Synapse is *very* database dependent (as of Oct 2015; this is improving in the near future however), and we like having the flexibility to sculpt our own queries. + +##### Where can I find a mobile app? + +The "Matrix Console" reference apps (ugly, geeky and powerful - intended for early adopter powerusers) can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=org.matrix.androidsdk.alpha) + and [Apple store](https://itunes.apple.com/gb/app/matrix-console/id970074271). + +| + +For the Android app, you can also install the latest development version +built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastBuild/artifact/console/build/outputs/apk/console-alpha-debug.apk). + +##### Where can I find a web app? + +As of Oct 2015, the best web app options are to use [Vector.im](https://vector.im) - a glossy web client written on top of [matrix-react-sdk](https://github.com/matrix-org/matrix-react-sdk), or the original [AngularJS based client](https://matrix.org/beta), which has serious performance problems and is not currently being maintained. In future a "Matrix Console" reference web app built on matrix-react-sdk will be released by matrix.org to complement the mobile apps above. + +| + +### QUESTIONS TO BE ANSWERED! + +This FAQ is a constant work in progress - patches and pull requests are *very* welcome to help us improve it. Some of the frequent questions where we need to write an answer include: + + * How do I rename servers? + * How do I change the TLS key of my server? + * How do I maintain my synapse's DB (e.g. prune old conversations)? + * How do I maintain my synapse's content repository (e.g. prune old content)? + * What are redactions? + * Why is the spec so big, especially relative to the XMPP baseline spec? + * How do I contribute to the spec? + * What is the privacy policy on Matrix.org? + * How precisely does E2E work? + * How does Matrix actually work architecturally? + * What IOT use cases are there for Matrix? + * Why is are the Matrix reference implementations written in so many different languages? + * How does push work? + * What's on the roadmap? + * How can I use Matrix to talk on Freenode or other IRC networks? + * Where can I learn more about Matrix? (link to PDFs of other presentations etc) + * Why HTTP? Doesn't HTTP suck? + * Why don't you use websockets? + * Why is synapse so resource intensive immediately after federating for the first time? + * \[your question goes here...\] + +| + +Any other questions? Please contact us in +[\#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org) or the [mailing +lists](/mailman/listinfo/matrix-users)! diff --git a/supporting-docs/guides/2015-08-21-application_services.md b/supporting-docs/guides/2015-08-21-application_services.md new file mode 100644 index 000000000..dd532d423 --- /dev/null +++ b/supporting-docs/guides/2015-08-21-application_services.md @@ -0,0 +1,143 @@ +--- +layout: post +title: Application services +categories: guides +--- + +# Application services + +Application services are distinct modules which which sit alongside a home server providing arbitrary extensible functionality decoupled from the home server implementation. Just like the rest of Matrix, they communicate via HTTP using JSON. Application services function in a very similar way to traditional clients, but they are given much more power than a normal client. They can reserve entire namespaces of room aliases and user IDs for their own purposes. They can silently monitor events in rooms, or any events directed at any user ID. This power allows application services to have extremely useful abilities which overall enhance the end user experience. + +| + +One of the main use cases for application services is protocol bridges. Our Matrix server on Matrix.org links in to various IRC channels and networks. This functionality was initially implemented as a simple bot which resided as a user on the Matrix rooms we wanted to link to freenode channels (#matrix, #matrix-dev, #openwebrtc and #vuc etc). There was nothing special about this bot; it is just treated as a client. However, as we started to rely on it more and more though, we realised that there were things that were impossible for simple client-side bots to do by themselves - for example, the bot could not reserve the virtual user IDs it wanted to create, and could not lazily bridge arbitrary IRC rooms on-the-fly - and this spurred the development of Application Services. + +| + +### Some of the features of the IRC application service we have since implemented include: + +- Specific channel-to-matrix room bridging : This is what the original IRC bot did. You can specify specific channels and specific room IDs, and messages will be bridged. +- Dynamic channel-to-matrix room bridging : This allows Matrix users to join any channel on an IRC network, rather than being forced to use one of the specific channels configured. +- Two-way PM support : IRC users can PM the virtual "M-" users and private Matrix rooms will be created. Likewise, Matrix users can invite the virtual "@irc_Nick:domain" user IDs to a room and a PM to the IRC nick will be made. +- IRC nick changing support: Matrix users are no longer forced to use "M-" nicks and can change them by sending "!nick" messages directly to the bridge. +- Ident support: This allows usernames to be authenticated for virtual IRC clients, which means IRC bans can be targeted at the Matrix user rather than the entire application service. + +| + +### The use of the Application Services API means: + +- The bot can reserve user IDs. This prevents humans from registering for @irc_... user IDs which would then clash with the operation of the bot. +- The bot can reserve room aliases. This prevents humans from register for #irc_... aliases which would then clash with the operation of the bot. +- The bot can trivially manage hundreds of users. Events are pushed to the application service directly. If you tried to do this as a client-side bot, you would need one event stream connection per virtual user. +- The bot can lazily create rooms on demand. This means Matrix users can join non-existent room aliases and have the application service quickly track an IRC channel and create a room with that alias, allowing the join request to succeed. + +| + +### Implementation details: + +- Written in Node.js, designed to be run using forever. +- Built on the generic matrix-appservice-node framework. +- Supports sending metrics in statsd format. +- Uses matrix-appservice-node to provide a standardised interface when writing application services, rather than an explicit web framework (though under the hood matrix-appservice-node is using Express). + +| + +At present, the IRC application service is in beta, and is being run on #matrix and #matrix-dev. If you want to give it a go, check it out on Github! + +| + +# What Application services can do for you +Application services have enormous potential for creating new and exciting ways to transform and enhance the core Matrix protocol. For example, you could aggregate information from multiple rooms into a summary room, or create throwaway virtual user accounts to proxy messages for a fixed user ID on-the-fly. As you may expect, all of this power assumes a high degree of trust between application services and home servers. Only home server admins can allow an application service to link up with their home server, and the application service is in no way federated to other home servers. You can think of application services as additional logic on the home server itself, without messing around with the book-keeping that home servers have to do. This makes adding useful functionality very easy. + +| + +### Example + +The application service (AS) API itself uses webhooks to communicate from the home server to the AS: + +- Room Alias Query API : The home server hits a URL on your application server to see if a room alias exists. +- User Query API : The home server hits a URL on your application server to see if a user ID exists. +- Push API : The home server hits a URL on your application server to notify you of new events for your users and rooms. + +A very basic application service may want to log all messages in rooms which have an alias starting with "#logged_" (side note: logging won't work if these rooms are using end-to-end encryption). + +Here's an example of a very basic application service using Python (with Flask and Requests) which logs room activity: + + # app_service.py: + + import json, requests  # we will use this later + from flask import Flask, jsonify, request + app = Flask(__name__) + + @app.route("/transactions/<transaction>", methods=["PUT"]) + def on_receive_events(transaction): + events = request.get_json()["events"] + for event in events: + print "User: %s Room: %s" % (event["user_id"], event["room_id"]) + print "Event Type: %s" % event["type"] + print "Content: %s" % event["content"] + return jsonify({}) + + if __name__ == "__main__": + app.run() + +Set your new application service running on port 5000 with: + + python app_service.py + +The home server needs to know that the application service exists before it will send requests to it. This is done via a registration YAML file which is specified in Synapse's main config file e.g. homeserver.yaml. The server admin needs to add the application service registration configuration file as an entry to this file. + + # homeserver.yaml + app_service_config_files: + - "/path/to/appservice/registration.yaml" + +NB: Note the "-" at the start; this indicates a list element. The registration file registration.yaml should look like: + + # registration.yaml + + # this is the base URL of the application service + url: "http://localhost:5000" + + # This is the token that the AS should use as its access_token when using the Client-Server API + # This can be anything you want. + as_token: wfghWEGh3wgWHEf3478sHFWE + + # This is the token that the HS will use when sending requests to the AS. + # This can be anything you want. + hs_token: ugw8243igya57aaABGFfgeyu + + # this is the local part of the desired user ID for this AS (in this case @logging:localhost) + sender_localpart: logging + namespaces: + users: [] + rooms: [] + aliases: + - exclusive: false + regex: "#logged_.*" + +**You will need to restart the home server after editing the config file before it will take effect.** + +| + +To test everything is working correctly, go ahead and explicitly create a room with the alias "#logged_test:localhost" and send a message into the room: the HS will relay the message to the AS by PUTing to /transactions/<tid> and you should see your AS print the event on the terminal. This will monitor any room which has an alias prefix of "#logged_", but it won't lazily create room aliases if they don't already exist. This means it will only log messages in the room you created before: #logged_test:localhost. Try joining the room "#logged_test2:localhost" without creating it, and it will fail. Let's fix that and add in lazy room creation: + + @app.route("/rooms/<alias>") + def query_alias(alias): + alias_localpart = alias.split(":")[0][1:] + requests.post( + # NB: "TOKEN" is the as_token referred to in registration.yaml + "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token=TOKEN", + json.dumps({ + "room_alias_name": alias_localpart + }), + headers={"Content-Type":"application/json"} + ) + return jsonify({}) + +This makes the application service lazily create a room with the requested alias whenever the HS queries the AS for the existence of that alias (when users try to join that room), allowing any room with the alias prefix #logged_ to be sent to the AS. Now try joining the room "#logged_test2:localhost" and it will work as you'd expect.  You can see that if this were a real bridge, the AS would have checked for the existence of #logged_test2 in the remote network, and then lazily-created it in Matrix as required. + +| + +Application services are powerful components which extend the functionality of home servers, but they are limited. They can only ever function in a "passive" way. For example, you cannot implement an application service which censors swear words in rooms, because there is no way to prevent the event from being sent. Aside from the fact that censoring will not work when using end-to-end encryption, all federated home servers would also need to reject the event in order to stop developing an inconsistent event graph. To "actively" monitor events, another component called a "Policy Server" is required, which is beyond the scope of this post.  Also, Application Services can result in a performance bottleneck, as all events on the homeserver must be ordered and sent to the registered application services.  If you are bridging huge amounts of traffic, you may be better off having your bridge directly talk the Server-Server federation API rather than the simpler Application Service API. + +I hope this demonstrates how easy it is to create an application service, along with a few ideas of the kinds of things you can do with them. Obvious uses include build protocol bridges, search engines, invisible bots, etc. For more information on the AS HTTP API, check out the new Application Service API section in the spec, or the raw drafts and spec in https://github.com/matrix-org/matrix-doc/. diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 11c34fb3a..c18c9a5fa 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -16,7 +16,7 @@ class Sections(object): def log(self, text): if self.debug: - print text + print "batesian:sections: %s" % text def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -27,12 +27,38 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if not isinstance(section, basestring): + if isinstance(section, basestring): + if section_key in section_dict: + raise Exception( + ("%s : Section %s already exists. It must have been " + + "generated dynamically. Check which render_ methods " + + "return a dict.") % + (func_name, section_key) + ) + section_dict[section_key] = section + self.log( + " Generated. Snippet => %s" % section[:60].replace("\n","") + ) + elif isinstance(section, dict): + self.log(" Generated multiple sections:") + for (k, v) in section.iteritems(): + if not isinstance(k, basestring) or not isinstance(v, basestring): + raise Exception( + ("Method %s returned multiple sections as a dict but " + + "expected the dict elements to be strings but they aren't.") % + (func_name, ) + ) + if k in section_dict: + raise Exception( + "%s tried to produce section %s which already exists." % + (func_name, k) + ) + section_dict[k] = v + self.log( + " %s => %s" % (k, v[:60].replace("\n","")) + ) + else: raise Exception( - "Section function '%s' didn't return a string!" % func_name + "Section function '%s' didn't return a string/dict!" % func_name ) - section_dict[section_key] = section - self.log( - " Generated. Snippet => %s" % section[:60].replace("\n","") - ) return section_dict \ No newline at end of file diff --git a/templating/batesian/units.py b/templating/batesian/units.py index c20a2d1f2..144cf245a 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,11 @@ class Units(object): def log(self, text): if self.debug: - print text + func_name = "" + trace = inspect.stack() + if len(trace) > 1 and len(trace[1]) > 2: + func_name = trace[1][3] + ":" + print "batesian:units:%s %s" % (func_name, text) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/build.py b/templating/build.py index dadf91ce5..b91e1da20 100755 --- a/templating/build.py +++ b/templating/build.py @@ -38,7 +38,7 @@ Processing """ from batesian import AccessKeyStore -from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template +from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta from argparse import ArgumentParser, FileType import importlib import json @@ -52,8 +52,8 @@ def create_from_template(template, sections): def check_unaccessed(name, store): unaccessed_keys = store.get_unaccessed_set() if len(unaccessed_keys) > 0: - print "Found %s unused %s keys." % (len(unaccessed_keys), name) - print unaccessed_keys + log("Found %s unused %s keys." % (len(unaccessed_keys), name)) + log(unaccessed_keys) def main(input_module, file_stream=None, out_dir=None, verbose=False): if out_dir and not os.path.exists(out_dir): @@ -78,8 +78,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): def wrap(input, wrap=80, initial_indent=""): if len(input) == 0: return initial_indent + # TextWrapper collapses newlines into single spaces; we do our own + # splitting on newlines to prevent this, so that newlines can actually + # be intentionally inserted in text. + input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) - return wrapper.fill(input) + output_lines = [wrapper.fill(line) for line in input_lines] + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters env = Environment( @@ -116,17 +121,31 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return # check the input files and substitute in sections where required - print "Parsing input template: %s" % file_stream.name - temp = Template(file_stream.read()) - print "Creating output for: %s" % file_stream.name + log("Parsing input template: %s" % file_stream.name) + temp_str = file_stream.read().decode("utf-8") + # do sanity checking on the template to make sure they aren't reffing things + # which will never be replaced with a section. + ast = env.parse(temp_str) + template_vars = meta.find_undeclared_variables(ast) + unused_vars = [var for var in template_vars if var not in sections] + if len(unused_vars) > 0: + raise Exception( + "You have {{ variables }} which are not found in sections: %s" % + (unused_vars,) + ) + # process the template + temp = Template(temp_str) + log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: - f.write(output) - print "Output file for: %s" % file_stream.name + f.write(output.encode("utf-8")) + log("Output file for: %s" % file_stream.name) check_unaccessed("units", units) +def log(line): + print "batesian: %s" % line if __name__ == '__main__': parser = ArgumentParser( @@ -170,7 +189,7 @@ if __name__ == '__main__': sys.exit(0) if not args.file: - print "No file supplied." + log("No file supplied.") parser.print_help() sys.exit(1) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 7bc250952..e75a75af8 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,28 +72,40 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def render_profile_http_api(self): - return self._render_http_api_group( - "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" - ) - def render_sync_http_api(self): - return self._render_http_api_group( - "sync" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_http_apis(self): + # map all swagger_apis to the form $GROUP_http_api + swagger_groups = self.units.get("swagger_apis").keys() + renders = {} + for group in swagger_groups: + sortFnOrPathList = None + if group == "presence": + sortFnOrPathList = ["status"] + elif group == "profile": + sortFnOrPathList=["displayname", "avatar_url"] + renders[group + "_http_api"] = self._render_http_api_group( + group, sortFnOrPathList + ) + return renders - def render_presence_http_api(self): - return self._render_http_api_group( - "presence", - sortFnOrPathList=["status"], - title_kind="~" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_events(self): + # map all event schemata to the form $EVENTTYPE_event with s/./_/g + # e.g. m_room_topic_event + schemas = self.units.get("event_schemas") + renders = {} + for event_type in schemas: + renders[event_type.replace(".", "_") + "_event"] = self._render_events( + lambda x: x == event_type, sorted + ) + return renders def render_room_events(self): def filterFn(eventType): @@ -102,6 +119,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -117,7 +137,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -138,12 +159,17 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") ce_types = self.units.get("common_event_fields") - return template.render(common_event=ce_types[type]) + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + return template.render( + common_event=ce_types[type], title_kind=subtitle_title_char + ) def render_common_event_fields(self): return self._render_ce_type("event") @@ -153,3 +179,4 @@ class MatrixSections(Sections): def render_common_state_event_fields(self): return self._render_ce_type("state_event") + diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 2a3f7ec5d..3f16be3da 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,5 +1,5 @@ {{common_event.title}} Fields -{{(7 + common_event.title | length) * '-'}} +{{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index ffae59e0f..fb876440c 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -7,20 +7,22 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== +======================= ================= =========================================== + {{table.title or "Content"}} Key Type Description +======================= ================= =========================================== {% for row in table.rows -%} {# -#} {# Row type needs to prepend spaces to line up with the type column (19 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} {# It also needs to then wrap inside the desc col (43 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} {% endfor -%} -================== ================= =========================================== +======================= ================= =========================================== {% endfor %} -Example:: +Example: + +.. code:: json {{example | jsonify(4, 4)}} diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 5e80f1686..86eacb145 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,5 +1,10 @@ ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} +{% if "alias_for_path" in endpoint -%} +``{{endpoint.path}}`` is an alias for `{{endpoint.alias_for_path}}`_. + +.. _`{{endpoint.alias_for_path}}`: #{{endpoint.alias_link}} +{% else -%} {{endpoint.desc | wrap(80)}} @@ -8,17 +13,17 @@ Request format: -================== ================= =========================================== - Parameter Value Description -================== ================= =========================================== +=========================================== ================= =========================================== + Parameter Value Description +=========================================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* --------------------------------------------------------------------------------- +--------------------------------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(19-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(37)}} +{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} {% endfor -%} {% endfor -%} -================== ================= =========================================== +=========================================== ================= =========================================== {% if endpoint.res_tables|length > 0 -%} Response format: @@ -26,26 +31,44 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - Param Type Description -================== ================= =========================================== +======================= ========================= ========================================== + Param Type Description +======================= ========================= ========================================== {% for row in table.rows -%} {# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} +{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} +{# It also needs to then wrap inside the desc col (42 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} {% endfor -%} -================== ================= =========================================== +======================= ========================= ========================================== {% endfor %} {% endif -%} -Example request:: +Example request: + +.. code:: http {{endpoint.example.req | indent_block(2)}} -Example response:: +{% if endpoint.example.responses|length > 0 -%} +Response{{"s" if endpoint.example.responses|length > 1 else "" }}: + +{% endif -%} + +{% for res in endpoint.example.responses -%} + +**Status code {{res["code"]}}:** - {{endpoint.example.res | indent_block(2)}} \ No newline at end of file +{{res["description"]}} + +Example + +.. code:: json + + {{res["example"] | indent_block(2)}} + +{% endfor %} +{% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160d..f78624517 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index d607160fd..be113b9c0 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -1,4 +1,12 @@ -"""Contains all the units for the spec.""" +""" +Contains all the units for the spec. + +This file loads swagger and JSON schema files and parses out the useful bits +and returns them as Units for use in Batesian. + +For the actual conversion of data -> RST (including templates), see the sections +file instead. +""" from batesian.units import Units import inspect import json @@ -8,6 +16,18 @@ import subprocess import urllib import yaml +V1_CLIENT_API = "../api/client-server/v1" +V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" +V1_EVENT_SCHEMA = "../event-schemas/schema/v1" +V2_CLIENT_API = "../api/client-server/v2_alpha" +CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" + +ROOM_EVENT = "core-event-schema/room_event.json" +STATE_EVENT = "core-event-schema/state_event.json" + + def get_json_schema_object_fields(obj, enforce_title=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -30,8 +50,24 @@ def get_json_schema_object_fields(obj, enforce_title=False): } tables = [fields] - props = obj.get("properties") parents = obj.get("allOf") + props = obj.get("properties") + if not props: + props = obj.get("patternProperties") + if props: + # try to replace horrible regex key names with pretty x-pattern ones + for key_name in props.keys(): + pretty_key = props[key_name].get("x-pattern") + if pretty_key: + props[pretty_key] = props[key_name] + del props[key_name] + if not props and not parents: + # Sometimes you just want to specify that a thing is an object without + # doing all the keys. Allow people to do that if they set a 'title'. + if obj.get("title"): + parents = [{ + "$ref": obj.get("title") + }] if not props and not parents: raise Exception( "Object %s has no properties or parents." % obj @@ -51,10 +87,20 @@ def get_json_schema_object_fields(obj, enforce_title=False): if props[key_name]["type"] == "object": if props[key_name].get("additionalProperties"): # not "really" an object, just a KV store - value_type = ( - "{string: %s}" % - props[key_name]["additionalProperties"]["type"] - ) + prop_val = props[key_name]["additionalProperties"]["type"] + if prop_val == "object": + nested_object = get_json_schema_object_fields( + props[key_name]["additionalProperties"], + enforce_title=True + ) + key = props[key_name]["additionalProperties"].get( + "x-pattern", "string" + ) + value_type = "{%s: %s}" % (key, nested_object[0]["title"]) + if not nested_object[0].get("no-table"): + tables += nested_object + else: + value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( props[key_name], @@ -75,6 +121,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables += nested_object else: value_type = "[%s]" % props[key_name]["items"]["type"] + array_enums = props[key_name]["items"].get("enum") + if array_enums: + if len(array_enums) > 1: + value_type = "[enum]" + desc += ( + " One of: %s" % json.dumps(array_enums) + ) + else: + desc += ( + " Must be '%s'." % array_enums[0] + ) else: value_type = props[key_name]["type"] if props[key_name].get("enum"): @@ -87,6 +144,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): desc += ( " Must be '%s'." % props[key_name]["enum"][0] ) + if isinstance(value_type, list): + value_type = " or ".join(value_type) fields["rows"].append({ "key": key_name, @@ -105,21 +164,23 @@ class MatrixUnits(Units): for path in api["paths"]: for method in api["paths"][path]: single_api = api["paths"][path][method] + full_path = api.get("basePath", "").rstrip("/") + path endpoint = { "title": single_api.get("summary", ""), "desc": single_api.get("description", single_api.get("summary", "")), "method": method.upper(), - "path": api.get("basePath", "") + path, + "path": full_path.strip(), "requires_auth": "security" in single_api, "rate_limited": 429 in single_api.get("responses", {}), "req_params": [], "res_tables": [], "example": { "req": "", - "res": "" + "responses": [], + "good_response": "" } } - self.log(".o.O.o. Endpoint: %s %s" % (method, path)) + self.log(" ------- Endpoint: %s %s ------- " % (method, path)) for param in single_api.get("parameters", []): # description desc = param.get("description", "") @@ -128,12 +189,24 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + + if param.get("enum"): + val_type = "enum" + desc += ( + " One of: %s" % json.dumps(param.get("enum")) + ) + refType = Units.prop(param, "schema/$ref/") # Error,Event schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads if not val_type and refType: val_type = refType # TODO: Resolve to human-readable. if not val_type and schemaFmt: val_type = schemaFmt + # handle top-level strings/bools + if not val_type and Units.prop(param, "schema/type") == "string": + val_type = "string" + if not val_type and Units.prop(param, "schema/type") == "boolean": + val_type = "boolean" if val_type: endpoint["req_params"].append({ "key": param["name"], @@ -147,8 +220,8 @@ class MatrixUnits(Units): # object with some keys; we'll add entries f.e one) if "schema" not in param: raise Exception( - "API endpoint group=%s path=%s method=%s param=%s"+ - " has no valid parameter value." % ( + ("API endpoint group=%s path=%s method=%s param=%s"+ + " has no valid parameter value.") % ( group_name, path, method, param ) ) @@ -160,14 +233,63 @@ class MatrixUnits(Units): ) # loop top-level json keys json_body = Units.prop(param, "schema/properties") + required_params = [] + if Units.prop(param, "schema/required"): + required_params = Units.prop(param, "schema/required") for key in json_body: + req_obj = json_body[key] + pdesc = req_obj["description"] + if key in required_params: + pdesc = "**Required.** " + pdesc + + is_array = req_obj["type"] == "array" + is_array_of_objects = ( + is_array and req_obj["items"]["type"] == "object" + ) endpoint["req_params"].append({ "key": key, "loc": "JSON body", - "type": json_body[key]["type"], - "desc": json_body[key]["description"] + "type": ( + req_obj["type"] if not is_array else + "array[%s]" % req_obj["items"]["type"] + ), + "desc": pdesc }) + if not is_array_of_objects and req_obj["type"] == "array": + continue + # Put in request.dot.notation for nested keys + if req_obj["type"] in ["object", "array"]: + if is_array_of_objects: + req_obj = req_obj["items"] + + req_tables = get_json_schema_object_fields(req_obj) + + if req_tables > 1: + for table in req_tables[1:]: + nested_key_name = [ + s["key"] for s in req_tables[0]["rows"] if + s["type"] == ("{%s}" % (table["title"],)) + ][0] + for row in table["rows"]: + row["key"] = "%s.%s" % (nested_key_name, row["key"]) + + key_sep = "[0]." if is_array else "." + for table in req_tables: + if table.get("no-table"): + continue + for row in table["rows"]: + nested_key = key + key_sep + row["key"] + endpoint["req_params"].append({ + "key": nested_key, + "loc": "JSON body", + "type": row["type"], + "desc": row["req_str"] + row["desc"] + }) + # endfor[param] + for row in endpoint["req_params"]: + self.log("Request parameter: %s" % row) + # group params by location to ease templating endpoint["req_param_by_loc"] = { # path: [...], query: [...], body: [...] @@ -177,11 +299,19 @@ class MatrixUnits(Units): endpoint["req_param_by_loc"][p["loc"]] = [] endpoint["req_param_by_loc"][p["loc"]].append(p) - # add example response if it has one - res = single_api["responses"][200] # get the 200 OK response - endpoint["example"]["res"] = res.get("examples", {}).get( - "application/json", "" - ) + good_response = None + for code, res in single_api.get("responses", {}).items(): + if not good_response and code == 200: + good_response = res + description = res.get("description", "") + example = res.get("examples", {}).get("application/json", "") + if description and example: + endpoint["example"]["responses"].append({ + "code": code, + "description": description, + "example": example, + }) + # form example request if it has one. It "has one" if all params # have either "x-example" or a "schema" with an "example". params_missing_examples = [ @@ -191,7 +321,7 @@ class MatrixUnits(Units): ) ] if len(params_missing_examples) == 0: - path_template = api.get("basePath", "") + path + path_template = api.get("basePath", "").rstrip("/") + path qps = {} body = "" for param in single_api.get("parameters", []): @@ -206,9 +336,15 @@ class MatrixUnits(Units): elif param["in"] == "query": qps[param["name"]] = param["x-example"] query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) - endpoint["example"]["req"] = "%s %s%s\n%s" % ( - method.upper(), path_template, query_string, body - ) + if body: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( + method.upper(), path_template, query_string, body + ) + else: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % ( + method.upper(), path_template, query_string + ) + else: self.log( "The following parameters are missing examples :( \n %s" % @@ -216,60 +352,137 @@ class MatrixUnits(Units): ) # add response params if this API has any. - res_type = Units.prop(res, "schema/type") - if res_type and res_type not in ["object", "array"]: - # response is a raw string or something like that - endpoint["res_tables"].append({ - "title": None, - "rows": [{ - "key": res["schema"].get("name", ""), - "type": res_type, - "desc": res.get("description", "") - }] - }) - elif res_type and Units.prop(res, "schema/properties"): # object - res_tables = get_json_schema_object_fields(res["schema"]) - for table in res_tables: - if "no-table" not in table: - endpoint["res_tables"].append(table) + if good_response: + self.log("Found a 200 response for this API") + res_type = Units.prop(good_response, "schema/type") + res_name = Units.prop(good_response, "schema/name") + if res_type and res_type not in ["object", "array"]: + # response is a raw string or something like that + good_table = { + "title": None, + "rows": [{ + "key": "<" + res_type + ">" if not res_name else res_name, + "type": res_type, + "desc": res.get("description", ""), + "req_str": "" + }] + } + if good_response.get("headers"): + for (header_name, header) in good_response.get("headers").iteritems(): + good_table["rows"].append({ + "key": header_name, + "type": "Header<" + header["type"] + ">", + "desc": header["description"], + "req_str": "" + }) + endpoint["res_tables"].append(good_table) + elif res_type and Units.prop(good_response, "schema/properties"): + # response is an object: + schema = good_response["schema"] + res_tables = get_json_schema_object_fields(schema) + for table in res_tables: + if "no-table" not in table: + endpoint["res_tables"].append(table) + elif res_type and Units.prop(good_response, "schema/items"): + # response is an array: + # FIXME: Doesn't recurse at all. + schema = good_response["schema"] + array_type = Units.prop(schema, "items/type") + if Units.prop(schema, "items/allOf"): + array_type = ( + Units.prop(schema, "items/title") + ) + endpoint["res_tables"].append({ + "title": schema.get("title", ""), + "rows": [{ + "key": "N/A", + "type": ("[%s]" % array_type), + "desc": schema.get("description", ""), + "req_str": "" + }] + }) + + for response_table in endpoint["res_tables"]: + self.log("Response: %s" % response_table["title"]) + for r in response_table["rows"]: + self.log("Row: %s" % r) + if len(endpoint["res_tables"]) == 0: + self.log( + "This API appears to have no response table. Are you " + + "sure this API returns no parameters?" + ) endpoints.append(endpoint) + aliases = single_api.get("x-alias", None) + if aliases: + alias_link = aliases["canonical-link"] + for alias in aliases["aliases"]: + endpoints.append({ + "method": method.upper(), + "path": alias, + "alias_for_path": full_path, + "alias_link": alias_link + }) + return { - "base": api.get("basePath"), + "base": api.get("basePath").rstrip("/"), "group": group_name, "endpoints": endpoints, } def load_swagger_apis(self): - path = "../api/client-server/v1" + paths = [ + V1_CLIENT_API, V2_CLIENT_API + ] apis = {} - for filename in os.listdir(path): - if not filename.endswith(".yaml"): + for path in paths: + is_v2 = (path == V2_CLIENT_API) + if not os.path.exists(V2_CLIENT_API): + self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) continue - self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: - # strip .yaml - group_name = filename[:-5] - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) - apis[group_name] = api + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + self.log("Reading swagger API: %s" % filename) + with open(os.path.join(path, filename), "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + if is_v2: + group_name = "v2_" + group_name + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta(api, group_name) + apis[group_name] = api return apis def load_common_event_fields(self): - path = "../event-schemas/schema/v1/core" + path = CORE_EVENT_SCHEMA event_types = {} - with open(path, "r") as f: - core_json = json.loads(f.read()) - for event_type in core_json["definitions"]: + + for (root, dirs, files) in os.walk(path): + for filename in files: + if not filename.endswith(".json"): + continue + + event_type = filename[:-5] # strip the ".json" + filepath = os.path.join(root, filename) + with open(filepath) as f: + try: + event_info = json.load(f) + except Exception as e: + raise ValueError( + "Error reading file %r" % (filepath,), e + ) + if "event" not in event_type: continue # filter ImageInfo and co - event_info = core_json["definitions"][event_type] + table = { "title": event_info["title"], "desc": event_info["description"], "rows": [] } + for prop in sorted(event_info["properties"]): row = { "key": prop, @@ -277,11 +490,12 @@ class MatrixUnits(Units): "desc": event_info["properties"][prop].get("description","") } table["rows"].append(row) + event_types[event_type] = table return event_types def load_event_examples(self): - path = "../event-schemas/examples/v1" + path = V1_EVENT_EXAMPLES examples = {} for filename in os.listdir(path): if not filename.startswith("m."): @@ -293,7 +507,7 @@ class MatrixUnits(Units): return examples def load_event_schemas(self): - path = "../event-schemas/schema/v1" + path = V1_EVENT_SCHEMA schemata = {} for filename in os.listdir(path): @@ -322,8 +536,8 @@ class MatrixUnits(Units): # add typeof base_defs = { - "core#/definitions/room_event": "Message Event", - "core#/definitions/state_event": "State Event" + ROOM_EVENT: "Message Event", + STATE_EVENT: "State Event" } if type(json_schema.get("allOf")) == list: schema["typeof"] = base_defs.get( @@ -346,6 +560,15 @@ class MatrixUnits(Units): Units.prop(json_schema, "properties/content") ) + # This is horrible because we're special casing a key on m.room.member. + # We need to do this because we want to document a non-content object. + if schema["type"] == "m.room.member": + invite_room_state = get_json_schema_object_fields( + json_schema["properties"]["invite_room_state"]["items"] + ) + schema["content_fields"].extend(invite_room_state) + + # grab msgtype if it is the right kind of event msgtype = Units.prop( json_schema, "properties/content/properties/msgtype/enum" @@ -360,7 +583,6 @@ class MatrixUnits(Units): "`m.room.message msgtypes`_." ) - # Assign state key info if it has some if schema["typeof"] == "State Event": skey_desc = Units.prop( @@ -374,7 +596,7 @@ class MatrixUnits(Units): return schemata def load_spec_meta(self): - path = "../CHANGELOG.rst" + path = CHANGELOG title_part = None version = None changelog_lines = [] @@ -405,7 +627,7 @@ class MatrixUnits(Units): if re.match("^v[0-9\.]+$", word): version = word[1:] # strip the 'v' - self.log("Version: %s Title part: %s Changelog lines: %s" % ( + self.log("Version: %s Title part: %s Changelog line count: %s" % ( version, title_part, len(changelog_lines) )) if not version or len(changelog_lines) == 0: @@ -416,6 +638,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__))