diff --git a/api/application-service/definitions/security.yaml b/api/application-service/definitions/security.yaml new file mode 100644 index 00000000..bcfc69c0 --- /dev/null +++ b/api/application-service/definitions/security.yaml @@ -0,0 +1,18 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +homeserverAccessToken: + type: apiKey + description: The ``hs_token`` provided by the application service's registration. + name: access_token + in: query diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index e6489cc5..32ac2c3c 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -19,13 +19,15 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: - "/_matrix/app/unstable/thirdparty/protocol/{protocol}": + "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- @@ -33,6 +35,8 @@ paths: with specific information about the various third party networks that an application service supports. operationId: getProtocolMetadata + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -72,7 +76,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user/{protocol}": + "/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- @@ -80,6 +84,8 @@ paths: User ID linked to a user on the third party network, given a set of user parameters. operationId: queryUserByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -125,12 +131,14 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location/{protocol}": + "/thirdparty/location/{protocol}": get: summary: Retrieve Matrix-side portal rooms leading to a third party location. description: |- Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -176,13 +184,15 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location": + "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. description: |- Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - homeserverAccessToken: [] parameters: - in: query name: alias @@ -221,12 +231,14 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user": + "/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. description: |- Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - homeserverAccessToken: [] parameters: - in: query name: userid diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index b885cb86..2fbc87d1 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -20,11 +20,13 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/rooms/{roomAlias}": get: @@ -36,6 +38,8 @@ paths: homeserver will send this request when it receives a request to join a room alias within the application service's namespace. operationId: queryRoomByAlias + security: + - homeserverAccessToken: [] parameters: - in: path name: roomAlias diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 0431b5e4..da363382 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -20,11 +20,13 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/users/{userId}": get: @@ -36,6 +38,8 @@ paths: send this request when it receives an event for an unknown user ID in the application service's namespace, such as a room invite. operationId: queryUserById + security: + - homeserverAccessToken: [] parameters: - in: path name: userId diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 8735cc8f..98181196 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -20,9 +20,11 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/transactions/{txnId}": put: @@ -35,6 +37,8 @@ paths: from message events via the presence of a ``state_key``, rather than via the event type. operationId: sendTransaction + security: + - homeserverAccessToken: [] parameters: - in: path name: txnId diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index 1cf66fe1..541df43c 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -47,13 +47,15 @@ paths: description: The lookup was successful. examples: application/json: { - "threepids": [ - { - "medium": "email", - "address": "monkey@banana.island" - } - ] - } + "threepids": [ + { + "medium": "email", + "address": "monkey@banana.island", + "validated_at": 1535176800000, + "added_at": 1535336848756 + } + ] + } schema: type: object properties: @@ -70,6 +72,19 @@ paths: address: type: string description: The third party identifier address. + validated_at: + type: integer + format: int64 + description: |- + The timestamp, in milliseconds, when the identifier was + validated by the identity service. + added_at: + type: integer + format: int64 + description: + The timestamp, in milliseconds, when the homeserver + associated the third party identifier with the user. + required: ['medium', 'address', 'validated_at', 'added_at'] tags: - User data post: @@ -133,6 +148,41 @@ paths: "$ref": "definitions/errors/error.yaml" tags: - User data + "/account/3pid/delete": + post: + summary: Deletes a third party identifier from the user's account + description: |- + Removes a third party identifier from the user's account. This might not + cause an unbind of the identifier from the identity service. + operationId: delete3pidFromAccount + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + properties: + medium: + type: string + description: The medium of the third party identifier being removed. + enum: ["email", "msisdn"] + example: "email" + address: + type: string + description: The third party address being removed. + example: "example@domain.com" + required: ['medium', 'address'] + responses: + 200: + description: |- + The homeserver has disassociated the third party identifier from the + user. + schema: + type: object + properties: {} + tags: + - User data "/account/3pid/email/requestToken": post: summary: Requests a validation token be sent to the given email address for the purpose of adding an email address to an account diff --git a/api/client-server/definitions/wellknown/homeserver.yaml b/api/client-server/definitions/wellknown/homeserver.yaml new file mode 100644 index 00000000..92ff34ed --- /dev/null +++ b/api/client-server/definitions/wellknown/homeserver.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: Homeserver Information +description: |- + Used by clients to discover homeserver information. +type: object +properties: + base_url: + type: string + description: The base URL for the homeserver for client-server connections. + example: https://matrix.example.com +required: + - base_url diff --git a/api/client-server/definitions/wellknown/identity_server.yaml b/api/client-server/definitions/wellknown/identity_server.yaml new file mode 100644 index 00000000..a8f7c31c --- /dev/null +++ b/api/client-server/definitions/wellknown/identity_server.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: Identity Server Information +description: |- + Used by clients to discover identity server information. +type: object +properties: + base_url: + type: string + description: The base URL for the identity server for client-server connections. + example: https://identity.example.com +required: + - base_url diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index e4b05629..72ec1fb6 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -117,6 +117,13 @@ paths: A display name to assign to the newly-created device. Ignored if ``device_id`` corresponds to a known device. example: Jungle Phone + inhibit_login: + type: boolean + description: |- + If true, an ``access_token`` and ``device_id`` should not be + returned from this call, therefore preventing an automatic + login. Defaults to false. + example: false responses: 200: description: The account has been registered. @@ -141,6 +148,7 @@ paths: description: |- An access token for the account. This access token can then be used to authorize other requests. + Required if the ``inhibit_login`` option is false. home_server: type: string description: |- @@ -155,6 +163,8 @@ paths: description: |- ID of the registered device. Will be the same as the corresponding parameter in the request, if one was specified. + Required if the ``inhibit_login`` option is false. + required: ['user_id'] 400: description: |- Part of the request was invalid. This may include one of the following error codes: diff --git a/api/client-server/room_state.yaml b/api/client-server/room_state.yaml index c04fb803..bda66eb8 100644 --- a/api/client-server/room_state.yaml +++ b/api/client-server/room_state.yaml @@ -87,6 +87,16 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation "/rooms/{roomId}/state/{eventType}": @@ -142,5 +152,15 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index f1997b61..21af0d1e 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -77,13 +77,14 @@ paths: - in: query name: set_presence type: string - enum: ["offline"] + enum: ["offline", "online", "unavailable"] description: |- Controls whether the client is automatically marked as online by polling this API. If this parameter is omitted then the client is automatically marked as online when it uses this API. Otherwise if the parameter is set to "offline" then the client is not marked as - being online when it uses this API. + being online when it uses this API. When set to "unavailable", the + client is marked as being idle. x-example: "offline" - in: query name: timeout diff --git a/api/client-server/wellknown.yaml b/api/client-server/wellknown.yaml new file mode 100644 index 00000000..24e190f9 --- /dev/null +++ b/api/client-server/wellknown.yaml @@ -0,0 +1,66 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Server Discovery API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /.well-known +produces: + - application/json +paths: + "/matrix/client": + get: + summary: Gets Matrix server discovery information about the domain. + description: |- + Gets discovery information about the domain. The file may include + additional keys, which MUST follow the Java package naming convention, + e.g. ``com.example.myapp.property``. This ensures property names are + suitably namespaced for each application and reduces the risk of + clashes. + + Note that this endpoint is not necessarily handled by the homeserver, + but by another webserver, to be used for discovering the homeserver URL. + operationId: getWellknown + responses: + 200: + description: Server discovery information. + examples: + application/json: { + "m.homeserver": { + "base_url": "https://matrix.example.com" + }, + "m.identity_server": { + "base_url": "https://identity.example.com" + } + } + schema: + type: object + properties: + m.homeserver: + description: Information about the homeserver to connect to. + "$ref": "definitions/wellknown/homeserver.yaml" + m.identity_server: + description: Optional. Information about the identity server to connect to. + "$ref": "definitions/wellknown/identity_server.yaml" + additionalProperties: + description: Application-dependent keys using Java package naming convention. + required: + - m.homeserver + 404: + description: No server discovery information available. + tags: + - Server administration diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 784bb5d6..a400bf95 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -18,15 +18,17 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: "/3pid/getValidated3pid": get: summary: Check whether ownership of a 3pid was validated. - description: A client can check whether ownership of a 3pid was validated + description: |- + Determines if a given 3pid has been validated by a user. operationId: getValidated3pid parameters: - in: query @@ -46,10 +48,10 @@ paths: description: Validation information for the session. examples: application/json: { - "medium": "email", - "validated_at": 1457622739026, - "address": "louise@bobs.burgers" - } + "medium": "email", + "validated_at": 1457622739026, + "address": "louise@bobs.burgers" + } schema: type: object properties: @@ -61,7 +63,10 @@ paths: description: The address of the 3pid being looked up. validated_at: type: integer - description: Timestamp indicating the time that the 3pid was validated. + description: |- + Timestamp, in milliseconds, indicating the time that the 3pid + was validated. + required: ['medium', 'address', 'validated_at'] 400: description: |- The session has not been validated. @@ -71,16 +76,20 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: - description: The Session ID or client secret were not found + description: The Session ID or client secret were not found. examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/bind": post: summary: Publish an association between a session and a Matrix user ID. @@ -90,7 +99,7 @@ paths: Future calls to ``/lookup`` for any of the session\'s 3pids will return this association. - Note: for backwards compatibility with older versions of this + Note: for backwards compatibility with previous drafts of this specification, the parameters may also be specified as ``application/x-form-www-urlencoded`` data. However, this usage is deprecated. @@ -101,10 +110,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "mxid": "@ears:matrix.org" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "mxid": "@ears:matrix.org" + } properties: sid: type: string @@ -121,19 +130,18 @@ paths: description: The association was published. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, - - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -157,7 +165,19 @@ paths: description: The unix timestamp at which the association was verified. signatures: type: object - description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + description: |- + The signatures of the verifying identity services which show that the + association should be trusted, if you trust the verifying identity + services. + $ref: "../../schemas/server-signatures.yaml" + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures 400: description: |- The association was not published. @@ -167,13 +187,17 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index 8431c9e8..dc3cd78e 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -18,8 +18,9 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: @@ -34,13 +35,13 @@ paths: that that user was able to read the email for that email address, and so we validate ownership of the email address. - Note that Home Servers offer APIs that proxy this API, adding + Note that homeservers offer APIs that proxy this API, adding additional behaviour on top, for example, ``/register/email/requestToken`` is designed specifically for use when registering an account and therefore will inform the user if the email address given is already registered on the server. - Note: for backwards compatibility with older versions of this + Note: for backwards compatibility with previous drafts of this specification, the parameters may also be specified as ``application/x-form-www-urlencoded`` data. However, this usage is deprecated. @@ -51,14 +52,14 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "email": "foo@example.com", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "email": "foo@example.com", + "send_attempt": 1 + } properties: client_secret: type: string - description: A unique string used to identify the validation attempt + description: A unique string used to identify the validation attempt. email: type: string description: The email address to validate. @@ -85,20 +86,28 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: - ``M_INVALID_EMAIL``: The email address provided was invalid. - ``M_EMAIL_SEND_ERROR``: The validation email could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_EMAIL", + "error": "The email address is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/email/submitToken": post: summary: Validate ownership of an email address. @@ -111,7 +120,7 @@ paths: associate the email address with any Matrix user ID. Specifically, calls to ``/lookup`` will not show a binding. - Note: for backwards compatibility with older versions of this + Note: for backwards compatibility with previous drafts of this specification, the parameters may also be specified as ``application/x-form-www-urlencoded`` data. However, this usage is deprecated. @@ -122,10 +131,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -143,14 +152,15 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of an email address. description: |- diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index 982dbff7..0b76a773 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -18,8 +18,9 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: @@ -29,7 +30,7 @@ paths: description: |- Sign invitation details. - The identity server will look up ``token`` which was stored in a call + The identity service will look up ``token`` which was stored in a call to ``store-invite``, and fetch the sender of the invite. operationId: blindlySignStuff parameters: @@ -38,24 +39,24 @@ paths: schema: type: object example: { - "mxid": "@foo:bar.com", - "token": "sometoken", - "private_key": "base64encodedkey" - } + "mxid": "@foo:bar.com", + "token": "sometoken", + "private_key": "base64encodedkey" + } properties: mxid: type: string description: The Matrix user ID of the user accepting the invitation. token: type: string - description: Token from the call to ``store-invite`` + description: The token from the call to ``store-invite``. private_key: type: string description: The private key, encoded as `Unpadded base64`_. required: ["mxid", "token", "private_key"] responses: 200: - description: The signedjson of the mxid, sender, and token. + description: The signed JSON of the mxid, sender, and token. schema: type: object properties: @@ -68,9 +69,11 @@ paths: signatures: type: object description: The signature of the mxid, sender, and token. + $ref: "../../schemas/server-signatures.yaml" token: type: string description: The token for the invitation. + required: ['mxid', 'sender', 'signatures', 'token'] examples: application/json: { "mxid": "@foo:bar.com", @@ -83,8 +86,11 @@ paths: "token": "abc123" } 404: - description: Token was not found. - example: { + description: The token was not found. + examples: + application/json: { "errcode": "M_UNRECOGNIZED", "error": "Didn't recognize token" } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index bfd2153e..1870a31f 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -1,6 +1,7 @@ # Copyright 2016 OpenMarket Ltd # Copyright 2017 Kamax.io # Copyright 2017 New Vector Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +21,9 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: @@ -46,31 +48,30 @@ paths: responses: 200: description: - The association for that 3pid, or the empty object if no association is known. + The association for that 3pid, or an empty object if no association is known. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, - - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: address: type: string - description: The 3pid address of the user being looked up. + description: The 3pid address of the user being looked up, matching the address requested. medium: type: string - description: The literal string "email". + description: A medium from the `3PID Types`_ Appendix, matching the medium requested. mxid: type: string description: The Matrix user ID associated with the 3pid. @@ -86,6 +87,15 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures "/bulk_lookup": post: summary: Lookup Matrix user IDs for a list of 3pids. @@ -110,10 +120,17 @@ paths: items: type: array title: 3PID mappings + minItems: 2 + maxItems: 2 items: - type: string - title: 3PID medium or address - description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + - type: string + - type: string + description: |- + An array of arrays containing the `3PID Types`_ with the ``medium`` + in first position and the ``address`` in second position. required: - "threepids" responses: @@ -134,9 +151,19 @@ paths: items: type: array title: 3PID mappings + minItems: 3 + maxItems: 3 items: - type: string - title: 3PID medium or address or the Matrix ID - description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + #- type: Matrix User ID + - type: string + - type: string + - type: string + description: |- + An array of array containing the `3PID Types`_ with the ``medium`` + in first position, the ``address`` in second position and Matrix user + ID in third position. required: - "threepids" diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index c2cc6cfe..836984d0 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -18,8 +18,9 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: @@ -34,13 +35,13 @@ paths: indicates that that user was able to read the SMS for that phone number, and so we validate ownership of the phone number. - Note that Home Servers offer APIs that proxy this API, adding + Note that homeservers offer APIs that proxy this API, adding additional behaviour on top, for example, ``/register/msisdn/requestToken`` is designed specifically for use when registering an account and therefore will inform the user if the phone number given is already registered on the server. - Note: for backwards compatibility with older versions of this + Note: for backwards compatibility with previous drafts of this specification, the parameters may also be specified as ``application/x-form-www-urlencoded`` data. However, this usage is deprecated. @@ -51,11 +52,11 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "country": "GB", - "phone_number": "07700900001", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "country": "GB", + "phone_number": "07700900001", + "send_attempt": 1 + } properties: client_secret: type: string @@ -91,20 +92,30 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: - ``M_INVALID_ADDRESS``: The phone number provided was invalid. - ``M_SEND_ERROR``: The validation SMS could not be sent. + - ``M_DESTINATION_REJECTED``: The identity service cannot deliver an + SMS to the provided country or region. + examples: + application/json: { + "errcode": "M_INVALID_ADDRESS", + "error": "The phone number is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/msisdn/submitToken": post: summary: Validate ownership of a phone number. @@ -117,7 +128,7 @@ paths: associate the phone number address with any Matrix user ID. Specifically, calls to ``/lookup`` will not show a binding. - Note: for backwards compatibility with older versions of this + Note: for backwards compatibility with previous drafts of this specification, the parameters may also be specified as ``application/x-form-www-urlencoded`` data. However, this usage is deprecated. @@ -128,10 +139,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -149,14 +160,15 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of a phone number. description: |- diff --git a/api/identity/ping.yaml b/api/identity/ping.yaml index 005160a3..2788d9d3 100644 --- a/api/identity/ping.yaml +++ b/api/identity/ping.yaml @@ -1,4 +1,5 @@ # Copyright 2018 Kamax Sàrl +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +15,7 @@ swagger: "2.0" info: - title: "Matrix Client-Identity Versions API" + title: "Matrix Identity Service Ping API" version: "1.0.0" host: localhost:8090 schemes: @@ -25,19 +26,19 @@ produces: paths: "/api/v1": get: - summary: Checks that an Identity server is available at this API endpopint. + summary: Checks that an Identity Service is available at this API endpoint. description: |- - Checks that an Identity server is available at this API endpopint. + Checks that an Identity Service is available at this API endpoint. - To discover that an Identity server is available at a specific URL, + To discover that an Identity Service is available at a specific URL, this endpoint can be queried and will return an empty object. This is primarly used for auto-discovery and health check purposes - by entities acting as a client for the Identity server. + by entities acting as a client for the Identity Service. operationId: ping responses: 200: - description: An Identity server is ready to serve requests. + description: An Identity Service is ready to serve requests. examples: application/json: {} schema: diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 00796975..6b17e7c6 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -18,8 +18,9 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: @@ -45,13 +46,25 @@ paths: The public key exists. examples: application/json: { - "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" - } + "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" + } schema: type: object properties: public_key: type: string + description: Unpadded Base64 encoded public key. + required: ['public_key'] + 404: + description: + The public key was not found. + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "The public key was not found" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/pubkey/isvalid": get: summary: Check whether a long-term public key is valid. @@ -72,14 +85,15 @@ paths: The validity of the public key. examples: application/json: { - "valid": true - } + "valid": true + } schema: type: object properties: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] "/pubkey/ephemeral/isvalid": get: summary: Check whether a short-term public key is valid. @@ -100,11 +114,12 @@ paths: The validity of the public key. examples: application/json: { - "valid": true - } + "valid": true + } schema: type: object properties: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index 6b847b5b..89d437a4 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -18,16 +18,17 @@ info: host: localhost:8090 schemes: - https - - http basePath: /_matrix/identity/api/v1 +consumes: + - application/json produces: - application/json paths: "/store-invite": post: - summary: Store pending invitations to a user\'s 3pid. + summary: Store pending invitations to a user's 3pid. description: |- - Store pending invitations to a user\'s 3pid. + Store pending invitations to a user's 3pid. In addition to the request parameters specified below, an arbitrary number of other parameters may also be specified. These may be used in @@ -47,6 +48,8 @@ paths: Also, the generated ephemeral public key will be listed as valid on requests to ``/_matrix/identity/api/v1/pubkey/ephemeral/isvalid``. + + Currently, invites may only be issued for 3pids of the ``email`` medium. operationId: storeInvite parameters: - in: body @@ -54,11 +57,11 @@ paths: schema: type: object example: { - "medium": "email", - "address": "foo@bar.baz", - "room_id": "!something:example.tld", - "sender": "@bob:example.com" - } + "medium": "email", + "address": "foo@bar.baz", + "room_id": "!something:example.tld", + "sender": "@bob:example.com" + } properties: medium: type: string @@ -84,21 +87,22 @@ paths: description: The generated token. public_keys: type: array - description: A list of [server\'s long-term public key, generated ephemeral public key]. + description: A list of [server's long-term public key, generated ephemeral public key]. items: type: string display_name: type: string description: The generated (redacted) display_name. + required: ['token', 'public_keys', 'display_name'] example: application/json: { - "token": "sometoken", - "public_keys": [ - "serverpublickey", - "ephemeralpublickey" - ], - "display_name": "f...@b..." - } + "token": "sometoken", + "public_keys": [ + "serverpublickey", + "ephemeralpublickey" + ], + "display_name": "f...@b..." + } 400: description: | An error has occured. @@ -108,7 +112,9 @@ paths: error code will be ``M_UNRECOGNIZED``. examples: application/json: { - "errcode": "M_THREEPID_IN_USE", - "error": "Binding already known", - "mxid": mxid - } + "errcode": "M_THREEPID_IN_USE", + "error": "Binding already known", + "mxid": "@alice:example.com" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index bb14ede2..d86b8538 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -23,7 +23,8 @@ allOf: hashes: type: object title: Event Hash - description: Hashes of the PDU, following the algorithm specified in `Signing Events`_. + description: |- + Content hashes of the PDU, following the algorithm specified in `Signing Events`_. example: { "sha256": "thishashcoversallfieldsincasethisisredacted" } diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 64991d22..446973ed 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -55,8 +55,8 @@ properties: prev_events: type: array description: |- - Event IDs and hashes of the most recent events in the room that the homeserver was aware - of when it made this event. + Event IDs and reference hashes for the most recent events in the room + that the homeserver was aware of when it made this event. items: type: array maxItems: 2 @@ -86,7 +86,7 @@ properties: auth_events: type: array description: |- - An event reference list containing the authorization events that would + Event IDs and reference hashes for the authorization events that would allow this event to be in the room. items: type: array diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index cf3988a2..c23163d7 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -49,7 +49,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization chain for the events. schema: type: object @@ -96,7 +97,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization chain for the events. schema: type: object diff --git a/api/server-server/user_keys.yaml b/api/server-server/user_keys.yaml new file mode 100644 index 00000000..63c74d20 --- /dev/null +++ b/api/server-server/user_keys.yaml @@ -0,0 +1,189 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation User Key Management API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/user/keys/claim": + post: + summary: Claims one-time encryption keys for a user. + description: |- + Claims one-time keys for use in pre-key messages. + operationId: claimUserEncryptionKeys + security: + - signedRequest: [] + parameters: + - in: body + name: body + type: object + required: true + schema: + type: object + properties: + one_time_keys: + type: object + description: |- + The keys to be claimed. A map from user ID, to a map from + device ID to algorithm name. + additionalProperties: + type: object + additionalProperties: + type: string + description: algorithm + example: "signed_curve25519" + example: { + "@alice:example.com": { + "JLAFKJWSCS": "signed_curve25519" + } + } + required: + - one_time_keys + responses: + 200: + description: The claimed keys + schema: + type: object + properties: + one_time_keys: + type: object + description: |- + One-time keys for the queried devices. A map from user ID, to a + map from devices to a map from ``:`` to the key object. + additionalProperties: + type: object + additionalProperties: + type: + - string + - object + required: ['one_time_keys'] + examples: + application/json: { + "one_time_keys": { + "@alice:example.com": { + "JLAFKJWSCS": { + "signed_curve25518:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + } + } + } + } + } + "/user/keys/query": + post: + summary: Download device identity keys. + description: |- + Returns the current devices and identity keys for the given users. + operationId: queryUserEncryptionKeys + security: + - signedRequest: [] + parameters: + - in: body + name: body + type: object + required: true + schema: + type: object + properties: + device_keys: + type: object + description: |- + The keys to be downloaded. A map from user ID, to a list of + device IDs, or to an empty list to indicate all devices for the + corresponding user. + additionalProperties: + type: array + items: + type: string + description: "Device ID" + example: { + "@alice:example.com": [] + } + required: ['device_keys'] + responses: + 200: + description: The device information. + schema: + type: object + properties: + device_keys: + type: object + description: |- + Information on the queried devices. A map from user ID, to a + map from device ID to device information. For each device, + the information returned will be the same as uploaded via + ``/keys/upload``, with the addition of an ``unsigned`` + property. + additionalProperties: + type: object + additionalProperties: + allOf: + - $ref: ../client-server/definitions/device_keys.yaml + properties: + unsigned: + title: UnsignedDeviceInfo + type: object + description: |- + Additional data added to the device key information + by intermediate servers, and not covered by the + signatures. + properties: + device_display_name: + type: string + description: + The display name which the user set on the device. + required: ['device_keys'] + examples: + application/json: { + "device_keys": { + "@alice:example.com": { + "JLAFKJWSCS": { + "user_id": "@alice:example.com", + "device_id": "JLAFKJWSCS", + "algorithms": [ + "m.olm.v1.curve25519-aes-sha256", + "m.megolm.v1.aes-sha" + ], + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + }, + "unsigned": { + "device_display_name": "Alice's mobile phone" + } + } + } + } + } diff --git a/changelogs/application_service.rst b/changelogs/application_service.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/application_service/newsfragments/.gitignore b/changelogs/application_service/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/application_service/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/application_service/pyproject.toml b/changelogs/application_service/pyproject.toml new file mode 100644 index 00000000..44d430e8 --- /dev/null +++ b/changelogs/application_service/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../application_service.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/changelogs/client_server/newsfragments/1176.new b/changelogs/client_server/newsfragments/1176.new new file mode 100644 index 00000000..41e30799 --- /dev/null +++ b/changelogs/client_server/newsfragments/1176.new @@ -0,0 +1 @@ +Specify how to control the power level required for ``@room`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1359.feature b/changelogs/client_server/newsfragments/1359.feature new file mode 100644 index 00000000..5354e69e --- /dev/null +++ b/changelogs/client_server/newsfragments/1359.feature @@ -0,0 +1 @@ +Add ``.well-known`` server discovery method \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1465.feature b/changelogs/client_server/newsfragments/1465.feature new file mode 100644 index 00000000..649cf222 --- /dev/null +++ b/changelogs/client_server/newsfragments/1465.feature @@ -0,0 +1 @@ +Share room decryption keys between devices diff --git a/changelogs/client_server/newsfragments/1547.feature b/changelogs/client_server/newsfragments/1547.feature new file mode 100644 index 00000000..76346f23 --- /dev/null +++ b/changelogs/client_server/newsfragments/1547.feature @@ -0,0 +1 @@ +Add a common standard for user, room, and group mentions in messages. diff --git a/changelogs/client_server/newsfragments/1558.clarification b/changelogs/client_server/newsfragments/1558.clarification new file mode 100644 index 00000000..3482d89c --- /dev/null +++ b/changelogs/client_server/newsfragments/1558.clarification @@ -0,0 +1 @@ +Update all event examples to be accurate representations of their associated events. diff --git a/changelogs/client_server/newsfragments/1567.feature b/changelogs/client_server/newsfragments/1567.feature new file mode 100644 index 00000000..0c19b4be --- /dev/null +++ b/changelogs/client_server/newsfragments/1567.feature @@ -0,0 +1 @@ +Document the ``validated_at`` and ``added_at`` fields on ``GET /acount/3pid``. diff --git a/changelogs/client_server/newsfragments/1567.new b/changelogs/client_server/newsfragments/1567.new new file mode 100644 index 00000000..15e3305b --- /dev/null +++ b/changelogs/client_server/newsfragments/1567.new @@ -0,0 +1 @@ +Add ``POST /account/3pid/delete`` diff --git a/changelogs/client_server/newsfragments/1589.feature b/changelogs/client_server/newsfragments/1589.feature new file mode 100644 index 00000000..8c8b1a86 --- /dev/null +++ b/changelogs/client_server/newsfragments/1589.feature @@ -0,0 +1 @@ +Add an ``inhibit_login`` registration option. diff --git a/changelogs/client_server/newsfragments/1590.clarification b/changelogs/client_server/newsfragments/1590.clarification new file mode 100644 index 00000000..27999193 --- /dev/null +++ b/changelogs/client_server/newsfragments/1590.clarification @@ -0,0 +1 @@ +Document the 403 error for sending state events. diff --git a/changelogs/client_server/newsfragments/1596.clarification b/changelogs/client_server/newsfragments/1596.clarification new file mode 100644 index 00000000..3dde069f --- /dev/null +++ b/changelogs/client_server/newsfragments/1596.clarification @@ -0,0 +1 @@ +specify how to handle multiple olm sessions with the same device \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/780.feature b/changelogs/client_server/newsfragments/780.feature new file mode 100644 index 00000000..74725754 --- /dev/null +++ b/changelogs/client_server/newsfragments/780.feature @@ -0,0 +1 @@ +Add more presence options to the ``set_presence`` parameter of ``/sync``. (Thanks @mujx!) diff --git a/changelogs/identity_service.rst b/changelogs/identity_service.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/identity_service/newsfragments/.gitignore b/changelogs/identity_service/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/identity_service/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/identity_service/pyproject.toml b/changelogs/identity_service/pyproject.toml new file mode 100644 index 00000000..7a64eb0b --- /dev/null +++ b/changelogs/identity_service/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../identity_service.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/changelogs/push_gateway.rst b/changelogs/push_gateway.rst new file mode 100644 index 00000000..33a7683c --- /dev/null +++ b/changelogs/push_gateway.rst @@ -0,0 +1,6 @@ +r0.1.0 +====== + +The first release of the Push Gateway specification. This release contains +a single endpoint, ``/notify``, that pushers may use to send push notifications +to clients. diff --git a/changelogs/push_gateway/newsfragments/.gitignore b/changelogs/push_gateway/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/push_gateway/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/push_gateway/pyproject.toml b/changelogs/push_gateway/pyproject.toml new file mode 100644 index 00000000..dad1bc04 --- /dev/null +++ b/changelogs/push_gateway/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../push_gateway.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/changelogs/server_server.rst b/changelogs/server_server.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/server_server/newsfragments/.gitignore b/changelogs/server_server/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/server_server/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/server_server/pyproject.toml b/changelogs/server_server/pyproject.toml new file mode 100644 index 00000000..98478527 --- /dev/null +++ b/changelogs/server_server/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../server_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index f2456d97..3e536ec3 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -44,16 +44,51 @@ except ImportError as e: raise +def load_file(path): + print("Loading reference: %s" % path) + if not path.startswith("file://"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.load(f) + + +def resolve_references(path, schema): + if isinstance(schema, dict): + # do $ref first + if '$ref' in schema: + value = schema['$ref'] + path = os.path.abspath(os.path.join(os.path.dirname(path), value)) + ref = load_file("file://" + path) + result = resolve_references(path, ref) + del schema['$ref'] + else: + result = {} + + for key, value in schema.items(): + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + def check_example_file(examplepath, schemapath): with open(examplepath) as f: - example = yaml.load(f) + example = resolve_references(examplepath, json.load(f)) with open(schemapath) as f: schema = yaml.load(f) fileurl = "file://" + os.path.abspath(schemapath) schema["id"] = fileurl - resolver = jsonschema.RefResolver(schemapath, schema, handlers={"file": load_yaml}) + resolver = jsonschema.RefResolver(schemapath, schema, handlers={"file": load_file}) print ("Checking schema for: %r %r" % (examplepath, schemapath)) try: @@ -71,6 +106,10 @@ def check_example_dir(exampledir, schemadir): if filename.startswith("."): # Skip over any vim .swp files. continue + cwd = os.path.basename(os.path.dirname(os.path.join(root, filename))) + if cwd == "core": + # Skip checking the underlying definitions + continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) if schemapath.find("#") >= 0: @@ -85,14 +124,6 @@ def check_example_dir(exampledir, schemadir): raise ValueError("Error validating examples") -def load_yaml(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return yaml.load(f) - - if __name__ == '__main__': try: check_example_dir("examples", "schema") diff --git a/event-schemas/examples/core/event.json b/event-schemas/examples/core/event.json new file mode 100644 index 00000000..8a469a5c --- /dev/null +++ b/event-schemas/examples/core/event.json @@ -0,0 +1,6 @@ +{ + "content": { + "key": "value" + }, + "type": "org.example.custom.event" +} diff --git a/event-schemas/examples/core/room_edu.json b/event-schemas/examples/core/room_edu.json new file mode 100644 index 00000000..80575f5d --- /dev/null +++ b/event-schemas/examples/core/room_edu.json @@ -0,0 +1,4 @@ +{ + "$ref": "event.json", + "room_id": "!jEsUZKDJdhlrceRyVU:domain.com" +} diff --git a/event-schemas/examples/core/room_event.json b/event-schemas/examples/core/room_event.json new file mode 100644 index 00000000..41837afb --- /dev/null +++ b/event-schemas/examples/core/room_event.json @@ -0,0 +1,10 @@ +{ + "$ref": "event.json", + "event_id": "$143273582443PhrSn:domain.com", + "room_id": "!jEsUZKDJdhlrceRyVU:domain.com", + "sender": "@example:domain.com", + "origin_server_ts": 1432735824653, + "unsigned": { + "age": 1234 + } +} diff --git a/event-schemas/examples/core/state_event.json b/event-schemas/examples/core/state_event.json new file mode 100644 index 00000000..910747ee --- /dev/null +++ b/event-schemas/examples/core/state_event.json @@ -0,0 +1,4 @@ +{ + "$ref": "room_event.json", + "state_key": "ArbitraryString" +} diff --git a/event-schemas/examples/m.call.answer b/event-schemas/examples/m.call.answer index f7d14439..a4cfc1e1 100644 --- a/event-schemas/examples/m.call.answer +++ b/event-schemas/examples/m.call.answer @@ -1,5 +1,6 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.call.answer", "content": { "version" : 0, "call_id": "12345", @@ -8,10 +9,5 @@ "type" : "answer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" } - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.call.answer", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.call.candidates b/event-schemas/examples/m.call.candidates index 8e6849bb..8f1f807a 100644 --- a/event-schemas/examples/m.call.candidates +++ b/event-schemas/examples/m.call.candidates @@ -1,5 +1,6 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.call.candidates", "content": { "version" : 0, "call_id": "12345", @@ -10,10 +11,5 @@ "candidate": "candidate:863018703 1 udp 2122260223 10.9.64.156 43670 typ host generation 0" } ] - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.call.candidates", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.call.hangup b/event-schemas/examples/m.call.hangup index 42e1f346..295f16e4 100644 --- a/event-schemas/examples/m.call.hangup +++ b/event-schemas/examples/m.call.hangup @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.call.hangup", "content": { "version" : 0, "call_id": "12345" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.call.hangup", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.call.invite b/event-schemas/examples/m.call.invite index 974a5b4c..fa482bd9 100644 --- a/event-schemas/examples/m.call.invite +++ b/event-schemas/examples/m.call.invite @@ -1,5 +1,6 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.call.invite", "content": { "version" : 0, "call_id": "12345", @@ -8,10 +9,5 @@ "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" } - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.call.invite", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.direct b/event-schemas/examples/m.direct index 92f13daa..e453dd59 100644 --- a/event-schemas/examples/m.direct +++ b/event-schemas/examples/m.direct @@ -1,9 +1,10 @@ { + "$ref": "core/event.json", "type": "m.direct", "content": { "@bob:example.com": [ - "!abcdefgh:example.com", - "!hgfedcba:example.com" - ] + "!abcdefgh:example.com", + "!hgfedcba:example.com" + ] } } diff --git a/event-schemas/examples/m.forwarded_room_key b/event-schemas/examples/m.forwarded_room_key new file mode 100644 index 00000000..ef1d6180 --- /dev/null +++ b/event-schemas/examples/m.forwarded_room_key @@ -0,0 +1,14 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8Llf...", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU", + "sender_claimed_ed25519_key": "aj40p+aw64yPIdsxoog8jhPu9i7l7NcFRecuOQblE3Y", + "forwarding_curve25519_key_chain": [ + "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" + ] + }, + "type": "m.forwarded_room_key" +} diff --git a/event-schemas/examples/m.ignored_user_list b/event-schemas/examples/m.ignored_user_list index f3a328f7..9963d13a 100644 --- a/event-schemas/examples/m.ignored_user_list +++ b/event-schemas/examples/m.ignored_user_list @@ -1,4 +1,5 @@ { + "$ref": "core/event.json", "type": "m.ignored_user_list", "content": { "ignored_users": { diff --git a/event-schemas/examples/m.presence b/event-schemas/examples/m.presence index 824ffcb7..36093cd9 100644 --- a/event-schemas/examples/m.presence +++ b/event-schemas/examples/m.presence @@ -1,10 +1,11 @@ { + "$ref": "core/event.json", + "sender": "@example:localhost", + "type": "m.presence", "content": { "avatar_url": "mxc://localhost:wefuiwegh8742w", "last_active_ago": 2478593, "presence": "online", "currently_active": false - }, - "sender": "@example:localhost", - "type": "m.presence" + } } diff --git a/event-schemas/examples/m.receipt b/event-schemas/examples/m.receipt index bd0b726c..c52d8540 100644 --- a/event-schemas/examples/m.receipt +++ b/event-schemas/examples/m.receipt @@ -1,13 +1,13 @@ { - "type": "m.receipt", - "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", - "content": { - "$1435641916114394fHBLK:matrix.org": { - "m.read": { - "@rikj:jki.re": { - "ts": 1436451550453 - } - } + "$ref": "core/room_edu.json", + "type": "m.receipt", + "content": { + "$1435641916114394fHBLK:matrix.org": { + "m.read": { + "@rikj:jki.re": { + "ts": 1436451550453 } + } } + } } diff --git a/event-schemas/examples/m.room.aliases b/event-schemas/examples/m.room.aliases index ca87510e..bb2fe21c 100644 --- a/event-schemas/examples/m.room.aliases +++ b/event-schemas/examples/m.room.aliases @@ -1,12 +1,8 @@ { - "age": 242352, - "content": { - "aliases": ["#somewhere:localhost", "#another:localhost"] - }, - "state_key": "localhost", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", + "$ref": "core/state_event.json", + "state_key": "domain.com", "type": "m.room.aliases", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + "content": { + "aliases": ["#somewhere:domain.com", "#another:domain.com"] + } } diff --git a/event-schemas/examples/m.room.avatar b/event-schemas/examples/m.room.avatar index 2080d96e..9b51e01f 100644 --- a/event-schemas/examples/m.room.avatar +++ b/event-schemas/examples/m.room.avatar @@ -1,5 +1,7 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.avatar", + "state_key": "", "content": { "info": { "h": 398, @@ -7,12 +9,6 @@ "mimetype": "image/jpeg", "size": 31037 }, - "url": "mxc://localhost/JWEIFJgwEIhweiWJE" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.avatar", - "state_key": "", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + "url": "mxc://domain.com/JWEIFJgwEIhweiWJE" + } } diff --git a/event-schemas/examples/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias index 59df586d..06c3226c 100644 --- a/event-schemas/examples/m.room.canonical_alias +++ b/event-schemas/examples/m.room.canonical_alias @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.canonical_alias", + "state_key": "", "content": { "alias": "#somewhere:localhost" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.canonical_alias", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.create b/event-schemas/examples/m.room.create index 34dabb53..18127497 100644 --- a/event-schemas/examples/m.room.create +++ b/event-schemas/examples/m.room.create @@ -1,12 +1,10 @@ { - "age": 242352, - "content": { - "creator": "@example:localhost" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", + "$ref": "core/state_event.json", "type": "m.room.create", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + "state_key": "", + "content": { + "creator": "@example:domain.com", + "room_version": "1", + "m.federate": true + } } diff --git a/event-schemas/examples/m.room.encrypted#megolm b/event-schemas/examples/m.room.encrypted#megolm index 1f9b7520..ac542e25 100644 --- a/event-schemas/examples/m.room.encrypted#megolm +++ b/event-schemas/examples/m.room.encrypted#megolm @@ -1,14 +1,11 @@ { + "$ref": "core/room_event.json", + "type": "m.room.encrypted", "content": { "algorithm": "m.megolm.v1.aes-sha2", "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", "device_id": "RJYKSTBOIE", "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ" - }, - "event_id": "$WLGTSEFSEF:localhost", - "room_id": "!Cuyf34gef24t:localhost", - "origin_server_ts": 1476648761524, - "sender": "@example:localhost", - "type": "m.room.encrypted" + } } diff --git a/event-schemas/examples/m.room.encrypted#olm b/event-schemas/examples/m.room.encrypted#olm index abb23c31..381651d9 100644 --- a/event-schemas/examples/m.room.encrypted#olm +++ b/event-schemas/examples/m.room.encrypted#olm @@ -1,6 +1,6 @@ { + "$ref": "core/room_event.json", "type": "m.room.encrypted", - "sender": "@example:localhost", "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "Szl29ksW/L8yZGWAX+8dY1XyFi+i5wm+DRhTGkbMiwU", diff --git a/event-schemas/examples/m.room.encryption b/event-schemas/examples/m.room.encryption index 08f15239..6158b937 100644 --- a/event-schemas/examples/m.room.encryption +++ b/event-schemas/examples/m.room.encryption @@ -1,13 +1,10 @@ { + "$ref": "core/state_event.json", + "type": "m.room.encryption", + "state_key": "", "content": { "algorithm": "m.megolm.v1.aes-sha2", "rotation_period_ms": 604800000, "rotation_period_msgs": 100 - }, - "event_id": "$WLGTSEFJJKJ:localhost", - "origin_server_ts": 1476648761524, - "sender": "@example:localhost", - "room_id": "!Cuyf34gef24t:localhost", - "state_key": "", - "type": "m.room.encryption" + } } diff --git a/event-schemas/examples/m.room.guest_access b/event-schemas/examples/m.room.guest_access index c636ff39..a6deff8c 100644 --- a/event-schemas/examples/m.room.guest_access +++ b/event-schemas/examples/m.room.guest_access @@ -1,12 +1,8 @@ { - "age": 242353, + "$ref": "core/state_event.json", + "type": "m.room.guest_access", + "state_key": "", "content": { "guest_access": "can_join" - }, - "state_key": "", - "origin_server_ts": 1431961217938, - "event_id": "$WLGTSEFSEG:localhost", - "type": "m.room.guest_access", - "room_id": "!Cuyf34gef24u:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility index 6fedc5dc..27c4fec3 100644 --- a/event-schemas/examples/m.room.history_visibility +++ b/event-schemas/examples/m.room.history_visibility @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.history_visibility", + "state_key": "", "content": { "history_visibility": "shared" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.history_visibility", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.join_rules b/event-schemas/examples/m.room.join_rules index 39e14fc5..2873be78 100644 --- a/event-schemas/examples/m.room.join_rules +++ b/event-schemas/examples/m.room.join_rules @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.join_rules", + "state_key": "", "content": { "join_rule": "public" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.join_rules", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.member b/event-schemas/examples/m.room.member index 2495145b..ce31ab8f 100644 --- a/event-schemas/examples/m.room.member +++ b/event-schemas/examples/m.room.member @@ -1,14 +1,10 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "state_key": "@alice:domain.com", + "type": "m.room.member", "content": { "membership": "join", - "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "avatar_url": "mxc://domain.com/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid" - }, - "state_key": "@alice:localhost", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.member", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state index 965669ad..c99c66c0 100644 --- a/event-schemas/examples/m.room.member#invite_room_state +++ b/event-schemas/examples/m.room.member#invite_room_state @@ -1,11 +1,12 @@ { - "age": 242352, + "$ref": "m.room.member", "content": { "membership": "invite", - "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "avatar_url": "mxc://domain.com/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid" }, "unsigned": { + "age": 1234, "invite_room_state": [ { "type": "m.room.name", @@ -22,11 +23,5 @@ } } ] - }, - "state_key": "@alice:localhost", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.member", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite index 244e1556..92b5d2ef 100644 --- a/event-schemas/examples/m.room.member#third_party_invite +++ b/event-schemas/examples/m.room.member#third_party_invite @@ -1,13 +1,13 @@ { - "age": 242352, + "$ref": "m.room.member", "content": { "membership": "invite", - "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "avatar_url": "mxc://domain.com/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid", "third_party_invite": { "display_name": "alice", "signed": { - "mxid": "@alice:localhost", + "mxid": "@alice:domain.com", "signatures": { "magic.forest": { "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" @@ -16,11 +16,5 @@ "token": "abc123" } } - }, - "state_key": "@alice:localhost", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.member", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio index 367eb954..4ce5a2a8 100644 --- a/event-schemas/examples/m.room.message#m.audio +++ b/event-schemas/examples/m.room.message#m.audio @@ -1,18 +1,14 @@ { - "age": 146, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "Bee Gees - Stayin' Alive", - "url": "mxc://localhost/ffed755USFFxlgbQYZGtryd", + "url": "mxc://domain.com/ffed755USFFxlgbQYZGtryd", "info": { "duration": 2140786, "size": 1563685, "mimetype": "audio/mpeg" }, "msgtype": "m.audio" - }, - "event_id": "$143273582443PhrSn:localhost", - "origin_server_ts": 1432735824653, - "room_id": "!jEsUZKDJdhlrceRyVU:localhost", - "type": "m.room.message", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote index 79292ddf..5fecb9a3 100644 --- a/event-schemas/examples/m.room.message#m.emote +++ b/event-schemas/examples/m.room.message#m.emote @@ -1,14 +1,10 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "thinks this is an example emote", "msgtype": "m.emote", "format": "org.matrix.custom.html", "formatted_body": "thinks this is an example emote" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.message", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file index e52c3a94..b518550a 100644 --- a/event-schemas/examples/m.room.message#m.file +++ b/event-schemas/examples/m.room.message#m.file @@ -1,5 +1,6 @@ { - "age": 146, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "something-important.doc", "filename": "something-important.doc", @@ -8,11 +9,6 @@ "size": 46144 }, "msgtype": "m.file", - "url": "mxc://localhost/FHyPlCeYUSFFxlgbQYZmoEoe" - }, - "event_id": "$143273582443PhrSn:localhost", - "origin_server_ts": 1432735824653, - "room_id": "!jEsUZKDJdhlrceRyVU:localhost", - "type": "m.room.message", - "sender": "@example:localhost" + "url": "mxc://domain.com/FHyPlCeYUSFFxlgbQYZmoEoe" + } } diff --git a/event-schemas/examples/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image index 91e72be2..60402eff 100644 --- a/event-schemas/examples/m.room.message#m.image +++ b/event-schemas/examples/m.room.message#m.image @@ -1,5 +1,6 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "filename.jpg", "info": { @@ -8,12 +9,7 @@ "mimetype": "image/jpeg", "size": 31037 }, - "url": "mxc://localhost/JWEIFJgwEIhweiWJE", + "url": "mxc://domain.com/JWEIFJgwEIhweiWJE", "msgtype": "m.image" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.message", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location index 75363f6f..1461a305 100644 --- a/event-schemas/examples/m.room.message#m.location +++ b/event-schemas/examples/m.room.message#m.location @@ -1,10 +1,11 @@ { - "age": 146, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "Big Ben, London, UK", "geo_uri": "geo:51.5008,0.1247", "info": { - "thumbnail_url": "mxc://localhost/FHyPlCeYUSFFxlgbQYZmoEoe", + "thumbnail_url": "mxc://domain.com/FHyPlCeYUSFFxlgbQYZmoEoe", "thumbnail_info": { "mimetype": "image/jpeg", "size": 46144, @@ -13,10 +14,5 @@ } }, "msgtype": "m.location" - }, - "event_id": "$143273582443PhrSn:localhost", - "origin_server_ts": 1432735824653, - "room_id": "!jEsUZKDJdhlrceRyVU:localhost", - "type": "m.room.message", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice index 876cbbb7..d33751da 100644 --- a/event-schemas/examples/m.room.message#m.notice +++ b/event-schemas/examples/m.room.message#m.notice @@ -1,14 +1,10 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "This is an example notice", "msgtype": "m.notice", "format": "org.matrix.custom.html", "formatted_body": "This is an example notice" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.message", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text index 48a97db8..ba1fb769 100644 --- a/event-schemas/examples/m.room.message#m.text +++ b/event-schemas/examples/m.room.message#m.text @@ -1,14 +1,10 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "This is an example text message", "msgtype": "m.text", "format": "org.matrix.custom.html", "formatted_body": "This is an example text message" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.message", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video index 576d80de..304fbfbd 100644 --- a/event-schemas/examples/m.room.message#m.video +++ b/event-schemas/examples/m.room.message#m.video @@ -1,10 +1,11 @@ { - "age": 146, + "$ref": "core/room_event.json", + "type": "m.room.message", "content": { "body": "Gangnam Style", - "url": "mxc://localhost/a526eYUSFFxlgbQYZmo442", + "url": "mxc://domain.com/a526eYUSFFxlgbQYZmo442", "info": { - "thumbnail_url": "mxc://localhost/FHyPlCeYUSFFxlgbQYZmoEoe", + "thumbnail_url": "mxc://domain.com/FHyPlCeYUSFFxlgbQYZmoEoe", "thumbnail_info": { "mimetype": "image/jpeg", "size": 46144, @@ -18,10 +19,5 @@ "mimetype": "video/mp4" }, "msgtype": "m.video" - }, - "event_id": "$143273582443PhrSn:localhost", - "origin_server_ts": 1432735824653, - "room_id": "!jEsUZKDJdhlrceRyVU:localhost", - "type": "m.room.message", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback index 16fe0ee0..e146e874 100644 --- a/event-schemas/examples/m.room.message.feedback +++ b/event-schemas/examples/m.room.message.feedback @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.room.message.feedback", "content": { "type": "delivered", "target_event_id": "$WEIGFHFW:localhost" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.message.feedback", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.name b/event-schemas/examples/m.room.name index 87db2008..e77e2b53 100644 --- a/event-schemas/examples/m.room.name +++ b/event-schemas/examples/m.room.name @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.name", + "state_key": "", "content": { "name": "The room name" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.name", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.pinned_events b/event-schemas/examples/m.room.pinned_events index 6f41e97d..10d71a8d 100644 --- a/event-schemas/examples/m.room.pinned_events +++ b/event-schemas/examples/m.room.pinned_events @@ -1,12 +1,8 @@ { - "age": 242352, - "content": { - "pinned": ["$someevent:localhost"] - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", + "$ref": "core/state_event.json", "type": "m.room.pinned_events", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + "state_key": "", + "content": { + "pinned": ["$someevent:domain.com"] + } } diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 0c8f8bc5..ad741e88 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -1,5 +1,7 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.power_levels", + "state_key": "", "content": { "ban": 50, "events": { @@ -14,12 +16,9 @@ "users": { "@example:localhost": 100 }, - "users_default": 0 - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.power_levels", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + "users_default": 0, + "notifications": { + "room": 20 + } + } } diff --git a/event-schemas/examples/m.room.redaction b/event-schemas/examples/m.room.redaction index e24a8cdb..42bc8411 100644 --- a/event-schemas/examples/m.room.redaction +++ b/event-schemas/examples/m.room.redaction @@ -1,14 +1,8 @@ { - "unsigned": { - "age": 242352 - }, - "content": { - "reason": "Spamming" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", + "$ref": "core/room_event.json", "type": "m.room.redaction", - "room_id": "!Cuyf34gef24t:localhost", "redacts": "$fukweghifu23:localhost", - "sender": "@example:localhost" + "content": { + "reason": "Spamming" + } } diff --git a/event-schemas/examples/m.room.third_party_invite b/event-schemas/examples/m.room.third_party_invite index 3f9d48fe..03f35375 100644 --- a/event-schemas/examples/m.room.third_party_invite +++ b/event-schemas/examples/m.room.third_party_invite @@ -1,5 +1,7 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.third_party_invite", + "state_key": "pc98", "content": { "display_name": "Alice Margatroid", "key_validity_url": "https://magic.forest/verifykey", @@ -8,11 +10,5 @@ "public_key": "def456", "key_validity_url": "https://magic.forest/verifykey" }] - }, - "state_key": "pc98", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.third_party_invite", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room.topic b/event-schemas/examples/m.room.topic index 65daa987..69e5d4f1 100644 --- a/event-schemas/examples/m.room.topic +++ b/event-schemas/examples/m.room.topic @@ -1,12 +1,8 @@ { - "age": 242352, + "$ref": "core/state_event.json", + "type": "m.room.topic", + "state_key": "", "content": { "topic": "A room topic" - }, - "state_key": "", - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.room.topic", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.room_key b/event-schemas/examples/m.room_key index 53f83e52..dba497b4 100644 --- a/event-schemas/examples/m.room_key +++ b/event-schemas/examples/m.room_key @@ -1,9 +1,10 @@ { + "$ref": "core/event.json", + "type": "m.room_key", "content": { "algorithm": "m.megolm.v1.aes-sha2", "room_id": "!Cuyf34gef24t:localhost", "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8LlfJL7qNBEY..." - }, - "type": "m.room_key" + } } diff --git a/event-schemas/examples/m.room_key_request#cancel_request b/event-schemas/examples/m.room_key_request#cancel_request new file mode 100644 index 00000000..c6eb25de --- /dev/null +++ b/event-schemas/examples/m.room_key_request#cancel_request @@ -0,0 +1,8 @@ +{ + "content": { + "action": "cancel_request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/examples/m.room_key_request#request b/event-schemas/examples/m.room_key_request#request new file mode 100644 index 00000000..8557f08e --- /dev/null +++ b/event-schemas/examples/m.room_key_request#request @@ -0,0 +1,14 @@ +{ + "content": { + "body": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU" + }, + "action": "request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/examples/m.sticker b/event-schemas/examples/m.sticker index f00e5b23..971cdc90 100644 --- a/event-schemas/examples/m.sticker +++ b/event-schemas/examples/m.sticker @@ -1,5 +1,6 @@ { - "age": 242352, + "$ref": "core/room_event.json", + "type": "m.sticker", "content": { "body": "Landing", "info": { @@ -16,10 +17,5 @@ "size": 73602 }, "url": "mxc://matrix.org/sHhqkFCvSkFwtmvtETOtKnLP" - }, - "origin_server_ts": 1431961217939, - "event_id": "$WLGTSEFSEF:localhost", - "type": "m.sticker", - "room_id": "!Cuyf34gef24t:localhost", - "sender": "@example:localhost" + } } diff --git a/event-schemas/examples/m.tag b/event-schemas/examples/m.tag index 53dbc921..0d61d91b 100644 --- a/event-schemas/examples/m.tag +++ b/event-schemas/examples/m.tag @@ -1,4 +1,5 @@ { + "$ref": "core/event.json", "type": "m.tag", "content": { "tags": { diff --git a/event-schemas/examples/m.typing b/event-schemas/examples/m.typing index 1d2c517b..416b9968 100644 --- a/event-schemas/examples/m.typing +++ b/event-schemas/examples/m.typing @@ -1,7 +1,7 @@ { - "type": "m.typing", - "room_id": "!z0mnsuiwhifuhwwfw:matrix.org", - "content": { - "user_ids": ["@alice:matrix.org", "@bob:example.com"] - } + "$ref": "core/room_edu.json", + "type": "m.typing", + "content": { + "user_ids": ["@alice:matrix.org", "@bob:example.com"] + } } diff --git a/event-schemas/schema/core-event-schema/state_event.yaml b/event-schemas/schema/core-event-schema/state_event.yaml index 020e9087..71c4137b 100644 --- a/event-schemas/schema/core-event-schema/state_event.yaml +++ b/event-schemas/schema/core-event-schema/state_event.yaml @@ -11,7 +11,11 @@ properties: state_key: description: A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this - key makes this event a State Event. The key MUST NOT start with '_'. + key makes this event a State Event. + + State keys starting with an ``@`` are reserved for referencing user IDs, such + as room members. With the exception of a few events, state events set with a + given user's ID as the state key MUST only be set by that user. type: string required: - state_key diff --git a/event-schemas/schema/m.forwarded_room_key b/event-schemas/schema/m.forwarded_room_key new file mode 100644 index 00000000..f0beed2b --- /dev/null +++ b/event-schemas/schema/m.forwarded_room_key @@ -0,0 +1,59 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to forward keys for end-to-end encryption. Typically + it is encrypted as an ``m.room.encrypted`` event, then sent as a `to-device`_ + event. +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm the key in this event is to be used with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session that the key is for. + session_key: + type: string + description: The key to be exchanged. + sender_claimed_ed25519_key: + type: string + description: |- + The Ed25519 key of the device which initiated the session originally. + It is 'claimed' because the receiving device has no way to tell that the + original room_key actually came from a device which owns the private part of + this key unless they have done device verification. + forwarding_curve25519_key_chain: + type: array + items: + type: string + description: |- + Chain of Curve25519 keys. It starts out empty, but each time the + key is forwarded to another device, the previous sender in the chain is added + to the end of the list. For example, if the key is forwarded from A to B to + C, this field is empty between A and B, and contains A's Curve25519 key between + B and C. + required: + - algorithm + - room_id + - session_id + - session_key + - sender_claimed_ed25519_key + - forwarding_curve25519_key_chain + - sender_key + type: object + type: + enum: + - m.forwarded_room_key + type: string +type: object diff --git a/event-schemas/schema/m.room.member b/event-schemas/schema/m.room.member index 5fb5356d..de14644d 100644 --- a/event-schemas/schema/m.room.member +++ b/event-schemas/schema/m.room.member @@ -105,7 +105,10 @@ properties: title: EventContent type: object state_key: - description: The ``user_id`` this membership event relates to. + description: |- + The ``user_id`` this membership event relates to. In all cases except for when ``membership`` is + ``join``, the user ID sending the event does not need to match the user ID in the ``state_key``, + unlike other events. Regular authorisation rules still apply. type: string type: enum: diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index b00d86a9..9bb12993 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -85,6 +85,18 @@ properties: ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if unspecified. type: integer + notifications: + properties: + room: + type: integer + description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. + additionalProperties: + type: integer + description: |- + The power level requirements for specific notification types. + This is a mapping from ``key`` to power level for that notifications key. + title: Notifications + type: object type: object state_key: description: A zero-length string. diff --git a/event-schemas/schema/m.room_key_request b/event-schemas/schema/m.room_key_request new file mode 100644 index 00000000..007d0086 --- /dev/null +++ b/event-schemas/schema/m.room_key_request @@ -0,0 +1,61 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to request keys for end-to-end encryption. It is sent as an + unencrypted `to-device`_ event. +properties: + content: + properties: + body: + description: |- + Information about the requested key. Required when ``action`` is + ``request``. + properties: + algorithm: + type: string + description: |- + The encryption algorithm the requested key in this event is to be used + with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session that the key is for. + required: + - algorithm + - room_id + - session_id + - sender_key + type: object + title: RequestedKeyInfo + action: + enum: + - request + - cancel_request + type: string + requesting_device_id: + description: ID of the device requesting the key. + type: string + request_id: + description: |- + A random string uniquely identifying the request for a key. If the key is + requested multiple times, it should be reused. It should also reused in order + to cancel a request. + type: string + required: + - action + - requesting_device_id + - request_id + type: object + type: + enum: + - m.room_key_request + type: string +type: object diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md new file mode 100644 index 00000000..ac3d21fa --- /dev/null +++ b/meta/releasing_a_spec.md @@ -0,0 +1,48 @@ +# How to release a specification + +There are several specifications that belong to matrix, such as the client-server +specification, server-server specification, and identity server specification. Each +of these gets released independently of each other with their own version numbers. + +Once a specification is ready for release, a branch should be created to track the +changes in and to hold potential future hotfixes. This should be the name of the +specification (as it appears in the directory structure of this project) followed +by "release-" and the release version. For example, if the Client-Server Specification +was getting an r0.4.0 release, the branch name would be `client_server/release-r0.4.0`. + +*Note*: Historical releases prior to this process may or may not have an appropriate +release branch. Releases after this document came into place will have an appropriate +branch. + +The remainder of the process is as follows: +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the release instructions provided in the README.md located there. +1. Update the changelog section of the specification you're releasing to make a reference + to the new version. +1. Update any version/link references across all specifications. +1. Ensure the `targets.yml` file lists the version correctly. +1. Commit the changes and PR them to master. +1. Tag the release with the format `client_server/r0.4.0`. +1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). + * This is done by making a PR to the `unstyled_docs/spec` folder for the version and + specification you're releasing. + * Don't forget to symlink the new release as `latest`. +1. Perform a release on GitHub to tag the release. +1. Yell from the mountaintop to the world about the new release. + +### Creating a release for a brand-new specification + +Some specifications may not have ever had a release, and therefore need a bit more work +to become ready. + +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the "new changelog" instructions provided in the README.md located there. +1. Open the specification RST file and make some changes: + * Using a released specification as a template, update the changelog section. + * Use the appropriate changelog variable in the RST. +1. Create/define the appropriate variables in `gendoc.py`. +1. Update `targets.yml`. +1. Update any version/link references across all specifications. +1. Follow the regular release process. diff --git a/proposals/1442-state-resolution.md b/proposals/1442-state-resolution.md new file mode 100644 index 00000000..1a2e82a3 --- /dev/null +++ b/proposals/1442-state-resolution.md @@ -0,0 +1,526 @@ +# State Resolution: Reloaded + + +Thoughts on the next iteration of the state resolution algorithm that aims to +mitigate currently known attacks + + +# Background + +The state of a room at an event is a mapping from key to event, which is built +up and updated by sending state events into the room. All the information about +the room is encoded in the state, from metadata like the name and topic to +membership of the room to security policies like bans and join rules. + +It is therefore important that─wherever possible─the view of the state of the +room is consistent across all servers. If different servers have different +views of the state then it can lead to the room bifurcating, due to differing +ideas on who is in the room, who is allowed to talk, etc. + +The difficulty comes when the room DAG forks and then merges again (which can +happen naturally if two servers send events at the same time or when a network +partition is resolved). The state after the merge has to be resolved from the +state of the two branches: the algorithm to resolve this is called the _state +resolution algorithm_. + +Since the result of state resolution must be consistent across servers, the +information that the algorithm can use is strictly limited to the information +that will always be available to all servers (including future servers that may +not even be in the room at that point) at any point in time where the +resolution needs to be calculated. In particular, this has the consequence that +the algorithm cannot use information from the room DAG, since servers are not +required to store events for any length of time. + +**As such, the state resolution algorithm is effectively a pure function from +sets of state to a single resolved set of state.** + +The final important property for state resolution is that it should not allow +malicious servers to avoid moderation action by forking and merging the room +DAG. For example, if a server gets banned and then forks the room before the +ban, any merge back should always ensure that the ban is still in the state. + + +# Current Algorithm + +The current state resolution is known to have some undesirable properties, +which can be summarized into two separate cases: + +1. Moderation evasion ─ where an attacker can avoid e.g. bans by forking and + joining the room DAG in particular ways. +1. State resets ─ where a server (often innocently) sends an event that points + to disparate parts of the graph, causing state resolution to pick old state + rather than later versions. + +These have the following causes: + +1. Conflicting state must pass auth checks to be eligible to be picked, but the + algorithm does not consider previous (superseded) state changes in a fork. + For example, where Alice gives Bob power and then Bob gives Charlie power on + one branch of a conflict, when the latter power level event is authed + against the original power level (where Bob didn't have power), it fails. +1. The algorithm relies on the deprecated and untrustable depth parameter to + try and ensure that the "most recent" state is picked. Without having a copy + of the complete room DAG the algorithm doesn't know that e.g. one topic + event came strictly after another in the DAG. For efficiency and storage + reasons servers are not required (or expected) to store the whole room DAG. +1. The algorithm always accepts events where there are no conflicting + alternatives in other forks. This means that if an admin changed the join + rules to `private`, then new joins on forks based on parts of the DAG which + predate that change would always be accepted without being authed against + the join_rules event. + + +# Desirable Properties + +As well as the important properties listed in the "Background" section, there +are also some other properties that would significantly improve the experience +of end users, though not strictly essential. These include: + +* Banning and changing power levels should "do the right thing", i.e. end + users shouldn't have to take extra steps to make the state resolution + produce the "right" results. +* Minimise occurences of "state resets". Servers will sometimes point to + disparate parts of the room DAG (due to a variety of reasons), which ideally + should not result in changes in the state. +* Be efficient; state resolution can happen a lot on some large rooms. Ideally + it would also support efficiently working on "state deltas" - i.e. the + ability to calculate state resolution incrementally from snapshots rather + than having to consider the full state of each fork each time a conflict is + resolved + + +# Ideas for New Algorithm + + +## Auth Chain + +The _auth events_ of a given event is the set of events which justify why a +given event is allowed to be sent into a room (e.g. an m.room.create, an +m.room.power_levels and the sender's m.room.membership). The _auth chain_ of an +event is its auth events and their auth events, recursively. The auth chains of +a set of events in a given room form a DAG. + +"Auth events" are events that can appear as auth events of an event. These +include power levels, membership etc.[^1] + +Servers in a room are required to have the full auth chain for all events that +they have seen, and so the auth chain is available to be used by state +resolution algorithms. + + +## Unconflicted State + +The current algorithm defines the notion of "unconflicted state" to be all +entries that for each set of state either has the same event or no entry. All +unconflicted state entries are included in the resolved state. This is +problematic due to the fact that any new entries introduced on forks always +appear in the resolved state, regardless of if they would pass the checks +applied to conflicted state. + +The new algorithm could redefine "unconflicted state" to be all entries which +both exist and are the same in every state set (as opposed to previously where +the entry didn't need to exist in every state set). + + +## Replacing Depth + +Since depth of an event cannot be reliably calculated without possessing the +full DAG, and cannot be trusted when provided by other servers, it can not be +used in future versions of state resolution. A potential alternative, however, +is to use "origin_server_ts". While it cannot be relied on to be accurate─an +attacker can set it to arbitrary values─it has the advantage over depth that +end users can clearly see when a server is using incorrect values. (Note that +server clocks don't need to be particularly accurate for the ordering to still +be more useful than other arbitrary orderings). + +It can also be assumed that in most cases the origin_server_ts for a given +benign server will be mostly consistent. For example, if a server sends a join +and then a leave in the vast majority of cases the leave would have a greater +origin_server_ts. + +This makes "origin_server_ts" a good candidate to be used as a last resort to +order events if necessary, where otherwise a different arbitrary ordering would +be used. However, it's important that there is some other mechanism to ensure +that malicious servers can't abuse origin_server_ts to ensure their state +always gets picked during resolution (In the proposal below we use the auth DAG +ordering to override users who set state with malicious origin_server_ts.) + + +## Ordering and Authing + +Roughly, the current algorithm tries to ensure that moderation evasion doesn't +happen by ordering conflicted events by depth and (re)authing them +sequentially. The exact implementation has several issues, but the idea of +ensuring that state events from forks still need to pass auth subject to e.g. +bans and power level changes is a powerful one, as it reduces the utility of +maliciously forking. + +For that to work we need to ensure that there is a suitable ordering that puts +e.g. bans before events sent in other forks. (However events can point to old +parts of the DAG, for a variety of reasons, and ideally in that case the +resolved state would closely match the recent state). + +Similarly care should be taken when multiple changes to e.g. power levels happen +in a fork. If Alice gives Bob power (A), then Bob gives Charlie power (B) and +then Charlie, say, changes the ban level (C). If you try and resolve two state +sets one of which has A and the other has C, C will not pass auth unless B is +also taken into account. This case can be handled if we also consider the +difference in auth chains between the two sets, which in the previous example +would include B. + +(This is also the root cause of the "Hotel California" issue, where left users +get spontaneously rejoined to rooms. This happens when a user has a sequence of +memberships changes of the form: leave (A), join (B) and then another leave (C). +In the current algorithm a resoluton of A and C would pick A, and a resolution +of A and B would then pick B, i.e. the join. This means that a suitably forked +graph can reset the state to B. This is fixed if when resolving A and C we also +consider B, since its in the auth chain of C.) + + +## Power Level Ordering + +Actions that malicious servers would try and evade are actions that require +greater power levels to perform, for example banning, reducing power level, +etc. We define "power events" as events that have the potential to remove the +ability of another user to do something.[^2] (Note that they are a subset of +auth events.) + +In all these cases it is desirable for those privileged actions to take +precedence over events in other forks. This can be achieved by first +considering "power events", and requiring the remaining events to pass auth +based on them. + + +## Mainline + +An issue caused by servers not storing the full room DAG is that one can't tell +how two arbitrary events are ordered. The auth chain gives a partial ordering +to certain events, though far from complete; however, all events do contain a +reference to the current power levels in their auth events. As such if two +state events reference two different power levels events, and one power levels' +auth chain references the other, then there is a strong likelihood that the +event referencing the latter power level came after the other event. + +A "mainline" is a list of power levels events created if you pick a particular +power levels event (usually the current resolved power level) and recursively +follow each power level referenced in auth_events back to the first power level +event. + +The mainline can then be used to induce an ordering on events by looking at +where the power level referenced in their auth_events is in the mainline (or +recursively following the chain of power level events back until one is found +that appears in the mainline). This effectively partitions the room into +epochs, where a new epoch is started whenever a new power level is sent. + +If this mainline ordering is combined with ordering by origin_server_ts, then +it gives an ordering that is correct for servers that don't lie about the time, +while giving a mechanism that can be used to deal if a server lied (by room +admins starting a new epoch). + +The natural course of action for a room admin to take when noticing a +user/server is misbehaving is to ban them from the room, rather than changing +the power levels. It would therefore be useful if banning a user or server +started a new epoch as well. This would require being able to create a mainline +that includes power level events and bans[^3], which would suggest that power +level and ban events would need to point to the latest ban event as well. (This +would be significantly easier if we maintained a list of bans in a single +event, however there is concern that would limit the number of possible bans in +a room.) + + +# Proposed Algorithm + +First we define: + +* **"State sets"** are the sets of state that the resolution algorithm tries + to resolve, i.e. the inputs to the algorithm. +* **"Power events"** are events that have the potential to remove the ability + of another user to do something. These are power levels, join rules, bans + and kicks. +* The **"unconflicted state map"** is the state where the value of each key + exists and is the same in every state set. The **"conflicted state map"** is + everything else. (Note that this is subtly different to the definition used + in the existing algorithm, which considered the merge of a present event + with an absent event to be unconflicted rather than conflicted) +* The "**auth difference"** is calculated by first calculating the full auth + chain for each state set and taking every event that doesn't appear in every + auth chain. +* The **"full conflicted set"** is the union of the conflicted state map and + auth difference. +* The **"reverse topological power ordering"**[^4] of a set of events is an + ordering of the given events, plus any events in their auth chains that + appear in the auth difference, topologically ordered by their auth chains + with ties broken such that x < y if: + + 1. x's sender has a greater power level than y (calculated by looking at + their respective auth events, or if + 2. x's origin_server_ts is less than y's, or if + 3. x's event_id is lexicographically less than y's + + This is also known as a lexicographical topological sort (i.e. this is the + unique topological ordering such that for an entry x all entries after it + must either have x in their auth chain or be greater than x as defined + above). This can be implemented using Kahn's algorithm. + +* The **"mainline ordering"** based on a power level event P of a set of + events is calculated as follows: + 1. Generate the list of power levels starting at P and recursively take the + power level from its auth events. This list is called the mainline, + ordered such that P is last. + 1. We say the "closest mainline event" of an event is the first power level + event encountered in mainline when iteratively descending through the + power level events in the auth events. + 1. Order the set of events such that x < y if: + 1. The closest mainline event of x appears strictly before the closest + of y in the mainline list, or if + 1. x's origin_server_ts is less than y's, or if + 1. x's event_id lexicographically sorts before y's +* The **"iterative auth checks"** algorithm is where given a sorted list of + events, the auth check algorithm is applied to each event in turn. The state + events used to auth are built up from previous events that passed the auth + checks, starting from a base set of state. If a required auth key doesn't + exist in the state, then the one in the event's auth_events is used. (See + _Variations_ and _Attack Vectors_ below). + +The algorithm proceeds as follows: + + +1. Take all power events and any events in their auth chains that appear in the + _full_ _conflicted set_ and order them by the _reverse topological power + ordering._ +1. Apply the _iterative auth checks_ algorithm based on the unconflicted state + map to get a partial set of resolved state. +1. Take all remaining events that weren't picked in step 1 and order them by + the _mainline ordering_ based on the power level in the partially resolved + state. +1. Apply the _iterative auth checks algorithm_ based on the partial resolved + state. +1. Update the result with the _unconflicted state_ to get the final resolved + state[^5]. (_Note_: this is different from the current algorithm, which + considered different event types at distinct stages) + +An example python implementation can be found on github +[here](https://github.com/matrix-org/matrix-test-state-resolution-ideas). + +Note that this works best if we also change which events to include as an +event's auth_events. See the "Auth Events" section below. + + +## Discussion + +Essentially, the algorithm works by producing a sorted list of all conflicted +events (and differences in auth chains), and applies the auth checks one by +one, building up the state as it goes. The list is produced in two parts: first +the power events and auth dependencies are ordered by power level of the +senders and resolved, then the remaining events are ordered using the +"mainline" of the resolved power levels and then resolved to produce the final +resolved state. + +(This is equivalent to linearizing the full conflicted set of events and +reapplying the usual state updates and auth rules.) + + +### Variations + +There are multiple options for what to use as the base state for _iterative +auth checks_ algorithm; while it needs to be some variation of auth events and +unconflicted events, it is unclear exactly which combination is best (and least +manipulatable by malicious servers). + +Care has to be taken if we want to ensure that old auth events that appear in +the _auth chain difference_ can't supercede unconflicted state entries. + +Due to auth chain differences being added to the resolved states during +_iterative auth checks_, we therefore need to re-apply the unconflicted state +at the end to ensure that they appear in the final resolved state. This feels +like an odd fudge that shouldn't be necessary, and may point to a flaw in the +proposed algorithm. + + +### State Resets + +The proposed algorithm still has some potentially unexpected behaviour. + +One example of this is when Alice sets a topic and then gets banned. If an event +gets created (potentially much later) that points to both before and after the +topic and ban then the proposed algorithm will resolve and apply the ban before +resolving the topic, causing the topic to be denied and dropped from the +resolved state. This will result in no topic being set in the resolved state. + + +### Auth Events + +The algorithm relies heavily on the ordering induced by the auth chain DAG. + +There are two types of auth events (not necessarily distinct): + +* Those that give authorization to do something +* Those that revoke authorization to do something. + +For example, invites/joins are in the former category, leaves/kicks/bans are in +the latter and power levels are both. + +Assuming[^6] revocations always point to (i.e., have in their auth chain) the +authorization event that they are revoking, and authorization events point to +revocations that they are superseding, then the algorithm will ensure that the +authorization events are applied in order (so generally the "latest" +authorization state would win). + +This helps ensure that e.g. an invite cannot be reused after a leave/kick, +since the leave (revocation) would have the invite in their auth chain. + +This idea also relies on revocations replacing the state that granted +authorization to do an action (and vice versa). For example, in the current +model bans (basically) revoke the ability for a particular user from being able +to join. If the user later gets unbanned and then rejoins, the join would point +to the join rules as the authorization that lets them join, but would not +(necessarily) point to the unban. This has the effect that if a state resolution +happened between the new join and the ban, the unban would not be included in +the resolution and so the join would be rejected. + +The changes to the current model that would be required to make the above +assumptions true would be, for example: + +1. By default permissions are closed. +1. Bans would need to be a list in either the join rules event or a separate + event type which all membership events pointed to. +1. Bans would only revoke the ability to join, not automatically remove users + from the room. +1. Change the defaults of join_rules to be closed by default + + +### Efficiency and Delta State Resolution + +The current (unoptimised) implementation of the algorithm is 10x slower than +the current algorithm, based on a single, large test case. While hopefully some +optimisations can be made, the ability to [incrementally calculate state +resolution via deltas](https://github.com/matrix-org/synapse/pull/3122) will +also mitigate some of the slow down. + +Another aspect that should be considered is the amount of data that is required +to perform the resolution. The current algorithm only requires the events for +the conflicted set, plus the events from the unconflicted set needed to auth +them. The proposed algorithm also requires the events in the auth chain +difference (calculating the auth chain difference may also require more data to +calculate). + +Delta state resolution is where if you have, say, two state sets and their +resolution, then you can use that result to work out the new resolution where +there has been a small change to the state sets. For the proposed algorithm, if +the following properties hold true then the result can be found by simply +applying steps 3 and 4 to the state deltas. The properties are: + + + +1. The delta contains no power events +1. The origin_server_ts of all events in state delta are strictly greater than + those in the previous state sets +1. Any event that has been removed must not have been used to auth subsequent + events (e.g. if we replaced a member event and that user had also set a + topic) + +These properties will likely hold true for most state updates that happen in a +room, allowing servers to use this more efficient algorithm the majority of the +time. + + +### Full DAG + +It's worth noting that if the algorithm had access to the full room DAG that it +would really only help by ensuring that the ordering in "reverse topological +ordering" and "mainline ordering" respected the ordering induced by the DAG. + +This would help, e.g., ensure the latest topic was always picked rather than +rely on origin_server_ts and mainline. As well as obviate the need to maintain +a separate auth chain, and the difficulties that entails (like having to +reapply the unconflicted state at the end). + + +### Attack Vectors + +The main potential attack vector that needs to be considered is in the +_iterative auth checks_ algorithm, and whether an attacker could make use of +the fact that it's based on the unconflicted state and/or auth events of the +event. + + +# Appendix + +The following is an example room DAG, where time flows down the page. We shall +work through resolving the state at both _Message 2_ and _Message 3_. + + +![alt_text](images/state-res.png) + + +(Note that green circles are events sent by Alice, blue circles sent by Bob and +black arrows point to previous events. The red arrows are the mainline computed +during resolution.) + +First we resolve the state at _Message 2_. The conflicted event types are the +power levels and topics, and since the auth chains are the same for both state +sets the auth difference is the empty set. + +Step 1: The _full conflicted set_ are the events _P2, P3, Topic 2 _and _Topic +3_, of which _P2_ and _P3_ are the only power events. Since Alice (the room +creator) has a greater power level than Bob (and neither _P2 _and _P3_ appear +in each other's auth chain), the reverse topological ordering is: [_P2, P3_]. + +Step 2: Now we apply the auth rules iteratively, _P2_ trivially passes based on +the unconflicted state, but _P3_ does not pass since after _P2_ Bob no longer +has sufficient power to set state. This results in the power levels resolving +to _P2_. + +Step 3: Now we work out the mainline based on P2, which is coloured in red on +the diagram. We use the mainline to order _Topic 2_ and _Topic 3_. _Topic 2_ +points to_ P1_, while the closest mainline to _Topic 3_ is also _P1_. We then +order based on the _origin_server_ts_ of the two events, let's assume that +gives us: [_Topic 2_, _Topic 3_]. + +Step 4: Iteratively applying the auth rules results in _Topic 2_ being allowed, +but _Topic 3 _being denied (since Bob doesn't have power to set state anymore), +so the topic is resolved to _Topic 2_. + +This gives the resolved state at _Message 2_ to be _P2 _and _Topic 2_. (This is +actually the same result as the existing algorithm gives) + +Now let's look at the state at _Message 3_. + +Step 1: The full conflicted set are simple: _Topic 2_ and _Topic 4_. There are +no conflicted power events. + +Step 2: N/A + +Step 3: _Topic 2_ points to _P1_ in the mainline, and _Topic 4_ points to _P2_ +in its auth events. Since _P2_ comes after _P1_ in the mainline, this gives an +ordering of [_Topic 2, Topic 4_]. + +Step 4: Iteratively applying the auth rules results in both topics passing the +auth checks, and so the last topic, _Topic 4_, is chosen. + +This gives the resolved state at _Message 3_ to be _Topic 4_. + + +## Notes + +[^1]: In the current room protocol these are: the create event, power levels, + membership, join rules and third party invites. See the + [spec](https://matrix.org/docs/spec/server_server/unstable.html#pdu-fields). + +[^2]: In the current protocol these are: power levels, kicks, bans and join + rules. + +[^3]: Future room versions may have a concept of server ban event that works + like existing bans, which would also be included + +[^4]: The topology being considered here is the auth chain DAG, rather than the + room DAG, so this ordering is only applicable to events which appear in the + auth chain DAG. + +[^5]: We do this so that, if we receive events with misleading auth_events, this + ensures that the unconflicted state at least is correct. + +[^6]: This isn't true in the current protocol + + + diff --git a/proposals/images/state-res.png b/proposals/images/state-res.png new file mode 100644 index 00000000..573d1927 Binary files /dev/null and b/proposals/images/state-res.png differ diff --git a/schemas/server-signatures.yaml b/schemas/server-signatures.yaml new file mode 100644 index 00000000..a1855256 --- /dev/null +++ b/schemas/server-signatures.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +example: { + "example.com": { + "ed25519:0": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } +} +additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string \ No newline at end of file diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 16c40af5..72659bb8 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) - with open(os.path.join(spec_dir, file_info), "r") as f: + with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f: rst = None if adjust_titles: rst = load_with_adjusted_titles( @@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): def build_spec(target, out_filename): log("Building templated file %s" % out_filename) - with open(out_filename, "wb") as outfile: + with open(out_filename, "w", encoding="utf-8") as outfile: for file_info in target["files"]: section = get_rst( file_info=file_info, @@ -195,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section.encode('UTF-8')) + outfile.write(section) """ @@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename): "^[" + re.escape("".join(title_styles)) + "]{3,}$" ) current_title_style = None - with open(filename, "r") as infile: - with open(out_filename, "w") as outfile: + with open(filename, "r", encoding="utf-8") as infile: + with open(out_filename, "w", encoding="utf-8") as outfile: for line in infile.readlines(): if not relative_title_matcher.match(line): if title_matcher.match(line): @@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename): def rst2html(i, o, stylesheets): log("rst2html %s -> %s" % (i, o)) - with open(i, "r") as in_file: - with open(o, "w") as out_file: + with open(i, "r", encoding="utf-8") as in_file: + with open(o, "w", encoding="utf-8") as out_file: publish_file( source=in_file, destination=out_file, @@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "rb") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() replacement = r'

\n\1' - with open(path, "wb") as f: + with open(path, "w", encoding="utf-8") as f: for line in lines: - line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write((line + "\n").encode('UTF-8')) + f.write(line + "\n") def run_through_template(input_files, set_verbose, substitutions): @@ -518,6 +517,18 @@ if __name__ == '__main__': "--server_release", "-s", action="store", default="unstable", help="The server-server release tag to generate, e.g. r1.2" ) + parser.add_argument( + "--appservice_release", "-a", action="store", default="unstable", + help="The appservice release tag to generate, e.g. r1.2" + ) + parser.add_argument( + "--push_gateway_release", "-p", action="store", default="unstable", + help="The push gateway release tag to generate, e.g. r1.2" + ) + parser.add_argument( + "--identity_release", "-i", action="store", default="unstable", + help="The identity service release tag to generate, e.g. r1.2" + ) parser.add_argument( "--list_targets", action="store_true", help="Do not update the specification. Instead print a list of targets.", @@ -536,12 +547,14 @@ if __name__ == '__main__': substitutions = { "%CLIENT_RELEASE_LABEL%": args.client_release, - # we hardcode a major version of r0. This ends up in the - # example API URLs. When we have released a new major version, - # we'll have to bump it. + # we hardcode the major versions. This ends up in the example + # API URLs. When we have released a new major version, we'll + # have to bump them. "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, - "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, + "%IDENTITY_RELEASE_LABEL%": args.identity_release, + "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index ed08f81d..76032850 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,8 +8,11 @@ cd `dirname $0`/.. mkdir -p assets -# generate specification/proposals.rst -./scripts/proposals.py +if [ "$CIRCLECI" != "true" ] +then + # generate specification/proposals.rst + ./scripts/proposals.py +fi # generate the spec docs ./scripts/gendoc.py -d assets/spec diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 1a93c723..4c07649d 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -31,6 +31,23 @@ class MatrixSections(Sections): def render_client_server_changelog(self): changelogs = self.units.get("changelogs") return changelogs["client_server"] + + # TODO: We should make this a generic variable instead of having to add functions all the time. + def render_push_gateway_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["push_gateway"] + + def render_identity_service_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["identity_service"] + + def render_server_server_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["server_server"] + + def render_application_service_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["application_service"] def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 81da3f6a..94b535b5 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -125,7 +125,7 @@ def resolve_references(path, schema): if '$ref' in schema: value = schema['$ref'] path = os.path.join(os.path.dirname(path), value) - with open(path) as f: + with open(path, encoding="utf-8") as f: ref = yaml.load(f, OrderedLoader) result = resolve_references(path, ref) del schema['$ref'] @@ -664,11 +664,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger API: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") group_name = "%s_%s" % (group_name, suffix) - api = yaml.load(f.read(), OrderedLoader) + api = yaml.load(f, OrderedLoader) api = resolve_references(filepath, api) api["__meta"] = self._load_swagger_meta( api, group_name @@ -698,11 +698,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger definition: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) group_name = "%s_%s" % (prefix, group_name) - definition = yaml.load(f.read(), OrderedLoader) + definition = yaml.load(f, OrderedLoader) definition = resolve_references(filepath, definition) if 'type' not in definition: continue @@ -741,7 +741,7 @@ class MatrixUnits(Units): event_type = filename[:-5] # strip the ".yaml" logger.info("Reading event schema: %s" % filepath) - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: event_schema = yaml.load(f, OrderedLoader) schema_info = process_data_type( @@ -754,6 +754,9 @@ class MatrixUnits(Units): def load_apis(self, substitutions): cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") + is_ver = substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable") + as_ver = substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable") + push_gw_ver = substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable") # we abuse the typetable to return this info to the templates return TypeTable(rows=[ @@ -766,16 +769,16 @@ class MatrixUnits(Units): fed_ver, "Federation between servers", ), TypeTableRow( - "`Application Service API `_", - "unstable", + "`Application Service API `_", + as_ver, "Privileged server plugins", ), TypeTableRow( - "`Identity Service API `_", + "`Identity Service API `_", "unstable", "Mapping of third party IDs to Matrix IDs", ), TypeTableRow( - "`Push Gateway API `_", - "unstable", + "`Push Gateway API `_", + push_gw_ver, "Push notifications for Matrix events", ), ]) @@ -791,8 +794,8 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) logger.info("Reading event example: %s" % filepath) try: - with open(filepath, "r") as f: - example = json.load(f) + with open(filepath, "r", encoding="utf-8") as f: + example = resolve_references(filepath, json.load(f)) examples[filename] = examples.get(filename, []) examples[filename].append(example) if filename != event_name: @@ -829,7 +832,7 @@ class MatrixUnits(Units): def read_event_schema(self, filepath): logger.info("Reading %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: json_schema = yaml.load(f, OrderedLoader) schema = { @@ -941,7 +944,7 @@ class MatrixUnits(Units): title_part = None changelog_lines = [] - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() prev_line = None for line in (tc_lines + lines): diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index fc89f031..92071611 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -1,5 +1,5 @@ .. Copyright 2016 Openmarket Ltd. -.. Copyright 2017 New Vector Ltd. +.. Copyright 2017, 2018 New Vector Ltd. .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -23,13 +23,37 @@ A homeserver is uniquely identified by its server name. This value is used in a number of identifiers, as described below. The server name represents the address at which the homeserver in question can -be reached by other homeservers. The complete grammar is:: +be reached by other homeservers. All valid server names are included by the +following grammar:: - server_name = host [ ":" port] - port = *DIGIT + server_name = hostname [ ":" port ] -where ``host`` is as defined by `RFC3986, section 3.2.2 -`_. + port = *DIGIT + + hostname = IPv4address / "[" IPv6address "]" / dns-name + + IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + + IPv6address = 2*45IPv6char + + IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." + ; 0-9, A-F, a-f, :, . + + dns-name = *255dns-char + + dns-char = DIGIT / ALPHA / "-" / "." + +— in other words, the server name is the hostname, followed by an optional +numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, +an IPv6 address literal surrounded with square brackets, or a DNS name. + +IPv4 literals must be a sequence of four decimal numbers in the +range 0 to 255, separated by ``.``. IPv6 literals must be as specified by +`RFC3513, section 2.2 `_. + +DNS names for use with Matrix should follow the conventional restrictions for +internet hostnames: they should consist of a series of labels separated by +``.``, where each label consists of the alphanumeric characters or hyphens. Examples of valid server names are: @@ -40,6 +64,20 @@ Examples of valid server names are: * ``[1234:5678::abcd]`` (IPv6 literal) * ``[1234:5678::abcd]:5678`` (IPv6 literal with explicit port) +.. Note:: + + This grammar is based on the standard for internet host names, as specified + by `RFC1123, section 2.1 `_, + with an extension for IPv6 literals. + +Server names must be treated case-sensitively: in other words, +``@user:matrix.org`` is a different person from ``@user:MATRIX.ORG``. + +Some recommendations for a choice of server name follow: + +* The length of the complete server name should not exceed 230 characters. +* Server names should not use upper-case characters. + Room Versions ~~~~~~~~~~~~~ @@ -51,7 +89,7 @@ not understanding the new rules. A room version is defined as a string of characters which MUST NOT exceed 32 codepoints in length. Room versions MUST NOT be empty and SHOULD contain only -the characters ``a-z``, ``0-9``, ``.``, and ``-``. +the characters ``a-z``, ``0-9``, ``.``, and ``-``. Room versions are not intended to be parsed and should be treated as opaque identifiers. Room versions consisting only of the characters ``0-9`` and ``.`` @@ -283,3 +321,45 @@ domain). .. TODO-spec - Need to specify precise grammar for Room Aliases. https://matrix.org/jira/browse/SPEC-391 + +matrix.to navigation +++++++++++++++++++++ + +.. NOTE:: + This namespacing is in place pending a ``matrix://`` (or similar) URI scheme. + This is **not** meant to be interpreted as an available web service - see + below for more details. + +Rooms, users, aliases, and groups may be represented as a "matrix.to" URI. +This URI can be used to reference particular objects in a given context, such +as mentioning a user in a message or linking someone to a particular point +in the room's history (a permalink). + +A matrix.to URI has the following format, based upon the specification defined +in RFC 3986: + + https://matrix.to/#// + +The identifier may be a room ID, room alias, user ID, or group ID. The extra +parameter is only used in the case of permalinks where an event ID is referenced. +The matrix.to URI, when referenced, must always start with ``https://matrix.to/#/`` +followed by the identifier. + +Clients should not rely on matrix.to URIs falling back to a web server if accessed +and instead should perform some sort of action within the client. For example, if +the user were to click on a matrix.to URI for a room alias, the client may open +a view for the user to participate in the room. + +Examples of matrix.to URIs are: + +* Room alias: ``https://matrix.to/#/#somewhere:domain.com`` +* Room: ``https://matrix.to/#/!somewhere:domain.com`` +* Permalink by room: ``https://matrix.to/#/!somewhere:domain.com/$event:example.org`` +* Permalink by room alias: ``https://matrix.to/#/#somewhere:domain.com/$event:example.org`` +* User: ``https://matrix.to/#/@alice:example.org`` +* Group: ``https://matrix.to/#/+example:domain.com`` + +.. Note:: + Room ID permalinks are unroutable as there is no reliable domain to send requests + to upon receipt of the permalink. Clients should do their best route Room IDs to + where they need to go, however they should also be aware of `issue #1579 `_. \ No newline at end of file diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 51280341..69d39d21 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -30,22 +30,37 @@ irrespective of the underlying homeserver implementation. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + + +.. topic:: Version: unstable +{{application_service_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. + + Application Services -------------------- -Application services are passive and can only observe events from a given -homeserver (HS). They can inject events into rooms they are participating in. +Application services are passive and can only observe events from homeserver. +They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the application service. This is achieved by manually configuring the homeserver -with information about the application service (AS). +with information about the application service. Registration ~~~~~~~~~~~~ @@ -169,6 +184,34 @@ An example registration file for an IRC-bridging application service is below: Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Legacy routes ++++++++++++++ + +Previous drafts of the application service specification had a mix of endpoints +that have been used in the wild for a significant amount of time. The application +service specification now defines a version on all endpoints to be more compatible +with the rest of the Matrix specification and the future. + +Homeservers should attempt to use the specified endpoints first when communicating +with application services. However, if the application service receives an http status +code that does not indicate success (ie: 404, 500, 501, etc) then the homeserver +should fall back to the older endpoints for the application service. + +The older endpoints have the exact same request body and response format, they +just belong at a different path. The equivalent path for each is as follows: + +* ``/_matrix/app/v1/transactions/{txnId}`` becomes ``/transactions/{txnId}`` +* ``/_matrix/app/v1/users/{userId}`` becomes ``/users/{userId}`` +* ``/_matrix/app/v1/rooms/{roomAlias}`` becomes ``/rooms/{roomAlias}`` +* ``/_matrix/app/v1/thirdparty/protocol/{protocol}`` becomes ``/_matrix/app/unstable/thirdparty/protocol/{protocol}`` +* ``/_matrix/app/v1/thirdparty/user/{user}`` becomes ``/_matrix/app/unstable/thirdparty/user/{user}`` +* ``/_matrix/app/v1/thirdparty/location/{location}`` becomes ``/_matrix/app/unstable/thirdparty/location/{location}`` +* ``/_matrix/app/v1/thirdparty/user`` becomes ``/_matrix/app/unstable/thirdparty/user`` +* ``/_matrix/app/v1/thirdparty/location`` becomes ``/_matrix/app/unstable/thirdparty/location`` + +Homeservers should periodically try again for the newer endpoints because the +application service may have been updated. + Pushing events ++++++++++++++ @@ -179,24 +222,24 @@ events. Each list of events includes a transaction ID, which works as follows: Typical HS ---> AS : Homeserver sends events with transaction ID T. - <--- : AS sends back 200 OK. + <--- : Application Service sends back 200 OK. AS ACK Lost HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. HS ---> AS : Homeserver retries with the same transaction ID of T. - <--- : AS sends back 200 OK. If the AS had processed these events - already, it can NO-OP this request (and it knows if it is the same - events based on the transaction ID). + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). The events sent to the application service should be linearised, as if they were from the event stream. The homeserver MUST maintain a queue of transactions to -send to the AS. If the application service cannot be reached, the homeserver -SHOULD backoff exponentially until the application service is reachable again. +send to the application service. If the application service cannot be reached, the +homeserver SHOULD backoff exponentially until the application service is reachable again. As application services cannot *modify* the events in any way, these requests can be made without blocking other aspects of the homeserver. Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID -on retries, as the AS may have already processed the events. +on retries, as the application service may have already processed the events. {{transactions_as_http_api}} @@ -289,26 +332,37 @@ An example request would be:: GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org Authorization: Bearer YourApplicationServiceTokenHere +.. TODO-TravisR: Temporarily take out timestamp massaging while we're releasing r0. + See https://github.com/matrix-org/matrix-doc/issues/1585 +.. Timestamp massaging + +++++++++++++++++++ + The application service may want to inject events at a certain time (reflecting + the time on the network they are tracking e.g. irc, xmpp). Application services + need to be able to adjust the ``origin_server_ts`` value to do this. -Timestamp massaging -+++++++++++++++++++ -The application service may want to inject events at a certain time (reflecting -the time on the network they are tracking e.g. irc, xmpp). Application services -need to be able to adjust the ``origin_server_ts`` value to do this. + Inputs: + - Application service token (``as_token``) + - Desired timestamp (in milliseconds since the unix epoch) -Inputs: - - Application service token (``as_token``) - - Desired timestamp (in milliseconds since the unix epoch) + Notes: + - This will only apply when sending events. -Notes: - - This will only apply when sending events. + :: -:: + PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 + Authorization: Bearer YourApplicationServiceTokenHere - PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 - Authorization: Bearer YourApplicationServiceTokenHere + Content: The event to send, as per the Client-Server API. + +Timestamp massaging ++++++++++++++++++++ - Content: The event to send, as per the Client-Server API. +Previous drafts of the Application Service API permitted application services +to alter the timestamp of their sent events by providing a ``ts`` query parameter +when sending an event. This API has been excluded from the first release due to +design concerns, however some servers may still support the feature. Please visit +`issue #1585 `_ for more +information. Server admin style permissions ++++++++++++++++++++++++++++++ @@ -326,7 +380,7 @@ users needs API changes in order to: - Have a 'passwordless' user. This involves bypassing the registration flows entirely. This is achieved by -including the AS token on a ``/register`` request, along with a login type of +including the ``as_token`` on a ``/register`` request, along with a login type of ``m.login.application_service`` to set the desired user ID without a password. :: @@ -363,13 +417,18 @@ additional parameters on the ``/publicRooms`` client-server endpoint. {{appservice_room_directory_cs_http_api}} -Event fields -~~~~~~~~~~~~ +Referencing messages from a third party network +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Application services should include an ``external_url`` in the ``content`` of +events it emits to indicate where the message came from. This typically applies +to application services that bridge other networks into Matrix, such as IRC, +where an HTTP URL may be available to reference. -.. TODO-TravisR: Fix this section to be a general "3rd party networks" section +Clients should provide users with a way to access the ``external_url`` if it +is present. Clients should additionally ensure the URL has a scheme of ``https`` +or ``http`` before making use of it. -We recommend that any events that originated from a remote network should -include an ``external_url`` field in their content to provide a way for Matrix -clients to link into the 'native' client from which the event originated. -For instance, this could contain the message-ID for emails/nntp posts, or a link -to a blog comment when bridging blog comment traffic in & out of Matrix. +The presence of an ``external_url`` on an event does not necessarily mean the +event was sent from an application service. Clients should be wary of the URL +contained within, as it may not be a legitimate reference to the event's source. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 71c60097..b377cbb8 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -189,6 +189,82 @@ headers to be returned by servers on all requests are: Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization +Server Discovery +---------------- + +In order to allow users to connect to a Matrix server without needing to +explicitly specify the homeserver's URL or other parameters, clients SHOULD use +an auto-discovery mechanism to determine the server's URL based on a user's +Matrix ID. Auto-discovery should only be done at login time. + +In this section, the following terms are used with specific meanings: + +``PROMPT`` + Retrieve the specific piece of information from the user in a way which + fits within the existing client user experience, if the client is inclined to + do so. Failure can take place instead if no good user experience for this is + possible at this point. + +``IGNORE`` + Stop the current auto-discovery mechanism. If no more auto-discovery + mechanisms are available, then the client may use other methods of + determining the required parameters, such as prompting the user, or using + default values. + +``FAIL_PROMPT`` + Inform the user that auto-discovery failed due to invalid/empty data and + ``PROMPT`` for the parameter. + +``FAIL_ERROR`` + Inform the user that auto-discovery did not return any usable URLs. Do not + continue further with the current login process. At this point, valid data + was obtained, but no homeserver is available to serve the client. No further + guess should be attempted and the user should make a conscientious decision + what to do next. + +Well-known URI +~~~~~~~~~~~~~~ + +The ``.well-known`` method uses a JSON file at a predetermined location to +specify parameter values. The flow for this method is as follows: + +1. Extract the server name from the user's Matrix ID by splitting the Matrix ID + at the first colon. +2. Extract the hostname from the server name. +3. Make a GET request to ``https://hostname/.well-known/matrix/client``. + + a. If the returned status code is 404, then ``IGNORE``. + b. If the returned status code is not 200, or the response body is empty, + then ``FAIL_PROMPT``. + c. Parse the response body as a JSON object + + i. If the content cannot be parsed, then ``FAIL_PROMPT``. + + d. Extract the ``base_url`` value from the ``m.homeserver`` property. This + value is to be used as the base URL of the homeserver. + + i. If this value is not provided, then ``FAIL_PROMPT``. + + e. Validate the homeserver base URL: + + i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. + ii. Clients SHOULD validate that the URL points to a valid homeserver + before accepting it by connecting to the |/_matrix/client/versions|_ + endpoint, ensuring that it does not return an error, and parsing and + validating that the data conforms with the expected response + format. If any step in the validation fails, then + ``FAIL_ERROR``. Validation is done as a simple check against + configuration errors, in order to ensure that the discovered address + points to a valid homeserver. + + f. If the ``m.identity_server`` property is present, extract the + ``base_url`` value for use as the base URL of the identity server. + Validation for this URL is done as in the step above, but using + ``/_matrix/identity/api/v1`` as the endpoint to connect to. If the + ``m.identity_server`` property is present, but does not have a + ``base_url`` value, then ``FAIL_ERROR``. + +{{wellknown_cs_http_api}} Client Authentication --------------------- @@ -1596,5 +1672,8 @@ have to wait in milliseconds before they can try again. .. |/user//account_data/| replace:: ``/user//account_data/`` .. _/user//account_data/: #put-matrix-client-%CLIENT_MAJOR_VERSION%-user-userid-account-data-type +.. |/_matrix/client/versions| replace:: ``/_matrix/client/versions`` +.. _/_matrix/client/versions: #get-matrix-client-versions + .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 .. _`3PID Types`: ../appendices.html#pid-types diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index 3b037caf..81ff0ede 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -1,6 +1,7 @@ .. Copyright 2016 OpenMarket Ltd .. Copyright 2017 Kamax.io .. Copyright 2017 New Vector Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -23,18 +24,32 @@ user identifiers. From time to time, it is useful to refer to users by other number. This identity service specification describes how mappings between third-party identifiers and Matrix user identifiers can be established, validated, and used. This description technically may apply to any 3pid, but in -practice has only been applied specifically to email addresses. +practice has only been applied specifically to email addresses and phone numbers. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + +.. topic:: Version: %IDENTITY_RELEASE_LABEL% +{{identity_service_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/identity_service.rst + + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. + General principles ------------------ @@ -56,6 +71,75 @@ is left as an exercise for the client. 3PID types are described in `3PID Types`_ Appendix. +API standards +------------- + +The mandatory baseline for identity service communication in Matrix is exchanging +JSON objects over HTTP APIs. HTTPS is required for communication, and all API calls +use a Content-Type of ``application/json``. In addition, strings MUST be encoded as +UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard error response". +This is a JSON object which looks like: + +.. code:: json + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. There may be +additional keys depending on the error, but the keys ``error`` and ``errcode`` +MUST always be present. + +Some standard error codes are below: + +:``M_NOT_FOUND``: + The resource requested could not be located. + +:``M_MISSING_PARAMS``: + The request was missing one or more parameters. + +:``M_INVALID_PARAM``: + The request contained one or more invalid parameters. + +:``M_SESSION_NOT_VALIDATED``: + The session has not been validated. + +:``M_NO_VALID_SESSION``: + A session could not be located for the given parameters. + +:``M_SESSION_EXPIRED``: + The session has expired and must be renewed. + +:``M_INVALID_EMAIL``: + The email address provided was not valid. + +:``M_EMAIL_SEND_ERROR``: + There was an error sending an email. Typically seen when attempting to verify + ownership of a given email address. + +:``M_INVALID_ADDRESS``: + The provided third party address was not valid. + +:``M_SEND_ERROR``: + There was an error sending a notification. Typically seen when attempting to + verify ownership of a given third party address. + +:``M_UNRECOGNIZED``: + The request contained an unrecognised value, such as an unknown token or medium. + +:``M_THREEPID_IN_USE``: + The third party identifier is already in use by another user. Typically this + error will have an additional ``mxid`` property to indicate who owns the + third party identifier. + +:``M_UNKNOWN``: + An unknown error has occurred. + Privacy ------- @@ -67,6 +151,22 @@ should allow a 3pid to be mapped to a Matrix user identity, but not in the other direction (i.e. one should not be able to get all 3pids associated with a Matrix user ID, or get all 3pids associated with a 3pid). +Web browser clients +------------------- + +It is realistic to expect that some clients will be written to be run within a web +browser or similar environment. In these cases, the identity service should respond to +pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers on all +requests. + +When a client approaches the server with a pre-flight (OPTIONS) request, the server +should respond with the CORS headers for that route. The recommended CORS headers +to be returned by servers on all requests are:: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + Status check ------------ @@ -77,25 +177,24 @@ Key management An identity service has some long-term public-private keypairs. These are named in a scheme ``algorithm:identifier``, e.g. ``ed25519:0``. When signing an -association, the Matrix standard JSON signing format is used, as specified in -the server-server API specification under the heading "Signing Events". +association, the standard `Signing JSON`_ algorithm applies. In the event of key compromise, the identity service may revoke any of its keys. An HTTP API is offered to get public keys, and check whether a particular key is valid. -The identity server may also keep track of some short-term public-private +The identity service may also keep track of some short-term public-private keypairs, which may have different usage and lifetime characteristics than the service's long-term keys. {{pubkey_is_http_api}} -Association Lookup +Association lookup ------------------ {{lookup_is_http_api}} -Establishing Associations +Establishing associations ------------------------- The flow for creating an association is session-based. @@ -114,6 +213,12 @@ session, within a 24 hour period since its most recent modification. Any attempts to perform these actions after the expiry will be rejected, and a new session should be created and used instead. +To start a session, the client makes a request to the appropriate ``/requestToken`` +endpoint. The user then receives a validation token which should be provided +to the client. The client then provides the token to the appropriate ``/submitToken`` +endpoint, completing the session. At this point, the client should ``/bind`` the +third party identifier or leave it for another entity to bind. + Email associations ~~~~~~~~~~~~~~~~~~ @@ -129,53 +234,31 @@ General {{associations_is_http_api}} -Invitation Storage +Invitation storage ------------------ An identity service can store pending invitations to a user's 3pid, which will be retrieved and can be either notified on or look up when the 3pid is associated with a Matrix user ID. -At a later point, if the owner of that particular 3pid binds it with a Matrix user ID, the identity server will attempt to make an HTTP POST to the Matrix user's homeserver which looks roughly as below:: - - POST https://bar.com:8448/_matrix/federation/v1/3pid/onbind - Content-Type: application/json - - { - "medium": "email", - "address": "foo@bar.baz", - "mxid": "@alice:example.tld", - "invites": [ - { - "medium": "email", - "address": "foo@bar.baz", - "mxid": "@alice:example.tld", - "room_id": "!something:example.tld", - "sender": "@bob:example.tld", - "signed": { - "mxid": "@alice:example.tld", - "signatures": { - "vector.im": { - "ed25519:0": "somesignature" - } - }, - "token": "sometoken" - } - } - ] - } - -Where the signature is produced using a long-term private key. +At a later point, if the owner of that particular 3pid binds it with a Matrix user +ID, the identity service will attempt to make an HTTP POST to the Matrix user's +homeserver via the `/3pid/onbind`_ endpoint. The request MUST be signed with a +long-term private key for the identity service. {{store_invite_is_http_api}} Ephemeral invitation signing ---------------------------- -To aid clients who may not be able to perform crypto themselves, the identity service offers some crypto functionality to help in accepting invitations. -This is less secure than the client doing it itself, but may be useful where this isn't possible. +To aid clients who may not be able to perform crypto themselves, the identity +service offers some crypto functionality to help in accepting invitations. +This is less secure than the client doing it itself, but may be useful where +this isn't possible. {{invitation_signing_is_http_api}} .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 .. _`3PID Types`: ../appendices.html#pid-types +.. _`Signing JSON`: ../appendices.html#signing-json +.. _`/3pid/onbind`: ../server_server/unstable.html#put-matrix-federation-v1-3pid-onbind diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 1c031ee1..d0ed201a 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -31,7 +31,7 @@ The client recieves the account data as events in the ``account_data`` sections of a ``/sync``. These events can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` +``account_data`` section of a room in ``/sync``. ``m.tag`` events appearing in ``/events`` will have a ``room_id`` with the room the tags are for. diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index fa461cc2..170b70f9 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -283,6 +283,31 @@ Device verification may reach one of several conclusions. For example: decrypted by such a device. For the Olm protocol, this is documented at https://matrix.org/git/olm/about/docs/signing.rst. +Key sharing +----------- + +If Bob has an encrypted conversation with Alice on his computer, and then logs in +through his phone for the first time, he may want to have access to the previously +exchanged messages. To address this issue, events exist for requesting and sending +keys from device to device. + +When a device is missing keys to decrypt messages, it can request the keys by +sending `m.room_key_request`_ to-device messages to other devices with +``action`` set to ``request``. If a device wishes to share the keys with that +device, it can forward the keys to the first device by sending an encrypted +`m.forwarded_room_key`_ to-device message. The first device should then send an +`m.room_key_request`_ to-device message with ``action`` set to +``cancel_request`` to the other devices that it had originally sent the key +request to; a device that receives a ``cancel_request`` should disregard any +previously-received ``request`` message with the same ``request_id`` and +``requesting_device_id``. + +.. NOTE:: + + Key sharing can be a big attack vector, thus it must be done very carefully. + A reasonable stategy is for a user's client to only send keys requested by the + verified devices of the same user. + Messaging Algorithms -------------------- @@ -391,6 +416,12 @@ this check, a client cannot be sure that the sender device owns the private part of the ed25519 key it claims to have in the Olm payload. This is crucial when the ed25519 key corresponds to a verified device. +If a client has multiple sessions established with another device, it should +use the session from which it last received a message. A client may expire old +sessions by defining a maximum number of olm sessions that it will maintain for +each device, and expiring sessions on a Least Recently Used basis. The maximum +number of olm sessions maintained per device should be at least 4. + ``m.megolm.v1.aes-sha2`` ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -464,6 +495,10 @@ Events {{m_room_key_event}} +{{m_room_key_request_event}} + +{{m_forwarded_room_key_event}} + Key management API ~~~~~~~~~~~~~~~~~~ diff --git a/specification/modules/mentions.rst b/specification/modules/mentions.rst new file mode 100644 index 00000000..4501b776 --- /dev/null +++ b/specification/modules/mentions.rst @@ -0,0 +1,74 @@ +.. Copyright 2018 New Vector Ltd. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +User, room, and group mentions +============================== + +.. _module:mentions: + +This module allows users to mention other users, rooms, and groups within +a room message. This is achieved by including a `matrix.to URI`_ in the HTML +body of an `m.room.message`_ event. This module does not have any server-specific +behaviour to it. + +Mentions apply only to `m.room.message`_ events where the ``msgtype`` is ``m.text``, +``m.emote``, or ``m.notice``. The ``format`` for the event must be ``org.matrix.custom.html`` +and therefore requires a ``formatted_body``. + +To make a mention, reference the entity being mentioned in the ``formatted_body`` +using an anchor, like so:: + + { + "body": "Hello Alice!", + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": "Hello Alice!" + } + + +Client behaviour +---------------- + +In addition to using the appropriate ``matrix.to URI`` for the mention, +clients should use the following guidelines when making mentions in events +to be sent: + +* When mentioning users, use the user's potentially ambigious display name for + the anchor's text. If the user does not have a display name, use the user's + ID. + +* When mentioning rooms, use the canonical alias for the room. If the room + does not have a canonical alias, prefer one of the aliases listed on the + room. If no alias can be found, fall back to the room ID. In all cases, + use the alias/room ID being linked to as the anchor's text. + +* When referencing groups, use the group ID as the anchor's text. + +The text component of the anchor should be used in the event's ``body`` where +the mention would normally be represented, as shown in the example above. + +Clients should display mentions differently from other elements. For example, +this may be done by changing the background color of the mention to indicate +that it is different from a normal link. + +If the current user is mentioned in a message (either by a mention as defined +in this module or by a push rule), the client should show that mention differently +from other mentions, such as by using a red background color to signify to the +user that they were mentioned. + +When clicked, the mention should navigate the user to the appropriate room, group, +or user information. + + +.. _`matrix.to URI`: ../appendices.html#matrix-to-navigation \ No newline at end of file diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 1972fa17..4a6dfb10 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -744,4 +744,4 @@ should send a "sync" command to instruct the client to get new events from the homeserver directly. -.. _`Push Gateway Specification`: ../push_gateway/unstable.html +.. _`Push Gateway Specification`: ../push_gateway/%PUSH_GATEWAY_RELEASE_LABEL%.html diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a6d8cbf7..faba7b62 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -38,11 +38,10 @@ single ``event_id``. Client behaviour ---------------- -In ``/initialSync``, receipts are listed in a separate top level ``receipts`` -key. In ``/sync``, receipts are contained in the ``ephemeral`` block for a -room. New receipts that come down the event streams are deltas which update -existing mappings. Clients should replace older receipt acknowledgements based -on ``user_id`` and ``receipt_type`` pairs. For example:: +In ``/sync``, receipts are listed under the ``ephemeral`` array of events +for a given room. New receipts that come down the event streams are deltas +which update existing mappings. Clients should replace older receipt acknowledgements +based on ``user_id`` and ``receipt_type`` pairs. For example:: Client receives m.receipt: user = @alice:example.com diff --git a/specification/modules/send_to_device.rst b/specification/modules/send_to_device.rst index 232becae..86288546 100644 --- a/specification/modules/send_to_device.rst +++ b/specification/modules/send_to_device.rst @@ -63,7 +63,7 @@ If the client sends messages to users on remote domains, those messages should be sent on to the remote servers via `federation`_. -.. _`federation`: ../server_server/latest.html#send-to-device-messages +.. _`federation`: ../server_server/latest.html#send-to-device-messaging .. TODO-spec: diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index e4a9d6ea..e4623887 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -21,13 +22,27 @@ the homeserver. This is managed by a distinct entity called the Push Gateway. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + +.. topic:: Version: %PUSH_GATEWAY_RELEASE_LABEL% +{{push_gateway_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/push_gateway.rst + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. +- `r0.1.0 `_ + Overview -------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index c225874e..01c3b65b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -64,53 +64,69 @@ request. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + +.. topic:: Version: %SERVER_RELEASE_LABEL% +{{server_server_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. -Server Discovery ----------------- - -Resolving Server Names -~~~~~~~~~~~~~~~~~~~~~~ +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/server_server.rst -Each matrix homeserver is identified by a server name consisting of a hostname -and an optional TLS port. -.. code:: +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - server_name = hostname [ ":" tls_port] - tls_port = *DIGIT +The following other versions are also available, in reverse chronological order: -.. ** +- `HEAD `_: Includes all changes since the latest versioned release. -If the port is present then the server is discovered by looking up an AAAA or -A record for the hostname and connecting to the specified TLS port. If the port -is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV -record for the hostname. If this record does not exist then the server is -discovered by looking up an AAAA or A record on the hostname and taking the -default fallback port number of 8448. -Homeservers may use SRV records to load balance requests between multiple TLS -endpoints or to failover to another endpoint if an endpoint fails. +Server discovery +---------------- -If the DNS name is a literal IP address, the port specified or the fallback -port should be used. +Resolving server names +~~~~~~~~~~~~~~~~~~~~~~ -When making requests to servers, use the DNS name of the target server in the -``Host`` header, regardless of the host given in the SRV record. For example, -if making a request to ``example.org``, and the SRV record resolves to ``matrix. -example.org``, the ``Host`` header in the request should be ``example.org``. The -port number for target server should not appear in the ``Host`` header. +Each matrix homeserver is identified by a server name consisting of a hostname +and an optional port, as described by the `grammar +<../appendices.html#server-name>`_. Server names should be resolved to an IP +address and port using the following process: + +* If the hostname is an IP literal, then that IP address should be used, + together with the given port number, or 8448 if no port is given. + +* Otherwise, if the port is present, then an IP address is discovered by + looking up an AAAA or A record for the hostname, and the specified port is + used. + +* If the hostname is not an IP literal and no port is given, the server is + discovered by first looking up a ``_matrix._tcp`` SRV record for the + hostname, which may give a hostname (to be looked up using AAAA or A queries) + and port. If the SRV record does not exist, then the server is discovered by + looking up an AAAA or A record on the hostname and taking the default + fallback port number of 8448. + + Homeservers may use SRV records to load balance requests between multiple TLS + endpoints or to failover to another endpoint if an endpoint fails. + +When making requests to servers, use the hostname of the target server in the +``Host`` header, regardless of any hostname given in the SRV record. For +example, if the server name is ``example.org``, and the SRV record resolves to +``matrix.example.org``, the ``Host`` header in the request should be +``example.org``. If an explicit port was given in the server name, it should be +included in the ``Host`` header; otherwise, no port number should be given in +the ``Host`` header. Server implementation ~~~~~~~~~~~~~~~~~~~~~~ {{version_ss_http_api}} -Retrieving Server Keys +Retrieving server keys ~~~~~~~~~~~~~~~~~~~~~~ .. NOTE:: @@ -359,111 +375,114 @@ The rules are as follows: 1. If type is ``m.room.create``: - a. Reject if it has any previous events - b. Reject if the domain of the ``room_id`` does not match the domain of the - ``sender``. - c. Reject if ``content.room_version`` key is an unrecognized version - d. Otherwise, allow. + a. If it has any previous events, reject. + b. If the domain of the ``room_id`` does not match the domain of the + ``sender``, reject. + c. If ``content.room_version`` is present and is not a recognised version, + reject. + d. If ``content`` has no ``creator`` field, reject. + e. Otherwise, allow. -#. Reject if event does not have a ``m.room.create`` in its ``auth_events`` +#. If event does not have a ``m.room.create`` in its ``auth_events``, reject. #. If type is ``m.room.aliases``: - a. Reject if event has no ``state_key`` - b. Allow if and only if sender's domain matches ``state_key`` + a. If event has no ``state_key``, reject. + b. If sender's domain doesn't matches ``state_key``, reject. + c. Otherwise, allow. #. If type is ``m.room.member``: - a. Reject if no ``state_key`` key or ``membership`` key in ``content``. + a. If no ``state_key`` key or ``membership`` key in ``content``, reject. - #. If ``membership`` is ``join``: + #. If ``membership`` is ``join``: - i. If the only previous event is an ``m.room.create`` - and the ``state_key`` is the creator, allow. + i. If the only previous event is an ``m.room.create`` + and the ``state_key`` is the creator, allow. - #. If the ``sender`` does not match ``state_key``, reject. + #. If the ``sender`` does not match ``state_key``, reject. - #. If the ``sender`` is banned, reject. + #. If the ``sender`` is banned, reject. - #. If the ``join_rule`` is ``invite`` then allow if membership state - is ``invite`` or ``join``. + #. If the ``join_rule`` is ``invite`` then allow if membership state + is ``invite`` or ``join``. - #. If the ``join_rule`` is ``public``, allow. + #. If the ``join_rule`` is ``public``, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``invite``: + #. If ``membership`` is ``invite``: - i. If ``content`` has ``third_party_invite`` key: + i. If ``content`` has ``third_party_invite`` key: - #. If *target user* is banned, reject. + #. If *target user* is banned, reject. - #. If ``content.third_party_invite`` does not have a - ``signed`` key, reject. + #. If ``content.third_party_invite`` does not have a + ``signed`` key, reject. - #. If ``signed`` does not have ``mxid`` and ``token`` keys, reject. + #. If ``signed`` does not have ``mxid`` and ``token`` keys, reject. - #. If ``mxid`` does not match ``state_key``, reject. + #. If ``mxid`` does not match ``state_key``, reject. - #. If there is no ``m.room.third_party_invite`` event in the - current room state with ``state_key`` matching ``token``, reject. + #. If there is no ``m.room.third_party_invite`` event in the + current room state with ``state_key`` matching ``token``, reject. - #. If ``sender`` does not match ``sender`` of the - ``m.room.third_party_invite``, reject. + #. If ``sender`` does not match ``sender`` of the + ``m.room.third_party_invite``, reject. - #. If any signature in ``signed`` matches any public key in the - ``m.room.third_party_invite`` event, allow. The public keys are - in ``content`` of ``m.room.third_party_invite`` as: + #. If any signature in ``signed`` matches any public key in the + ``m.room.third_party_invite`` event, allow. The public keys are + in ``content`` of ``m.room.third_party_invite`` as: - #. A single public key in the ``public_key`` field. - #. A list of public keys in the ``public_keys`` field. + #. A single public key in the ``public_key`` field. + #. A list of public keys in the ``public_keys`` field. - #. Otherwise, reject. + #. Otherwise, reject. - #. If the ``sender``'s current membership state is not ``join``, reject. + #. If the ``sender``'s current membership state is not ``join``, reject. - #. If *target user*'s current membership state is ``join`` or ``ban``, - reject. + #. If *target user*'s current membership state is ``join`` or ``ban``, + reject. - #. If the ``sender``'s power level is greater than or equal to the *invite - level*, allow. + #. If the ``sender``'s power level is greater than or equal to the *invite + level*, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``leave``: + #. If ``membership`` is ``leave``: - i. If the ``sender`` matches ``state_key``, allow if and only if that user's - current membership state is ``invite`` or ``join``. + i. If the ``sender`` matches ``state_key``, allow if and only if that user's + current membership state is ``invite`` or ``join``. - #. If the ``sender``'s current membership state is not ``join``, reject. + #. If the ``sender``'s current membership state is not ``join``, reject. - #. If the *target user*'s current membership state is ``ban``, and the - ``sender``'s power level is less than the *ban level*, reject. + #. If the *target user*'s current membership state is ``ban``, and the + ``sender``'s power level is less than the *ban level*, reject. - #. If the ``sender``'s power level is greater than or equal to the *kick - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. + #. If the ``sender``'s power level is greater than or equal to the *kick + level*, and the *target user*'s power level is less than the + ``sender``'s power level, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``ban``: + #. If ``membership`` is ``ban``: - i. If the ``sender``'s current membership state is not ``join``, reject. + i. If the ``sender``'s current membership state is not ``join``, reject. - #. If the ``sender``'s power level is greater than or equal to the *ban - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. + #. If the ``sender``'s power level is greater than or equal to the *ban + level*, and the *target user*'s power level is less than the + ``sender``'s power level, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. Otherwise, the membership is unknown. Reject. + #. Otherwise, the membership is unknown. Reject. #. If the ``sender``'s current membership state is not ``join``, reject. #. If type is ``m.room.third_party_invite``: - a. Allow if and only if ``sender``'s current power level is greater than - or equal to the *invite level*. + a. Allow if and only if ``sender``'s current power level is greater than + or equal to the *invite level*. #. If the event type's *required power level* is greater than the ``sender``'s power level, reject. @@ -473,39 +492,39 @@ The rules are as follows: #. If type is ``m.room.power_levels``: - a. If ``users`` key in ``content`` is not a dictionary with keys that are - valid user IDs with values that are integers (or a string that is an - integer), reject. + a. If ``users`` key in ``content`` is not a dictionary with keys that are + valid user IDs with values that are integers (or a string that is an + integer), reject. - #. If there is no previous ``m.room.power_levels`` event in the room, allow. + #. If there is no previous ``m.room.power_levels`` event in the room, allow. - #. For each of the keys ``users_default``, ``events_default``, - ``state_default``, ``ban``, ``redact``, ``kick``, ``invite``, as well as - each entry being changed under the ``events`` or ``users`` keys: + #. For each of the keys ``users_default``, ``events_default``, + ``state_default``, ``ban``, ``redact``, ``kick``, ``invite``, as well as + each entry being changed under the ``events`` or ``users`` keys: - i. If the current value is higher than the ``sender``'s current power level, - reject. + i. If the current value is higher than the ``sender``'s current power level, + reject. - #. If the new value is higher than the ``sender``'s current power level, - reject. + #. If the new value is higher than the ``sender``'s current power level, + reject. - #. For each entry being changed under the ``users`` key, other than the - ``sender``'s own entry: + #. For each entry being changed under the ``users`` key, other than the + ``sender``'s own entry: - i. If the current value is equal to the ``sender``'s current power level, - reject. + i. If the current value is equal to the ``sender``'s current power level, + reject. - #. Otherwise, allow. + #. Otherwise, allow. #. If type is ``m.room.redaction``: - a. If the ``sender``'s power level is greater than or equal to the *redact - level*, allow. + a. If the ``sender``'s power level is greater than or equal to the *redact + level*, allow. - #. If the domain of the ``event_id`` of the event being redacted is the same - as the domain of the ``event_id`` of the ``m.room.redaction``, allow. + #. If the domain of the ``event_id`` of the event being redacted is the same + as the domain of the ``event_id`` of the ``m.room.redaction``, allow. - #. Otherwise, reject. + #. Otherwise, reject. #. Otherwise, allow. @@ -858,7 +877,7 @@ identifier. Public Room Directory --------------------- -To compliment the `Client-Server API`_'s room directory, homeservers need a +To complement the `Client-Server API`_'s room directory, homeservers need a way to query the public rooms for another server. This can be done by making a request to the ``/publicRooms`` endpoint for the server the room directory should be retrieved for. @@ -944,6 +963,19 @@ nothing else. {{openid_ss_http_api}} + +End-to-End Encryption +--------------------- + +This section complements the `End-to-End Encryption module`_ of the Client-Server +API. For detailed information about end-to-end encryption, please see that module. + +The APIs defined here are designed to be able to proxy much of the client's request +through to federation, and have the response also be proxied through to the client. + +{{user_keys_ss_http_api}} + + Send-to-device messaging ------------------------ @@ -1015,149 +1047,127 @@ Signing Events Signing events is complicated by the fact that servers can choose to redact non-essential parts of an event. -Before signing the event, the ``unsigned`` and ``signature`` members are -removed, it is encoded as `Canonical JSON`_, and then hashed using SHA-256. The -resulting hash is then stored in the event JSON in a ``hash`` object under a -``sha256`` key. +Adding hashes and signatures to outgoing events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code:: python +Before signing the event, the *content hash* of the event is calculated as +described below. The hash is encoded using `Unpadded Base64`_ and stored in the +event object, in a ``hashes`` object, under a ``sha256`` key. - def hash_event(event_json_object): +The event object is then *redacted*, following the `redaction +algorithm`_. Finally it is signed as described in `Signing JSON`_, using the +server's signing key (see also `Retrieving server keys`_). - # Keys under "unsigned" can be modified by other servers. - # They are useful for conveying information like the age of an - # event that will change in transit. - # Since they can be modifed we need to exclude them from the hash. - unsigned = event_json_object.pop("unsigned", None) +The signature is then copied back to the original event object. - # Signatures will depend on the current value of the "hashes" key. - # We cannot add new hashes without invalidating existing signatures. - signatures = event_json_object.pop("signatures", None) +See `Persistent Data Unit schema`_ for an example of a signed event. - # The "hashes" key might contain multiple algorithms if we decide to - # migrate away from SHA-2. We don't want to include an existing hash - # output in our hash so we exclude the "hashes" dict from the hash. - hashes = event_json_object.pop("hashes", {}) - - # Encode the JSON using a canonical encoding so that we get the same - # bytes on every server for the same JSON object. - event_json_bytes = encode_canonical_json(event_json_bytes) - # Add the base64 encoded bytes of the hash to the "hashes" dict. - hashes["sha256"] = encode_base64(sha256(event_json_bytes).digest()) +Validating hashes and signatures on received events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When a server receives an event over federation from another server, the +receiving server should check the hashes and signatures on that event. - # Add the "hashes" dict back the event JSON under a "hashes" key. - event_json_object["hashes"] = hashes - if unsigned is not None: - event_json_object["unsigned"] = unsigned - return event_json_object +First the signature is checked. The event is redacted following the `redaction +algorithm`_, and the resultant object is checked for a signature from the +originating server, following the algorithm described in `Checking for a signature`_. +Note that this step should succeed whether we have been sent the full event or +a redacted copy. -The event is then stripped of all non-essential keys both at the top level and -within the ``content`` object. Any top-level keys not in the following list -MUST be removed: +If the signature is found to be valid, the expected content hash is calculated +as described below. The content hash in the ``hashes`` property of the received +event is base64-decoded, and the two are compared for equality. -.. code:: - - auth_events - depth - event_id - hashes - membership - origin - origin_server_ts - prev_events - prev_state - room_id - sender - signatures - state_key - type - -A new ``content`` object is constructed for the resulting event that contains -only the essential keys of the original ``content`` object. If the original -event lacked a ``content`` object at all, a new empty JSON object is created -for it. - -The keys that are considered essential for the ``content`` object depend on the -the ``type`` of the event. These are: - -.. code:: +If the hash check fails, then it is assumed that this is because we have only +been given a redacted version of the event. To enforce this, the receiving +server should use the redacted copy it calculated rather than the full copy it +received. - type is "m.room.aliases": - aliases +Calculating the content hash for an event +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - type is "m.room.create": - creator +The *content hash* of an event covers the complete event including the +*unredacted* contents. It is calculated as follows. - type is "m.room.history_visibility": - history_visibility +First, any existing ``unsigned``, ``signature``, and ``hashes`` members are +removed. The resulting object is then encoded as `Canonical JSON`_, and the +JSON is hashed using SHA-256. - type is "m.room.join_rules": - join_rule - type is "m.room.member": - membership - - type is "m.room.power_levels": - ban - events - events_default - kick - redact - state_default - users - users_default - -The resulting stripped object with the new ``content`` object and the original -``hashes`` key is then signed using the JSON signing algorithm outlined below: +Example code +~~~~~~~~~~~~ .. code:: python - def sign_event(event_json_object, name, key): - - # Make sure the event has a "hashes" key. - if "hashes" not in event_json_object: - event_json_object = hash_event(event_json_object) + def hash_and_sign_event(event_object, signing_key, signing_name): + # First we need to hash the event object. + content_hash = compute_content_hash(event_object) + event_object["hashes"] = {"sha256": encode_unpadded_base64(content_hash)} # Strip all the keys that would be removed if the event was redacted. # The hashes are not stripped and cover all the keys in the event. # This means that we can tell if any of the non-essential keys are # modified or removed. - stripped_json_object = strip_non_essential_keys(event_json_object) + stripped_object = strip_non_essential_keys(event_object) # Sign the stripped JSON object. The signature only covers the # essential keys and the hashes. This means that we can check the # signature even if the event is redacted. - signed_json_object = sign_json(stripped_json_object) + signed_object = sign_json(stripped_object, signing_key, signing_name) # Copy the signatures from the stripped event to the original event. - event_json_object["signatures"] = signed_json_oject["signatures"] - return event_json_object + event_object["signatures"] = signed_object["signatures"] + + def compute_content_hash(event_object): + # take a copy of the event before we remove any keys. + event_object = dict(event_object) + + # Keys under "unsigned" can be modified by other servers. + # They are useful for conveying information like the age of an + # event that will change in transit. + # Since they can be modifed we need to exclude them from the hash. + event_object.pop("unsigned", None) -Servers can then transmit the entire event or the event with the non-essential -keys removed. If the entire event is present, receiving servers can then check -the event by computing the SHA-256 of the event, excluding the ``hash`` object. -If the keys have been redacted, then the ``hash`` object is included when -calculating the SHA-256 hash instead. + # Signatures will depend on the current value of the "hashes" key. + # We cannot add new hashes without invalidating existing signatures. + event_object.pop("signatures", None) -New hash functions can be introduced by adding additional keys to the ``hash`` -object. Since the ``hash`` object cannot be redacted a server shouldn't allow -too many hashes to be listed, otherwise a server might embed illict data within -the ``hash`` object. For similar reasons a server shouldn't allow hash values -that are too long. + # The "hashes" key might contain multiple algorithms if we decide to + # migrate away from SHA-2. We don't want to include an existing hash + # output in our hash so we exclude the "hashes" dict from the hash. + event_object.pop("hashes", None) + + # Encode the JSON using a canonical encoding so that we get the same + # bytes on every server for the same JSON object. + event_json_bytes = encode_canonical_json(event_object) + + return hashlib.sha256(event_json_bytes) .. TODO - [[TODO(markjh): We might want to specify a maximum number of keys for the - ``hash`` and we might want to specify the maximum output size of a hash]] - [[TODO(markjh) We might want to allow the server to omit the output of well - known hash functions like SHA-256 when none of the keys have been redacted]] + + [[TODO(markjh): Since the ``hash`` object cannot be redacted a server + shouldn't allow too many hashes to be listed, otherwise a server might embed + illict data within the ``hash`` object. + + We might want to specify a maximum number of keys for the + ``hash`` and we might want to specify the maximum output size of a hash]] + + [[TODO(markjh) We might want to allow the server to omit the output of well + known hash functions like SHA-256 when none of the keys have been redacted]] + .. |/query/directory| replace:: ``/query/directory`` .. _/query/directory: #get-matrix-federation-v1-query-directory -.. _`Invitation storage`: ../identity_service/unstable.html#invitation-storage -.. _`Identity Service API`: ../identity_service/unstable.html -.. _`Client-Server API`: ../client_server/unstable.html +.. _`Invitation storage`: ../identity_service/%IDENTITY_RELEASE_LABEL%.html#invitation-storage +.. _`Identity Service API`: ../identity_service/%IDENTITY_RELEASE_LABEL%.html +.. _`Client-Server API`: ../client_server/%CLIENT_RELEASE_LABEL%.html .. _`Inviting to a room`: #inviting-to-a-room .. _`Canonical JSON`: ../appendices.html#canonical-json .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 +.. _`Server ACLs`: ../client_server/%CLIENT_RELEASE_LABEL%.html#module-server-acls +.. _`redaction algorithm`: ../client_server/%CLIENT_RELEASE_LABEL%.html#redactions +.. _`Signing JSON`: ../appendices.html#signing-json +.. _`Checking for a signature`: ../appendices.html#checking-for-a-signature +.. _`Device Management module`: ../client_server/%CLIENT_RELEASE_LABEL%.html#device-management +.. _`End-to-End Encryption module`: ../client_server/%CLIENT_RELEASE_LABEL%.html#end-to-end-encryption diff --git a/specification/targets.yaml b/specification/targets.yaml index acf4b6ac..56e9ec34 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -13,7 +13,7 @@ targets: application_service: files: - application_service_api.rst - version_label: unstable + version_label: "%APPSERVICE_RELEASE_LABEL%" server_server: files: - server_server_api.rst @@ -25,7 +25,7 @@ targets: push_gateway: files: - push_gateway.rst - version_label: unstable + version_label: "%PUSH_GATEWAY_RELEASE_LABEL%" appendices: files: - appendices.rst @@ -68,6 +68,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/third_party_networks.rst - modules/openid.rst - modules/server_acls.rst + - modules/mentions.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"]