From 1e656d836ead371bb395a768e35173a6cf41f5ea Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:35:44 +0100 Subject: [PATCH 001/166] spec notifications key on power level event and provide @room in example --- event-schemas/examples/m.room.power_levels | 5 ++++- event-schemas/schema/m.room.power_levels | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 0c8f8bc53..2b0aaca55 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -14,7 +14,10 @@ "users": { "@example:localhost": 100 }, - "users_default": 0 + "users_default": 0, + "notifications": { + "room": 20 + } }, "state_key": "", "origin_server_ts": 1431961217939, diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index 13a44c709..f348b52d0 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -85,6 +85,14 @@ properties: ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if unspecified. type: number + notifications: + additionalProperties: + type: number + description: |- + The power level requirements for specific notification types. + This is a mapping from ``key`` to power level for that notifications key. + title: Notification power level requirements + type: object type: object state_key: description: A zero-length string. From 60ae73b179c53213ce952251c7e9f4d14c323fef Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:37:23 +0100 Subject: [PATCH 002/166] specify default --- event-schemas/schema/m.room.power_levels | 1 + 1 file changed, 1 insertion(+) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index f348b52d0..a00746f00 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -91,6 +91,7 @@ properties: description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. + Defaults to 50 for unspecified keys. title: Notification power level requirements type: object type: object From c305317fa57b918d68731f69a0ab21367debde0b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:41:24 +0100 Subject: [PATCH 003/166] explicitly specify @room --- event-schemas/schema/m.room.power_levels | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index a00746f00..25d33f085 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -86,13 +86,16 @@ properties: unspecified. type: number notifications: + properties: + room: + type: number + description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. additionalProperties: type: number description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. - Defaults to 50 for unspecified keys. - title: Notification power level requirements + title: Notifications type: object type: object state_key: From 9ca62edda0a664f65ac7ff31b659a77efce7edb1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 25 May 2018 19:39:54 +0100 Subject: [PATCH 004/166] Document new application service registration file options. --- .../application_service.yaml | 2 +- specification/application_service_api.rst | 89 +++++++++++++++---- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index c39ce198f..a63774a24 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -43,7 +43,7 @@ paths: x-example: "35" - in: body name: body - description: A list of events + description: A list of events. schema: type: object example: { diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..6eb8e87e2 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -83,34 +83,93 @@ regular expressions and look like: users: - exclusive: true - regex: @irc.freenode.net_.* + regex: "@irc.freenode.net_.*" + group_id: "+irc:matrix.org" + +Application Services may define the following namespaces (with none being explicitly required): + ++------------------+-----------------------------------------------------------+ +| Name | Description | ++==================+===========================================================+ +| users | Events which are sent from certain users. | ++------------------+-----------------------------------------------------------+ +| aliases | Events which are sent in rooms with certain room aliases. | ++------------------+-----------------------------------------------------------+ +| rooms | Events which are sent in rooms with certain room IDs. | ++------------------+-----------------------------------------------------------+ + +Each individual namespace MUST declare the following fields: + ++------------------+-----------------------------------------------------------------------------------------------------------------------------------+ +| Name | Description | ++==================+===================================================================================================================================+ +| exclusive | **Required** A true or false value stating whether this Application Service has exclusive access to events within this namespace. | ++------------------+-----------------------------------------------------------------------------------------------------------------------------------+ +| regex | **Required** A regular expression defining which values this namespace includes. | ++------------------+-----------------------------------------------------------------------------------------------------------------------------------+ + +An optional ``group_id`` field may be added to the users namespace: + ++------------------+-----------------------------------------------------------+ +| Name | Description | ++==================+===========================================================+ +| group_id | All matching users will be considered part of this group. | ++------------------+-----------------------------------------------------------+ + +.. WARNING:: + + Users that are matched by ``group_id`` should not be publically listed by + Homeservers. The intention is to differentiate users, perhaps with a flair, + rather than having a list of people to spam. The registration is represented by a series of key-value pairs, which this -specification will present as YAML. An example HS configuration required to pass -traffic to the AS is: +specification will present as YAML. See below for the possible options along +with their explanation: + ++------------------+----------------------------------------------------------------------------------------------------------+ +| Name | Description | ++==================+==========================================================================================================+ +| id | **Required.** A unique, user-defined ID of the Application Service which will never change. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| url | **Required.** The base URL for the Application Service. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| as_token | **Required.** A unique token for Application Services to use to authenticate requests to Homeservers. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| hs_token | **Required.** A unique token for Homeservers to use to authenticate requests to Application Services. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| sender_localpart | **Required.** The localpart of the user associated with the Application Service. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| namespaces | **Required.** A list of "users", "aliases" and "rooms" namespaces that the Application Service controls. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| rate_limited | Whether requests from masqueraded users are rate-limited. The sender is excluded. | ++------------------+----------------------------------------------------------------------------------------------------------+ +| protocols | The external protocols which the Application Service provides (e.g. IRC). | ++------------------+----------------------------------------------------------------------------------------------------------+ + +An example registration file for an IRC-bridging Application Service is below: .. code-block:: yaml - id: - url: - as_token: - hs_token: - sender_localpart: + id: "IRC Bridge" + url: "http://127.0.0.1:1234" + as_token: "30c05ae90a248a4188e620216fa72e349803310ec83e2a77b34fe90be6081f46" + hs_token: "312df522183efd404ec1cd22d2ffa4bbc76a8c1ccf541dd692eef281356bb74e" + sender_localpart: "_irc_bot" # Will result in @_irc_bot:domain.com namespaces: - users: # Namespaces of users which should be delegated to the AS - - exclusive: - regex: - - ... - aliases: [] # Namespaces of room aliases which should be delegated to the AS - rooms: [] # Namespaces of room ids which should be delegated to the AS + users: + - exclusive: true + regex: "@irc_bridge_.*" + aliases: + - exclusive: false + regex: "#irc_bridge_.*" + rooms: [] .. WARNING:: If the homeserver in question has multiple application services, each ``as_token`` and ``id`` MUST be unique per application service as these are used to identify the application service. The homeserver MUST enforce this. - Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From d6fb5afd16ad1682d90c9dd3bd505e9baf60e513 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 28 May 2018 23:30:40 +0100 Subject: [PATCH 005/166] Clarified group_id group visibility, url possibilities, regex starters Changed Application Service capatalization to be consistent with the rest of the document. --- specification/application_service_api.rst | 68 ++++++++++++----------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 6eb8e87e2..31983a0cc 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -83,10 +83,10 @@ regular expressions and look like: users: - exclusive: true - regex: "@irc.freenode.net_.*" + regex: "@_irc.freenode.net_.*" group_id: "+irc:matrix.org" -Application Services may define the following namespaces (with none being explicitly required): +Application services may define the following namespaces (with none being explicitly required): +------------------+-----------------------------------------------------------+ | Name | Description | @@ -103,22 +103,24 @@ Each individual namespace MUST declare the following fields: +------------------+-----------------------------------------------------------------------------------------------------------------------------------+ | Name | Description | +==================+===================================================================================================================================+ -| exclusive | **Required** A true or false value stating whether this Application Service has exclusive access to events within this namespace. | +| exclusive | **Required** A true or false value stating whether this application service has exclusive access to events within this namespace. | +------------------+-----------------------------------------------------------------------------------------------------------------------------------+ | regex | **Required** A regular expression defining which values this namespace includes. | +------------------+-----------------------------------------------------------------------------------------------------------------------------------+ -An optional ``group_id`` field may be added to the users namespace: +An application service's users and regex field MUST begin with an underscore (``_``), in +order to provide a visually clear distinction between AS users and regular +users. An optional ``group_id`` field may be added to the ``users`` namespace: -+------------------+-----------------------------------------------------------+ -| Name | Description | -+==================+===========================================================+ -| group_id | All matching users will be considered part of this group. | -+------------------+-----------------------------------------------------------+ ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Name | Description | ++==================+============================================================================================================================================================================================================================================================================+ +| group_id | An existing group that all matching user IDs will be considered a part of. Users who are joined to this group through an application service are not to be listed when querying for the group's members, however the group should be listed when querying a user's groups. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. WARNING:: - Users that are matched by ``group_id`` should not be publically listed by + Users that are matched by ``group_id`` should not be publicly listed by Homeservers. The intention is to differentiate users, perhaps with a flair, rather than having a list of people to spam. @@ -127,27 +129,27 @@ The registration is represented by a series of key-value pairs, which this specification will present as YAML. See below for the possible options along with their explanation: -+------------------+----------------------------------------------------------------------------------------------------------+ -| Name | Description | -+==================+==========================================================================================================+ -| id | **Required.** A unique, user-defined ID of the Application Service which will never change. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| url | **Required.** The base URL for the Application Service. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| as_token | **Required.** A unique token for Application Services to use to authenticate requests to Homeservers. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| hs_token | **Required.** A unique token for Homeservers to use to authenticate requests to Application Services. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| sender_localpart | **Required.** The localpart of the user associated with the Application Service. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| namespaces | **Required.** A list of "users", "aliases" and "rooms" namespaces that the Application Service controls. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| rate_limited | Whether requests from masqueraded users are rate-limited. The sender is excluded. | -+------------------+----------------------------------------------------------------------------------------------------------+ -| protocols | The external protocols which the Application Service provides (e.g. IRC). | -+------------------+----------------------------------------------------------------------------------------------------------+ - -An example registration file for an IRC-bridging Application Service is below: ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| Name | Description | ++==================+====================================================================================================================================================+ +| id | **Required.** A unique, user-defined ID of the application service which will never change. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| url | **Required.** The URL for the application service. May include a path after the domain name. Optionally set to ``null`` if no traffic is required. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| as_token | **Required.** A unique token for application services to use to authenticate requests to Homeservers. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| hs_token | **Required.** A unique token for Homeservers to use to authenticate requests to application services. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| sender_localpart | **Required.** The localpart of the user associated with the application service. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| namespaces | **Required.** A list of ``users``, ``aliases`` and ``rooms`` namespaces that the application service controls. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| rate_limited | Whether requests from masqueraded users are rate-limited. The sender is excluded. | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| protocols | The external protocols which the application service provides (e.g. IRC). | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ + +An example registration file for an IRC-bridging application service is below: .. code-block:: yaml @@ -159,10 +161,10 @@ An example registration file for an IRC-bridging Application Service is below: namespaces: users: - exclusive: true - regex: "@irc_bridge_.*" + regex: "@_irc_bridge_.*" aliases: - exclusive: false - regex: "#irc_bridge_.*" + regex: "#_irc_bridge_.*" rooms: [] .. WARNING:: From 0dd330962d7bf54e65737d5ca0e5f9a074c237e3 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 3 Jul 2018 14:14:26 -0400 Subject: [PATCH 006/166] initial draft of .well-known discovery --- .../definitions/wellknown/homeserver.yaml | 23 ++++++ .../definitions/wellknown/homeserver.yaml~ | 23 ++++++ .../wellknown/identity_server.yaml | 23 ++++++ api/client-server/wellknown.yaml | 63 ++++++++++++++++ specification/client_server_api.rst | 72 +++++++++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 api/client-server/definitions/wellknown/homeserver.yaml create mode 100644 api/client-server/definitions/wellknown/homeserver.yaml~ create mode 100644 api/client-server/definitions/wellknown/identity_server.yaml create mode 100644 api/client-server/wellknown.yaml diff --git a/api/client-server/definitions/wellknown/homeserver.yaml b/api/client-server/definitions/wellknown/homeserver.yaml new file mode 100644 index 000000000..7efba8169 --- /dev/null +++ b/api/client-server/definitions/wellknown/homeserver.yaml @@ -0,0 +1,23 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: Homeserver Information +description: |- + Used by clients to discover homeserver information. +type: object +properties: + base_url: + type: string + description: The base URL for the homeserver for client-server connections. +required: + - base_url diff --git a/api/client-server/definitions/wellknown/homeserver.yaml~ b/api/client-server/definitions/wellknown/homeserver.yaml~ new file mode 100644 index 000000000..e42dfbf3b --- /dev/null +++ b/api/client-server/definitions/wellknown/homeserver.yaml~ @@ -0,0 +1,23 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: Authentication Data +description: |- + Used by clients to submit authentication information to the interactive-authentication API +type: object +properties: + base_url: + type: string + description: The base URL for the homeserver for client-server connections. +required: + - base_url diff --git a/api/client-server/definitions/wellknown/identity_server.yaml b/api/client-server/definitions/wellknown/identity_server.yaml new file mode 100644 index 000000000..eb0e0bafc --- /dev/null +++ b/api/client-server/definitions/wellknown/identity_server.yaml @@ -0,0 +1,23 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: Identity Server Information +description: |- + Used by clients to discover identity server information. +type: object +properties: + base_url: + type: string + description: The base URL for the identity server for client-server connections. +required: + - base_url diff --git a/api/client-server/wellknown.yaml b/api/client-server/wellknown.yaml new file mode 100644 index 000000000..44d9ef731 --- /dev/null +++ b/api/client-server/wellknown.yaml @@ -0,0 +1,63 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server server discovery API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /.well-known +produces: + - application/json +paths: + "/matrix/client": + get: + summary: Gets Matrix server discovery information about the domain. + description: |- + Gets discovery information about the domain. The file may include + additional keys, which SHOULD follow the Java package naming convention, + e.g. ``com.example.myapp.property``. This ensures property names are + suitably namespaced for each application and reduces the risk of + clashes. + + **FIXME:** do we need to add a note that this endpoint is not + necessarily handled by the homeserver, but by another webserver? Or + does the context make this clear enough? + operationId: getWellknown + responses: + 200: + description: Server discovery information + examples: + application/json: { + "m.homeserver": { + "base_url": "https://matrix.example.com" + }, + "m.identity_server": { + "base_url": "https://identity.example.com" + } + } + schema: + type: object + properties: + m.homeserver: + description: Information about the homeserver to connect to. + "$ref": "definitions/wellknown/homeserver.yaml" + m.identity_server: + description: Information about the identity server to connect to. + "$ref": "definitions/wellknown/identity_server.yaml" + 404: + description: No server discovery information available + tags: + - Server administration diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index dec3a4f44..e0befddf4 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -164,6 +164,78 @@ recommended. {{versions_cs_http_api}} +Server Discovery +~~~~~~~~~~~~~~~~ + +In order to allow users to connect to a Matrix server without needing to +explicitly specify the homeserver's URL or other parameters, clients may use an +auto-discovery mechanism to determine the server's URL based on a user's +Matrix ID. Auto-discovery should only be done at login time, with the +discovered values retained for the duration of the user's session. + +In this section, the following terms are used with specific meanings: + +``PROMPT`` + Retrieve the specific piece of information from the user in a way which + fits within the existing client UX, if the client is inclined to do so. + Failure can take place instead if no good UX is possible at this point. + +``IGNORE`` + Stop the current auto-discovery mechanism. If no more auto-discovery + mechanisms are available, then the client may use other methods of + determining the required parameters, such as prompting the user, or using + default values. + +``FAIL_PROMPT`` + Inform the user that auto-discovery failed due to invalid/empty data and + ``PROMPT`` for the parameter. + +``FAIL_ERROR`` + Inform the user that auto-discovery did not return any usable URLs. Do not + continue further with the current login process. At this point, valid data + was obtained, but no homeserver is available to serve the client. No further + guess should be attempted and the user should make a conscientious decision + what to do next. + +Well-known URI +++++++++++++++ + +The ``.well-known`` method uses a JSON file at a predetermined location to +specify parameter values. The flow for this method is as follows: + +1. Extract the server name from the user's Matrix ID by splitting the Matrix ID + at the first colon. +2. Extract the DNS name from the server name. +3. Make a GET request to ``https://dns_name/.well-known/matrix/client``. + + a. If the returned status code is 404, then ``IGNORE``. + b. If the returned status code is not 200, or the response body is empty, + then ``FAIL_PROMPT``. + c. Parse the response body as a JSON object + + i. If the content cannot be parsed, then ``FAIL_PROMPT``. + + d. Extract the ``base_url`` value from the ``m.homeserver`` property. This + value is to be used as the base URL of the homeserver. + + i. If this value is not provided, then ``FAIL_PROMPT``. + + e. Validate the homeserver base URL: + + i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. + ii. Clients should validate that the URL points to a valid homeserver + before accepting it. Currently, the suggested way of validating is + to connect to the ``/_matrix/client/versions`` endpoint, and to parse + and validate the data. If any step in the validation fails, then + ``FAIL_ERROR``. + + f. If the ``m.identity_server`` property is present, extract the + ``base_url`` value for use as the base URL of the identity server. This + value can be validated as in the step above, but using + ``/_matrix/identity/api/v1``. + +{{wellknown_cs_http_api}} + Client Authentication --------------------- From 5bc29eb11c5c07c47e6fb4afe0c7c1caba12cbf2 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 4 Jul 2018 17:01:35 -0400 Subject: [PATCH 007/166] remove accidentally-committed backup file --- .../definitions/wellknown/homeserver.yaml~ | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 api/client-server/definitions/wellknown/homeserver.yaml~ diff --git a/api/client-server/definitions/wellknown/homeserver.yaml~ b/api/client-server/definitions/wellknown/homeserver.yaml~ deleted file mode 100644 index e42dfbf3b..000000000 --- a/api/client-server/definitions/wellknown/homeserver.yaml~ +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -title: Authentication Data -description: |- - Used by clients to submit authentication information to the interactive-authentication API -type: object -properties: - base_url: - type: string - description: The base URL for the homeserver for client-server connections. -required: - - base_url From ce1e2c0904a793b25527e88e060e0d0f6809cc1f Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 4 Jul 2018 17:58:37 -0400 Subject: [PATCH 008/166] incorporate feedback from reviewers --- api/client-server/wellknown.yaml | 13 +++++++----- changelogs/client_server.rst | 4 ++++ specification/client_server_api.rst | 31 ++++++++++++++++------------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/api/client-server/wellknown.yaml b/api/client-server/wellknown.yaml index 44d9ef731..8d19f38a8 100644 --- a/api/client-server/wellknown.yaml +++ b/api/client-server/wellknown.yaml @@ -27,14 +27,13 @@ paths: summary: Gets Matrix server discovery information about the domain. description: |- Gets discovery information about the domain. The file may include - additional keys, which SHOULD follow the Java package naming convention, + additional keys, which MUST follow the Java package naming convention, e.g. ``com.example.myapp.property``. This ensures property names are suitably namespaced for each application and reduces the risk of clashes. - **FIXME:** do we need to add a note that this endpoint is not - necessarily handled by the homeserver, but by another webserver? Or - does the context make this clear enough? + Note that this endpoint is not necessarily handled by the homeserver, + but by another webserver, to be used for discovering the homeserver URL. operationId: getWellknown responses: 200: @@ -55,8 +54,12 @@ paths: description: Information about the homeserver to connect to. "$ref": "definitions/wellknown/homeserver.yaml" m.identity_server: - description: Information about the identity server to connect to. + description: Optional. Information about the identity server to connect to. "$ref": "definitions/wellknown/identity_server.yaml" + additionalProperties: + description: Application-dependent keys using Java package naming convention. + required: + - m.homeserver 404: description: No server discovery information available tags: diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..c6eb740ad 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -16,6 +16,10 @@ Unreleased changes - Add sticker message event definition. (`#1158 `_). + - Server discovery: + - Add ``.well-known`` discovery method + (`#1359 `_). + - Spec clarifications: - Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index e0befddf4..fbeb87cbc 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -168,17 +168,17 @@ Server Discovery ~~~~~~~~~~~~~~~~ In order to allow users to connect to a Matrix server without needing to -explicitly specify the homeserver's URL or other parameters, clients may use an -auto-discovery mechanism to determine the server's URL based on a user's -Matrix ID. Auto-discovery should only be done at login time, with the -discovered values retained for the duration of the user's session. +explicitly specify the homeserver's URL or other parameters, clients SHOULD use +an auto-discovery mechanism to determine the server's URL based on a user's +Matrix ID. Auto-discovery should only be done at login time. In this section, the following terms are used with specific meanings: ``PROMPT`` Retrieve the specific piece of information from the user in a way which - fits within the existing client UX, if the client is inclined to do so. - Failure can take place instead if no good UX is possible at this point. + fits within the existing client user experience, if the client is inclined to + do so. Failure can take place instead if no good user experience for this is + possible at this point. ``IGNORE`` Stop the current auto-discovery mechanism. If no more auto-discovery @@ -223,16 +223,19 @@ specify parameter values. The flow for this method is as follows: e. Validate the homeserver base URL: i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. - ii. Clients should validate that the URL points to a valid homeserver - before accepting it. Currently, the suggested way of validating is - to connect to the ``/_matrix/client/versions`` endpoint, and to parse - and validate the data. If any step in the validation fails, then - ``FAIL_ERROR``. + ii. Clients SHOULD validate that the URL points to a valid homeserver + before accepting it by connecting to the ``/_matrix/client/versions`` + endpoint, and parsing and validating the data. If any step in the + validation fails, then ``FAIL_ERROR``. Validation is done as a simple + check against configuration errors, before sending sensitive + information such as a user's password to the server. f. If the ``m.identity_server`` property is present, extract the - ``base_url`` value for use as the base URL of the identity server. This - value can be validated as in the step above, but using - ``/_matrix/identity/api/v1``. + ``base_url`` value for use as the base URL of the identity server. + Validation for this URL is done as in the step above, but using + ``/_matrix/identity/api/v1`` as the endpoint to connect to. If the + ``m.identity_server`` property is present, but does not have a + ``base_url`` value, then ``FAIL_ERROR``. {{wellknown_cs_http_api}} From 8ffac01efec2295e584e22bdb72753ef59d34fbd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 20:57:00 -0600 Subject: [PATCH 009/166] Document OpenID in the client-server API Part of https://github.com/matrix-org/matrix-doc/issues/857 Reference: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/rest/client/v2_alpha/openid.py#L31-L58 --- api/client-server/openid.yaml | 103 +++++++++++++++++++++++++++++++ specification/modules/openid.rst | 24 +++++++ specification/targets.yaml | 1 + 3 files changed, 128 insertions(+) create mode 100644 api/client-server/openid.yaml create mode 100644 specification/modules/openid.rst diff --git a/api/client-server/openid.yaml b/api/client-server/openid.yaml new file mode 100644 index 000000000..4b89232ea --- /dev/null +++ b/api/client-server/openid.yaml @@ -0,0 +1,103 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server OpenID API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/user/{userId}/openid/request_token": + post: + summary: Get an OpenID token object to verify the requester's identity. + description: |- + Gets an OpenID token object that the requester may supply to another + service to verify their identity in Matrix. The generated token is only + valid for exchanging for user information from the federation API for + OpenID. + + The access token generated is only valid for the OpenID API. It cannot + be used to request another OpenID access token or call ``/sync``, for + example. + operationId: requestOpenIdToken + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + description: |- + The user to request and OpenID token for. Should be the user who + is authenticated for the request. + required: true + x-example: "@alice:example.com" + - in: body + name: body + description: An empty object. Reserved for future expansion. + required: true + schema: + type: object + example: {} + responses: + 200: + description: |- + OpenID token information. This response is nearly compatible with the + response documented in the `OpenID 1.0 Specification `_ + with the only difference being the lack of an ``id_token``. Instead, + the Matrix homeserver's name is provided. + examples: + application/json: { + "access_token": "SomeT0kenHere", + "token_type": "Bearer", + "matrix_server_name": "example.com", + "expires_in": 3600, + } + schema: + type: object + properties: + access_token: + type: string + description: |- + An access token the consumer may use to verify the identity of + the person who generated the token. This is given to the federation + API ``GET /openid/userinfo``. + token_type: + type: string + description: The string ``Bearer``. + matrix_server_name: + type: string + description: |- + The homeserver domain the consumer should use when attempting to + verify the user's identity. + expires_in: + type: int + description: |- + The number of seconds before this token expires and a new one must + be generated. + required: ['access_token', 'token_type', 'matrix_server_name', 'expires_in'] + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/errors/rate_limited.yaml" + tags: + - OpenID diff --git a/specification/modules/openid.rst b/specification/modules/openid.rst new file mode 100644 index 000000000..63406719c --- /dev/null +++ b/specification/modules/openid.rst @@ -0,0 +1,24 @@ +.. Copyright 2018 New Vector Ltd. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +OpenID +====== + +.. _module:openid: + +This module allows users to verify their identity with a third party service. The +third party service does need to be matrix-aware in that it will need to know to +resolve matrix homeservers to exchange the user's token for identity information. + +{{openid_cs_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 53957e0af..5480bbc5a 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -66,6 +66,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/stickers.rst - modules/report_content.rst - modules/third_party_networks.rst + - modules/openid.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From dcae88c29011e8e1a4349137b42a6b63181b5385 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 22:13:21 -0600 Subject: [PATCH 010/166] Document OpenID in the server-server API Part of https://github.com/matrix-org/matrix-doc/issues/857 Reference: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/transport/server.py#L543-L557 --- api/server-server/openid.yaml | 63 +++++++++++++++++++++++++++++ specification/server_server_api.rst | 12 ++++++ 2 files changed, 75 insertions(+) create mode 100644 api/server-server/openid.yaml diff --git a/api/server-server/openid.yaml b/api/server-server/openid.yaml new file mode 100644 index 000000000..0eac48c84 --- /dev/null +++ b/api/server-server/openid.yaml @@ -0,0 +1,63 @@ +# Copyright 2017 Kamax.io +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation OpenID API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/openid/userinfo": + get: + summary: Exchange an OpenID token for user information + description: |- + Exchanges an OpenID access token for information about the user + who generated the token. Currently this only exposes the Matrix + User ID of the owner. + operationId: exchangeOpenIdToken + parameters: + - in: path + name: access_token + type: string + description: |- + The OpenID access token to get information about the owner for. + required: true + x-example: SomeT0kenHere + responses: + 200: + description: |- + Information about the user who generated the OpenID access token. + schema: + type: object + properties: + sub: + type: string + description: The Matrix User ID who generated the token. + example: "@alice:example.com" + required: ['sub'] + 401: + description: The token was not recognized or has expired. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_UNKNOWN_TOKEN", + "error": "Access token unknown or expired" + } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 84a766397..e6fc89366 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -995,6 +995,18 @@ that can be made. {{query_ss_http_api}} +OpenID +------ + +Third party services can exchange an access token previously generated by the +`Client-Server API` for information about a user. This can help verify that a +user is who they say they are without granting full access to the user's account. + +Access tokens generated by the OpenID API are only good for the OpenID API and +nothing else. + +{{openid_ss_http_api}} + Send-to-device messaging ------------------------ From 05a2427c7304292e48d9501574b7bdf006ce3153 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 19:51:05 -0600 Subject: [PATCH 011/166] Document how presence EDUs work between servers It's worth noting that Synapse does not make use of the `poll` or `unpoll` fields, and therefore the wording has been updated to permit servers to reject users. In the case of synapse, it would automatically reject everyone in the list by nature of ignoring it. --- .../event-schemas/m.presence.accept.yaml | 45 +++++++++ .../event-schemas/m.presence.deny.yaml | 54 +++++++++++ .../event-schemas/m.presence.invite.yaml | 44 +++++++++ .../definitions/event-schemas/m.presence.yaml | 97 +++++++++++++++++++ scripts/templating/matrix_templates/units.py | 1 + specification/server_server_api.rst | 67 ++++--------- 6 files changed, 258 insertions(+), 50 deletions(-) create mode 100644 api/server-server/definitions/event-schemas/m.presence.accept.yaml create mode 100644 api/server-server/definitions/event-schemas/m.presence.deny.yaml create mode 100644 api/server-server/definitions/event-schemas/m.presence.invite.yaml create mode 100644 api/server-server/definitions/event-schemas/m.presence.yaml diff --git a/api/server-server/definitions/event-schemas/m.presence.accept.yaml b/api/server-server/definitions/event-schemas/m.presence.accept.yaml new file mode 100644 index 000000000..3b80ac44d --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.presence.accept.yaml @@ -0,0 +1,45 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Presence Invite Accept EDU +description: |- + An EDU representing approval for the observing user to subscribe to the + presence of the the observed user. +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.presence_accept`` + example: "m.presence_accept" + content: + type: object + description: The invite information. + title: Invite Information + properties: + observed_user: + type: string + description: |- + The user ID that has approved the ``observer_user`` to + subscribe to their presence. + example: "@alice:elsewhere.com" + observer_user: + type: string + description: |- + The user ID that requested to subscribe to the presence of + the ``observed_user``. + example: "@john:matrix.org" + required: ['observer_user', 'observed_user'] diff --git a/api/server-server/definitions/event-schemas/m.presence.deny.yaml b/api/server-server/definitions/event-schemas/m.presence.deny.yaml new file mode 100644 index 000000000..1383866c0 --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.presence.deny.yaml @@ -0,0 +1,54 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Presence Invite Deny EDU +description: |- + An EDU representing a declination or revocation for the observing user + to subscribe to the presence of the observed user. +example: { + "origin": "domain.com", + "destination": "elsewhere.org", + "edu_type": "m.presence_deny", + "content": { + "observed_user": "@alice:elsewhere.org", + "observer_user": "@john:domain.com" + } +} +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.presence_deny`` + example: "m.presence_deny" + content: + type: object + description: The invite information. + title: Invite Information + properties: + observed_user: + type: string + description: |- + The user ID that has declined or revoked the ``observer_user`` from + subscribing to their presence. + example: "@alice:elsewhere.com" + observer_user: + type: string + description: |- + The user ID that requested to subscribe to the presence of + the ``observed_user``. + example: "@john:matrix.org" + required: ['observer_user', 'observed_user'] diff --git a/api/server-server/definitions/event-schemas/m.presence.invite.yaml b/api/server-server/definitions/event-schemas/m.presence.invite.yaml new file mode 100644 index 000000000..eeb00ae32 --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.presence.invite.yaml @@ -0,0 +1,44 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Presence Invite EDU +description: |- + An EDU representing a request to subscribe to a user's presence. +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.presence_invite`` + example: "m.presence_invite" + content: + type: object + description: The invite information. + title: Invite Information + properties: + observed_user: + type: string + description: |- + The user ID the ``observer_user`` would like to subscribe + to the presence of. + example: "@alice:elsewhere.com" + observer_user: + type: string + description: |- + The user ID that is wishing to subscribe to the presence of + the ``observed_user``. + example: "@john:matrix.org" + required: ['observer_user', 'observed_user'] diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml new file mode 100644 index 000000000..bebf8211b --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -0,0 +1,97 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Presence EDU +description: |- + An EDU representing presence updates for users of the sending homeserver. + Can also be used to request additional presence updates for users of the + receiving homeserver. +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.presence`` + example: "m.presence" + content: + type: object + description: The presence updates and requests. + title: Presence Update + properties: + push: + type: array + description: |- + A list of presence updates that the receiving server is likely + to be interested in, or is subscribed to. + items: + type: object + title: User Presence Update + properties: + user_id: + type: string + description: The user ID this presence EDU is for. + example: "@john:matrix.org" + presence: + type: enum + enum: ['offline', 'unavailable', 'online'] + description: The presence of the user. + example: "online" + status_msg: + type: string + description: An optional description to accompany the presence. + example: "Making cupcakes" + last_active_ago: + type: integer + format: int64 + description: |- + The number of milliseconds that have ellapsed since the user + last did something. + example: 5000 + currently_active: + type: boolean + description: |- + Whether or not the user is currently using a device of theirs. + Defaults to false. + example: true + required: ['user_id', 'presence', 'last_active_ago'] + poll: + type: array + description: |- + New user IDs that the sending server would like to subscribe to the + presence of. The sending server should not include users it has already + requested to be subscribed to. + + The receiving server should ensure the sending server has reasonable + interest in subscribing to the provided users. The receiver may ignore + a request to subscribe to a user the sender does not have reasonable + interest in. Reasonable interest may be residing in a room with the user, + being subscribed to a presence list, or some other requirement. + + If non-empty, the receiving server should immediately send the presence + updates to the sender for the users requested. + items: + type: string + example: ["@alice:elsewhere.org"] + unpoll: + type: array + description: |- + New user IDs the sending server is no longer interested in receiving + presence updates for. The sending server should not include users it + has previously requested to be unsubscribed from. + items: + type: string + example: ["@bob:elsewhere.org"] + required: ['push'] diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 88f7b86c2..bb8210976 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -49,6 +49,7 @@ SWAGGER_DEFINITIONS = { os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", + os.path.join(matrix_doc_dir, "api/server-server/definitions/presence"): "ss_presence", } EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema") diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index a3d3f83aa..d5177a047 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -838,57 +838,16 @@ Presence The server API for presence is based entirely on exchange of the following EDUs. There are no PDUs or Federation Queries involved. -Performing a presence update and poll subscription request:: +Servers should only send presence updates for users that the receiving server +would be interested in. This can include the receiving server sharing a room +with a given user, or a user on the receiving server has added one of the +sending server's users to their presence list. - EDU type: m.presence - - Content keys: - push: (optional): list of push operations. - Each should be an object with the following keys: - user_id: string containing a User ID - presence: "offline"|"unavailable"|"online"|"free_for_chat" - status_msg: (optional) string of free-form text - last_active_ago: milliseconds since the last activity by the user - - poll: (optional): list of strings giving User IDs - - unpoll: (optional): list of strings giving User IDs - -The presence of this combined message is two-fold: it informs the recipient -server of the current status of one or more users on the sending server (by the -``push`` key), and it maintains the list of users on the recipient server that -the sending server is interested in receiving updates for, by adding (by the -``poll`` key) or removing them (by the ``unpoll`` key). The ``poll`` and -``unpoll`` lists apply *changes* to the implied list of users; any existing IDs -that the server sent as ``poll`` operations in a previous message are not -removed until explicitly requested by a later ``unpoll``. - -On receipt of a message containing a non-empty ``poll`` list, the receiving -server should immediately send the sending server a presence update EDU of its -own, containing in a ``push`` list the current state of every user that was in -the original EDU's ``poll`` list. - -Sending a presence invite:: - - EDU type: m.presence_invite - - Content keys: - observed_user: string giving the User ID of the user whose presence is - requested (i.e. the recipient of the invite) - observer_user: string giving the User ID of the user who is requesting to - observe the presence (i.e. the sender of the invite) - -Accepting a presence invite:: - - EDU type: m.presence_accept - - Content keys - as for m.presence_invite - -Rejecting a presence invite:: - - EDU type: m.presence_deny - - Content keys - as for m.presence_invite +Servers may also request additional users by including them in the ``poll`` +array on an ``m.presence`` update. The receiving server may ignore users +requested in this array. The receiving server should maintain a list of +subscribed users for the sending server, which is appended to by the ``poll`` +array and deleted from by the ``unpoll`` array. .. TODO-doc - Explain the timing-based round-trip reduction mechanism for presence @@ -896,6 +855,14 @@ Rejecting a presence invite:: - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence +{{definition_ss_event_schemas_m_presence}} + +{{definition_ss_event_schemas_m_presence_invite}} + +{{definition_ss_event_schemas_m_presence_accept}} + +{{definition_ss_event_schemas_m_presence_accept}} + Querying for information ------------------------ From e03bfbc47b5079a7bd54bbce1a4996524b6fc617 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 22:52:33 -0600 Subject: [PATCH 012/166] Document how read receipts work over federation Federation format: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/receipts.py#L153-L166 Population of the fields that the above uses to construct the EDU: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/receipts.py#L48-L56 --- .../definitions/event-schemas/m.receipt.yaml | 82 +++++++++++++++++++ specification/server_server_api.rst | 12 +++ 2 files changed, 94 insertions(+) create mode 100644 api/server-server/definitions/event-schemas/m.receipt.yaml diff --git a/api/server-server/definitions/event-schemas/m.receipt.yaml b/api/server-server/definitions/event-schemas/m.receipt.yaml new file mode 100644 index 000000000..b8638427e --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.receipt.yaml @@ -0,0 +1,82 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Receipt EDU +description: |- + An EDU representing receipt updates for users of the sending homeserver. + When receiving receipts, the server should only update entries that are + listed in the EDU. Receipts previously received that do not appear in the + EDU should not be removed or otherwise manipulated. +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.receipt`` + example: "m.receipt" + content: + type: object + description: |- + Receipts for a particular room. The string key is the room ID for + which the receipts under it belong. + additionalProperties: + type: object + title: Room Receipts + properties: + # We strongly define the receipt type to help spec future ones later + # on. At that point, m.read can become optional (maybe). + "m.read": + type: object + description: Read receipts for users in the room. + title: User Read Receipt + properties: + event_ids: + type: array + description: |- + The event ID that the user has read up to. Must be exactly + one element in length. + minItems: 1 + maxItems: 1 + items: + type: string + example: ['$read_this_event:matrix.org'] + data: + type: object + description: Metadata for the read receipt. + title: Read Receipt Metadata + properties: + ts: + type: integer + format: int64 + description: |- + A POSIX timestamp in milliseconds for when the user read + the event specified in the read receipt. + example: 1533358089009 + required: ['ts'] + required: ['event_ids', 'data'] + required: ['m.read'] + example: { + "!some_room:domain.com": { + "m.read": { + "@john:matrix.org": { + "event_ids": ["$read_this_event:matrix.org"], + "data": { + "ts": 1533358089009 + } + } + } + } + } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index a3d3f83aa..f19f41a61 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -896,6 +896,18 @@ Rejecting a presence invite:: - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence +Receipts +-------- + +Receipts are EDUs used to communicate a marker for a given event. Currently the +only kind of receipt supported is a "read receipt", or where in the timeline a +user has read up to. + +Read receipts for events events that a user sent do not need to be sent. It is +implied that by sending the event the user has read up to the event. + +{{definition_ss_event_schemas_m_receipt}} + Querying for information ------------------------ From 25c77ab2d0a9dd6a67454e27107ee4bdcbfe86b2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:43:07 -0600 Subject: [PATCH 013/166] Define authorization requirements on federation swagger APIs --- api/server-server/backfill.yaml | 6 ++++++ api/server-server/definitions/security.yaml | 19 +++++++++++++++++++ api/server-server/event_auth.yaml | 6 ++++++ api/server-server/events.yaml | 8 ++++++++ api/server-server/invites.yaml | 4 ++++ api/server-server/joins.yaml | 6 ++++++ api/server-server/leaving.yaml | 6 ++++++ api/server-server/public_rooms.yaml | 4 ++++ api/server-server/query.yaml | 9 +++++++++ api/server-server/third_party_invite.yaml | 4 ++++ api/server-server/transactions.yaml | 4 ++++ 11 files changed, 76 insertions(+) create mode 100644 api/server-server/definitions/security.yaml diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index f9f105e2e..6b3cfaef0 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/backfill/{roomId}": get: @@ -33,6 +35,8 @@ paths: Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it are retrieved, up to the total number given by the ``limit``. operationId: backfillRoom + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -85,6 +89,8 @@ paths: walk of the ``prev_events`` for the ``latest_events``, ignoring any events in ``earliest_events`` and stopping at the ``limit``. operationId: getMissingPreviousEvents + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/definitions/security.yaml b/api/server-server/definitions/security.yaml new file mode 100644 index 000000000..6c9cc8085 --- /dev/null +++ b/api/server-server/definitions/security.yaml @@ -0,0 +1,19 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +signedRequest: + type: apiKey + description: |- + The ``Authorization`` header defined in the `Authentication`_ section. + name: Authorization + in: header diff --git a/api/server-server/event_auth.yaml b/api/server-server/event_auth.yaml index f55afddc8..8857131fa 100644 --- a/api/server-server/event_auth.yaml +++ b/api/server-server/event_auth.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/event_auth/{roomId}/{eventId}": get: @@ -31,6 +33,8 @@ paths: description: |- Retrieves the complete auth chain for a given event. operationId: getEventAuth + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -72,6 +76,8 @@ paths: bottom-up after sorting each chain by depth then by event ID. The differences are then discovered and returned as the response to this API call. operationId: compareEventAuth + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index e87a06859..cf3988a20 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -22,6 +22,8 @@ schemes: basePath: /_matrix/federation/v1 produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/state/{roomId}": get: @@ -29,6 +31,8 @@ paths: description: |- Retrieves a snapshot of a room's state at a given event. operationId: getRoomState + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -74,6 +78,8 @@ paths: event IDs. This performs the same function as calling ``/state/{roomId}``, however this returns just the event IDs rather than the full events. operationId: getRoomStateIds + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -117,6 +123,8 @@ paths: description: |- Retrieves a single event. operationId: getEvent + security: + - signedRequest: [] parameters: - in: path name: eventId diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index 98d53d498..6d905e178 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/invite/{roomId}/{eventId}": put: @@ -33,6 +35,8 @@ paths: homeserver and the invited homeserver, it can be sent to all of the servers in the room by the inviting homeserver. operationId: sendInvite + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 141429450..e76a0aa60 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/make_join/{roomId}/{userId}": get: @@ -32,6 +34,8 @@ paths: Asks the receiving server to return information that the sending server will need to prepare a join event to get into the room. operationId: makeJoin + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -145,6 +149,8 @@ paths: Submits a signed join event to the resident server for it to accept it into the room's graph. operationId: sendJoin + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml index 28bcf42cf..be08acba6 100644 --- a/api/server-server/leaving.yaml +++ b/api/server-server/leaving.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/make_leave/{roomId}/{userId}": get: @@ -32,6 +34,8 @@ paths: Asks the receiving server to return information that the sending server will need to prepare a leave event to get out of the room. operationId: makeLeave + security: + - signedRequest: [] parameters: - in: path name: roomId @@ -151,6 +155,8 @@ paths: Submits a signed leave event to the resident server for it to accept it into the room's graph. operationId: sendLeave + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/public_rooms.yaml b/api/server-server/public_rooms.yaml index 6cd07449b..d162568f8 100644 --- a/api/server-server/public_rooms.yaml +++ b/api/server-server/public_rooms.yaml @@ -22,6 +22,8 @@ schemes: basePath: /_matrix/federation/v1 produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/publicRooms": get: @@ -31,6 +33,8 @@ paths: rooms that are listed on another homeserver's directory, just those listed on the receiving homeserver's directory. operationId: getPublicRooms + security: + - signedRequest: [] parameters: - in: query name: limit diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index f569549e3..dc14724c5 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -23,6 +23,8 @@ schemes: basePath: /_matrix/federation/v1 produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/query/{queryType}": get: @@ -32,6 +34,8 @@ paths: arguments are dependent on which type of query is being made. Known query types are specified as their own endpoints as an extension to this definition. operationId: queryInfo + security: + - signedRequest: [] parameters: - in: path name: queryType @@ -54,6 +58,8 @@ paths: Servers may wish to cache the response to this query to avoid requesting the information too often. operationId: queryRoomDirectory + security: + - signedRequest: [] parameters: - in: query name: room_alias @@ -110,6 +116,9 @@ paths: Servers may wish to cache the response to this query to avoid requesting the information too often. + operationId: queryProfile + security: + - signedRequest: [] parameters: - in: query name: user_id diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml index 754a3282e..5c12247cf 100644 --- a/api/server-server/third_party_invite.yaml +++ b/api/server-server/third_party_invite.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/exchange_third_party_invite/{roomId}": put: @@ -34,6 +36,8 @@ paths: an invite as per the `Inviting to a room`_ section before returning a response to this request. operationId: exchangeThirdPartyInvite + security: + - signedRequest: [] parameters: - in: path name: roomId diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 4f4c6b28e..8d810ad59 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/send/{txnId}": put: @@ -36,6 +38,8 @@ paths: The sending server must wait and retry for a 200 OK response before sending a transaction with a different ``txnId`` to the receiving server. operationId: sendTransaction + security: + - signedRequest: [] parameters: - in: path name: txnId From fcca80dad8329332733a23038a9840db397af4c8 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 14 Aug 2018 17:58:57 -0400 Subject: [PATCH 014/166] various minor fixes - formatting fixes - add examples to homeserver/identity server discovery schema - replace DNS name with hostname --- .../definitions/wellknown/homeserver.yaml | 7 ++++--- .../definitions/wellknown/identity_server.yaml | 7 ++++--- api/client-server/wellknown.yaml | 8 ++++---- specification/client_server_api.rst | 14 +++++++------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/api/client-server/definitions/wellknown/homeserver.yaml b/api/client-server/definitions/wellknown/homeserver.yaml index 7efba8169..92ff34ed3 100644 --- a/api/client-server/definitions/wellknown/homeserver.yaml +++ b/api/client-server/definitions/wellknown/homeserver.yaml @@ -16,8 +16,9 @@ description: |- Used by clients to discover homeserver information. type: object properties: - base_url: - type: string - description: The base URL for the homeserver for client-server connections. + base_url: + type: string + description: The base URL for the homeserver for client-server connections. + example: https://matrix.example.com required: - base_url diff --git a/api/client-server/definitions/wellknown/identity_server.yaml b/api/client-server/definitions/wellknown/identity_server.yaml index eb0e0bafc..a8f7c31cf 100644 --- a/api/client-server/definitions/wellknown/identity_server.yaml +++ b/api/client-server/definitions/wellknown/identity_server.yaml @@ -16,8 +16,9 @@ description: |- Used by clients to discover identity server information. type: object properties: - base_url: - type: string - description: The base URL for the identity server for client-server connections. + base_url: + type: string + description: The base URL for the identity server for client-server connections. + example: https://identity.example.com required: - base_url diff --git a/api/client-server/wellknown.yaml b/api/client-server/wellknown.yaml index 8d19f38a8..24e190f96 100644 --- a/api/client-server/wellknown.yaml +++ b/api/client-server/wellknown.yaml @@ -13,7 +13,7 @@ # limitations under the License. swagger: '2.0' info: - title: "Matrix Client-Server server discovery API" + title: "Matrix Client-Server Server Discovery API" version: "1.0.0" host: localhost:8008 schemes: @@ -26,7 +26,7 @@ paths: get: summary: Gets Matrix server discovery information about the domain. description: |- - Gets discovery information about the domain. The file may include + Gets discovery information about the domain. The file may include additional keys, which MUST follow the Java package naming convention, e.g. ``com.example.myapp.property``. This ensures property names are suitably namespaced for each application and reduces the risk of @@ -37,7 +37,7 @@ paths: operationId: getWellknown responses: 200: - description: Server discovery information + description: Server discovery information. examples: application/json: { "m.homeserver": { @@ -61,6 +61,6 @@ paths: required: - m.homeserver 404: - description: No server discovery information available + description: No server discovery information available. tags: - Server administration diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 6d5645241..d2b7aa61f 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -219,12 +219,12 @@ Well-known URI ++++++++++++++ The ``.well-known`` method uses a JSON file at a predetermined location to -specify parameter values. The flow for this method is as follows: +specify parameter values. The flow for this method is as follows: 1. Extract the server name from the user's Matrix ID by splitting the Matrix ID at the first colon. -2. Extract the DNS name from the server name. -3. Make a GET request to ``https://dns_name/.well-known/matrix/client``. +2. Extract the hostname from the server name. +3. Make a GET request to ``https://hostname/.well-known/matrix/client``. a. If the returned status code is 404, then ``IGNORE``. b. If the returned status code is not 200, or the response body is empty, @@ -233,17 +233,17 @@ specify parameter values. The flow for this method is as follows: i. If the content cannot be parsed, then ``FAIL_PROMPT``. - d. Extract the ``base_url`` value from the ``m.homeserver`` property. This + d. Extract the ``base_url`` value from the ``m.homeserver`` property. This value is to be used as the base URL of the homeserver. i. If this value is not provided, then ``FAIL_PROMPT``. e. Validate the homeserver base URL: - i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. + i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. ii. Clients SHOULD validate that the URL points to a valid homeserver before accepting it by connecting to the ``/_matrix/client/versions`` - endpoint, and parsing and validating the data. If any step in the + endpoint, and parsing and validating the data. If any step in the validation fails, then ``FAIL_ERROR``. Validation is done as a simple check against configuration errors, before sending sensitive information such as a user's password to the server. @@ -251,7 +251,7 @@ specify parameter values. The flow for this method is as follows: f. If the ``m.identity_server`` property is present, extract the ``base_url`` value for use as the base URL of the identity server. Validation for this URL is done as in the step above, but using - ``/_matrix/identity/api/v1`` as the endpoint to connect to. If the + ``/_matrix/identity/api/v1`` as the endpoint to connect to. If the ``m.identity_server`` property is present, but does not have a ``base_url`` value, then ``FAIL_ERROR``. From a264120b387a52d8506057148b681d1faf938c1c Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 14 Aug 2018 18:06:03 -0400 Subject: [PATCH 015/166] put server discovery as its own section --- specification/client_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index d2b7aa61f..d775f2c04 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -183,7 +183,7 @@ headers to be returned by servers on all requests are: Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization Server Discovery -~~~~~~~~~~~~~~~~ +---------------- In order to allow users to connect to a Matrix server without needing to explicitly specify the homeserver's URL or other parameters, clients SHOULD use @@ -216,7 +216,7 @@ In this section, the following terms are used with specific meanings: what to do next. Well-known URI -++++++++++++++ +~~~~~~~~~~~~~~ The ``.well-known`` method uses a JSON file at a predetermined location to specify parameter values. The flow for this method is as follows: From fde48e7ee85369809955494bd9aa4c8df6c159c6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 15:12:36 -0600 Subject: [PATCH 016/166] Specify how room versioning works This is the spec PR for https://github.com/matrix-org/matrix-doc/issues/1425 Room version upgrades are not part of MSC1425. Documented aspects: * room_version on the create event * creating a room with a specific version (useful for testing) * make_join behaviour * error code documentation * grammar of room versions Based upon https://docs.google.com/document/d/1urKgReoHqxX8R_XtySB17dPi-DZcKhqTEL2_s895Wz0/edit --- api/client-server/create_room.yaml | 12 +++++-- api/server-server/joins.yaml | 33 +++++++++++++++++++ event-schemas/schema/m.room.create | 3 ++ .../appendices/identifier_grammar.rst | 31 +++++++++++++++++ specification/client_server_api.rst | 7 ++++ 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 66b5578a5..1795d547a 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -137,13 +137,21 @@ paths: type: string description: The invitee's third party identifier. required: ["id_server", "medium", "address"] + room_version: + type: string + description: |- + The room version to set for the room. If not provided, the homeserver is + to use its configured default. If provided, the homeserver will return a + 400 error with the errcode ``M_UNSUPPORTED_ROOM_VERSION`` if it does not + support the room version. + example: "1" creation_content: title: CreationContent type: object description: |- Extra keys to be added to the content of the ``m.room.create``. - The server will clobber the following keys: ``creator``. Future - versions of the specification may allow the server to clobber + The server will clobber the following keys: ``creator``, ``room_version``. + Future versions of the specification may allow the server to clobber other keys. initial_state: type: array diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 141429450..f054f1e0e 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -45,6 +45,15 @@ paths: description: The user ID the join event will be for. required: true x-example: "@someone:example.org" + - in: query + type: array + items: + type: string + name: ver + description: |- + The room versions the sending server has support for. Defaults + to ``[1]``. + x-example: ["1", "2"] responses: 200: description: |- @@ -138,6 +147,30 @@ paths: ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] ] } + 400: + description: |- + The request is invalid or the room the server is attempting + to join has a version that is not listed in the ``ver`` + parameters. + + The error should be passed through to clients so that they + may give better feedback to users. + schema: + allOf: + - $ref: "../client-server/definitions/errors/error.yaml" + - type: object + properties: + room_version: + type: string + description: |- + The version of the room. Required if the ``errcode`` + is ``M_INCOMPATIBLE_ROOM_VERSION``. + examples: + application/json: { + "errcode": "M_INCOMPATIBLE_ROOM_VERSION", + "error": "Your homeserver does not support the features required to join this room", + "room_version": "3" + } "/send_join/{roomId}/{eventId}": put: summary: Submit a signed join event to a resident server diff --git a/event-schemas/schema/m.room.create b/event-schemas/schema/m.room.create index a07ab90fc..d12b9ccd7 100644 --- a/event-schemas/schema/m.room.create +++ b/event-schemas/schema/m.room.create @@ -11,6 +11,9 @@ properties: m.federate: description: Whether users on other servers can join this room. Defaults to ``true`` if key does not exist. type: boolean + room_version: + description: The version of the room. Defaults to ``"1"`` if the key does not exist. + type: string required: - creator type: object diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index 7156c7d57..fc89f031b 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -41,6 +41,37 @@ Examples of valid server names are: * ``[1234:5678::abcd]:5678`` (IPv6 literal with explicit port) +Room Versions +~~~~~~~~~~~~~ + +Room versions are used to change properties of rooms that may not be compatible +with other servers. For example, changing the rules for event authorization would +cause older servers to potentially end up in a split-brain situation due to them +not understanding the new rules. + +A room version is defined as a string of characters which MUST NOT exceed 32 +codepoints in length. Room versions MUST NOT be empty and SHOULD contain only +the characters ``a-z``, ``0-9``, ``.``, and ``-``. + +Room versions are not intended to be parsed and should be treated as opaque +identifiers. Room versions consisting only of the characters ``0-9`` and ``.`` +are reserved for future versions of the Matrix protocol. + +The complete grammar for a legal room version is:: + + room_version = 1*room_version_char + room_version_char = DIGIT + / %x61-7A ; a-z + / "-" / "." + +Examples of valid room versions are: + +* ``1`` (would be reserved by the Matrix protocol) +* ``1.2`` (would be reserved by the Matrix protocol) +* ``1.2-beta`` +* ``com.example.version`` + + Common Identifier Format ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 58f68f25e..bd9b8dca6 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -148,6 +148,13 @@ Some requests have unique error codes: :``M_SERVER_NOT_TRUSTED``: The client's request used a third party server, eg. ID server, that this server does not trust. +:``M_UNSUPPORTED_ROOM_VERSION``: + The client's request to create a room used a room version that the server does not support. + +:``M_INCOMPATIBLE_ROOM_VERSION``: + The client attempted to join a room that has a version the server does not support. Inspect the + ``room_version`` property of the error response for the room's version. + .. _sect:txn_ids: The client-server API typically uses ``HTTP PUT`` to submit requests with a From 7751750396c2496d3f3e2bfbaf2dc6efe8df9c9a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:23:33 -0600 Subject: [PATCH 017/166] changelog --- changelogs/client_server/newsfragments/1516.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1516.feature diff --git a/changelogs/client_server/newsfragments/1516.feature b/changelogs/client_server/newsfragments/1516.feature new file mode 100644 index 000000000..8e9b26df3 --- /dev/null +++ b/changelogs/client_server/newsfragments/1516.feature @@ -0,0 +1 @@ +Add support for Room Versions. From 25d01aa4312b64131edd3097d687df3902093eff Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:33:09 -0600 Subject: [PATCH 018/166] Dedicate a section on how to use access tokens Fixes https://github.com/matrix-org/matrix-doc/issues/1042. --- specification/client_server_api.rst | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 58f68f25e..27cd9a0f2 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -191,10 +191,6 @@ previously obtained credentials in the form of an ``access_token`` query parameter or through an Authorization Header of ``Bearer $access_token``. An access token is typically obtained via the `Login`_ or `Registration`_ processes. -When credentials are required but missing or invalid, the HTTP call will -return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or -``M_UNKNOWN_TOKEN`` respectively. - .. NOTE:: This specification does not mandate a particular format for the access @@ -202,6 +198,19 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or to choose an appropriate format. Server implementors may like to investigate `macaroons `_. +Using access tokens +~~~~~~~~~~~~~~~~~~~ + +Access tokens may be provided in two ways, both of which the homeserver MUST +support: + +1. Via a query string parameter, ``access_token=TheTokenHere``. +#. Via a request header, ``Authorization: Bearer TheTokenHere``. + +When credentials are required but missing or invalid, the HTTP call will +return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or +``M_UNKNOWN_TOKEN`` respectively. + Relationship between access tokens and devices ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From b159f218576aec308745283b3ffdcb4a4561bdd5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:34:23 -0600 Subject: [PATCH 019/166] changelog --- changelogs/client_server/newsfragments/1517.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1517.clarification diff --git a/changelogs/client_server/newsfragments/1517.clarification b/changelogs/client_server/newsfragments/1517.clarification new file mode 100644 index 000000000..b16928c1e --- /dev/null +++ b/changelogs/client_server/newsfragments/1517.clarification @@ -0,0 +1 @@ +Clarify how access tokens are meant to be supplied to the homeserver. \ No newline at end of file From ca87876f1ba2a16f88a60ae9880d1db54ddf496b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:37:52 -0600 Subject: [PATCH 020/166] Clarify that the Authorization header is preferred --- specification/client_server_api.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 27cd9a0f2..e64572ab0 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -207,6 +207,11 @@ support: 1. Via a query string parameter, ``access_token=TheTokenHere``. #. Via a request header, ``Authorization: Bearer TheTokenHere``. +Clients are encouraged to use the ``Authorization`` header where possible +to prevent the access token being leaked in access/HTTP logs. The query +string should only be used in cases where the ``Authorization`` header is +unaccessible for the client. + When credentials are required but missing or invalid, the HTTP call will return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or ``M_UNKNOWN_TOKEN`` respectively. From d6c54b0278bbf284c1232878fec41f24d9c9b468 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:39:01 -0600 Subject: [PATCH 021/166] unaccessible isn't a word --- specification/client_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index e64572ab0..7cd14f7f5 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -210,7 +210,7 @@ support: Clients are encouraged to use the ``Authorization`` header where possible to prevent the access token being leaked in access/HTTP logs. The query string should only be used in cases where the ``Authorization`` header is -unaccessible for the client. +inaccessible for the client. When credentials are required but missing or invalid, the HTTP call will return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or From 45c68e323a17f3e6b25b1c21038a5a6d600b3e03 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 17:25:30 -0600 Subject: [PATCH 022/166] Add general clarity to the /createRoom endpoint This commit does a number of things: * Minor formatting/alignment changes * Document the room_alias response key. This could be deprecated now, or forfeited, if needed. * Remove the guest_can_join parameter - it is not actually supported * Document the previously undocumented power_level_content_override parameter * Clarify that the room_id is required on the response * More clearly spell out which events are created as part of the request * Clarify how the room alias becomes the canonical alias * Clarify how the `visibility` may be used to determine a default preset to apply * Document the `m.federate` creation content parameter, adding an option for the homeserver to define a default value References: * Preset being inferred by the visibility: https://github.com/matrix-org/synapse/blob/cd32c19a604549b4518d748c07149d140bc9e063/synapse/handlers/room.py#L172-L177 * Power level content overrides: * https://github.com/matrix-org/synapse/blob/master/synapse/handlers/room.py#L198 * https://github.com/matrix-org/synapse/blob/master/synapse/handlers/room.py#L335-L359 * Aliases becoming canonical: https://github.com/matrix-org/synapse/blob/master/synapse/handlers/room.py#L366-L370 * `m.federate` landing in the create event: https://github.com/matrix-org/synapse/blob/master/synapse/handlers/room.py#L311-L315 Fixes https://github.com/matrix-org/matrix-doc/issues/1243 Fixes https://github.com/matrix-org/matrix-doc/issues/1471 Inspired by https://github.com/matrix-org/matrix-doc/issues/1213 --- api/client-server/create_room.yaml | 67 ++++++++++++++----- event-schemas/power_level_content_schema.yaml | 56 ++++++++++++++++ event-schemas/schema/m.room.power_levels | 44 +----------- 3 files changed, 107 insertions(+), 60 deletions(-) create mode 100644 event-schemas/power_level_content_schema.yaml diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 66b5578a5..be99c4ab1 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -39,16 +39,20 @@ paths: apply the events implied by the request in the following order: 0. A default ``m.room.power_levels`` event, giving the room creator - (and not other members) permission to send state events. + (and not other members) permission to send state events. Overridden + by the ``power_level_content_override`` parameter. - 1. Events set by the ``preset``. + 1. Events set by the ``preset``. Currently these are the ``m.room.join_rules``, + ``m.room.history_visibility``, and ``m.room.guest_access`` state events. 2. Events listed in ``initial_state``, in the order that they are listed. - 3. Events implied by ``name`` and ``topic``. + 3. Events implied by ``name`` and ``topic`` (``m.room.name`` and ``m.room.topic`` + state events). - 4. Invite events implied by ``invite`` and ``invite_3pid``. + 4. Invite events implied by ``invite`` and ``invite_3pid`` (``m.room.member`` with + ``membership: invite`` and ``m.room.third_party_invite``). The available presets do the following with respect to room state: @@ -60,6 +64,9 @@ paths: ``public_chat`` ``public`` ``shared`` ``forbidden`` ======================== ============== ====================== ================ ========= + The server will create a ``m.room.create`` event in the room with the + requesting user as the creator, alongside other keys provided in the + ``creation_content``. operationId: createRoom security: - accessToken: [] @@ -70,14 +77,14 @@ paths: schema: type: object example: { - "preset": "public_chat", - "room_alias_name": "thepub", - "name": "The Grand Duke Pub", - "topic": "All about happy hour", - "creation_content": { - "m.federate": false - } + "preset": "public_chat", + "room_alias_name": "thepub", + "name": "The Grand Duke Pub", + "topic": "All about happy hour", + "creation_content": { + "m.federate": false } + } properties: visibility: type: string @@ -89,6 +96,11 @@ paths: ``private`` visibility if this key is not included. NB: This should not be confused with ``join_rules`` which also uses the word ``public``. + + If no ``preset`` is specificed, the server may use the visbility + to determine which preset to use. A visbility of ``public`` + equates to a preset of ``public_chat`` and ``private`` visibility + equates to a preset of ``private_chat``. room_alias_name: type: string description: |- @@ -98,6 +110,9 @@ paths: created the room. For example, if this was set to "foo" and sent to the homeserver "example.com" the complete room alias would be ``#foo:example.com``. + + The complete room alias will become the canonical alias for + the room. name: type: string description: |- @@ -145,6 +160,14 @@ paths: The server will clobber the following keys: ``creator``. Future versions of the specification may allow the server to clobber other keys. + properties: + "m.federate": + type: boolean + description: |- + Whether users on other servers can join this room. Defaults + to ``true`` or what the server specifies. Setting this to + ``false`` may prevent users from being invited as part of + this room creation request if they reside on other servers. initial_state: type: array description: |- @@ -181,10 +204,14 @@ paths: This flag makes the server set the ``is_direct`` flag on the ``m.room.member`` events sent to the users in ``invite`` and ``invite_3pid``. See `Direct Messaging`_ for more information. - guest_can_join: - type: boolean + power_level_content_override: + title: Power Level Event Content description: |- - Allows guests to join the room. See `Guest Access`_ for more information. + The power level content to override in the default power level + event. This object is applied on top of the generated power + level event prior to it being sent to the room. Defaults + to overriding nothing. + $ref: "definitions/event-schemas/power_level_content_schema.yaml" responses: 200: description: Information about the newly created room. @@ -196,10 +223,17 @@ paths: type: string description: |- The created room's ID. + room_alias: + type: string + description: |- + The complete room alias for the room, if a room alias was created + for the room. + required: ['room_id'] examples: application/json: { - "room_id": "!sefiuhWgwghwWgh:example.com" - } + "room_id": "!sefiuhWgwghwWgh:example.com", + "room_alias": "#thepub:example.com" + } 400: description: |- @@ -218,6 +252,5 @@ paths: ``M_INVALID_ROOM_STATE``). schema: "$ref": "definitions/errors/error.yaml" - tags: - Room creation diff --git a/event-schemas/power_level_content_schema.yaml b/event-schemas/power_level_content_schema.yaml new file mode 100644 index 000000000..5859d56ee --- /dev/null +++ b/event-schemas/power_level_content_schema.yaml @@ -0,0 +1,56 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + ban: + description: The level required to ban a user. Defaults to 50 if unspecified. + type: number + events: + additionalProperties: + type: number + description: The level required to send specific event types. This is a mapping from event type to power level required. + title: Event power levels + type: object + events_default: + description: |- + The default level required to send message events. Can be + overridden by the ``events`` key. Defaults to 0 if unspecified. + type: number + invite: + description: The level required to invite a user. Defaults to 50 if unspecified. + type: number + kick: + description: The level required to kick a user. Defaults to 50 if unspecified. + type: number + redact: + description: The level required to redact an event. Defaults to 50 if unspecified. + type: number + state_default: + description: |- + The default level required to send state events. Can be overridden + by the ``events`` key. Defaults to 50 if unspecified, but 0 if + there is no ``m.room.power_levels`` event at all. + type: number + users: + additionalProperties: + type: number + description: The power levels for specific users. This is a mapping from ``user_id`` to power level for that user. + title: User power levels + type: object + users_default: + description: |- + The default power level for every user in the room, unless their + ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if + unspecified. + type: number +type: object \ No newline at end of file diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index 13a44c709..174ada820 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -43,49 +43,7 @@ description: |- properties: content: - properties: - ban: - description: The level required to ban a user. Defaults to 50 if unspecified. - type: number - events: - additionalProperties: - type: number - description: The level required to send specific event types. This is a mapping from event type to power level required. - title: Event power levels - type: object - events_default: - description: |- - The default level required to send message events. Can be - overridden by the ``events`` key. Defaults to 0 if unspecified. - type: number - invite: - description: The level required to invite a user. Defaults to 50 if unspecified. - type: number - kick: - description: The level required to kick a user. Defaults to 50 if unspecified. - type: number - redact: - description: The level required to redact an event. Defaults to 50 if unspecified. - type: number - state_default: - description: |- - The default level required to send state events. Can be overridden - by the ``events`` key. Defaults to 50 if unspecified, but 0 if - there is no ``m.room.power_levels`` event at all. - type: number - users: - additionalProperties: - type: number - description: The power levels for specific users. This is a mapping from ``user_id`` to power level for that user. - title: User power levels - type: object - users_default: - description: |- - The default power level for every user in the room, unless their - ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if - unspecified. - type: number - type: object + $ref: "../power_level_content_schema.yaml" state_key: description: A zero-length string. pattern: '^$' From ef7570e62d59d2feecc2da28504dc9b32f298f94 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 17:29:06 -0600 Subject: [PATCH 023/166] Changelog --- changelogs/client_server/newsfragments/1518.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1518.clarification diff --git a/changelogs/client_server/newsfragments/1518.clarification b/changelogs/client_server/newsfragments/1518.clarification new file mode 100644 index 000000000..3e2adea34 --- /dev/null +++ b/changelogs/client_server/newsfragments/1518.clarification @@ -0,0 +1 @@ +Document additional parameters on the ``/createRoom`` API. \ No newline at end of file From 2eab07ade4ffe0a619e315436ceb947124ec6566 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 16 Aug 2018 11:45:20 -0600 Subject: [PATCH 024/166] Fix header in server-server API --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 5f791a70e..b5676b780 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -606,7 +606,7 @@ the events it is missing. {{backfill_ss_http_api}} Retrieving events ----------------- +----------------- In some circumstances, a homeserver may be missing a particular event or information about the room which cannot be easily determined from backfilling. These APIs provide From c891e4a957d4e004517c3fbe5978da4764b7555b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 16 Aug 2018 12:39:47 -0600 Subject: [PATCH 025/166] Require the push gateway URL to be of a specific path --- api/client-server/pusher.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 34050d3f6..3bedf866e 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -222,7 +222,9 @@ paths: type: string description: |- Required if ``kind`` is ``http``. The URL to use to send - notifications to. + notifications to. MUST be an HTTPS URL with a path pointing + to ``/_matrix/push/v1/notify``. + example: "https://push-gateway.location.here/_matrix/push/v1/notify" format: type: string description: |- From 17a0dcc7d396a1ce9ca55141a5491e47b62c1f62 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 16 Aug 2018 23:11:07 +0100 Subject: [PATCH 026/166] add newsfragment for #1176 --- changelogs/client_server/newsfragments/1176.new | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1176.new diff --git a/changelogs/client_server/newsfragments/1176.new b/changelogs/client_server/newsfragments/1176.new new file mode 100644 index 000000000..41e30799b --- /dev/null +++ b/changelogs/client_server/newsfragments/1176.new @@ -0,0 +1 @@ +Specify how to control the power level required for ``@room`` \ No newline at end of file From 5b30d33b89541d5398901fe73eeac73dc1a29cc2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 02:51:41 -0600 Subject: [PATCH 027/166] Simpler language --- api/client-server/pusher.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 3bedf866e..d232baf9f 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -222,8 +222,8 @@ paths: type: string description: |- Required if ``kind`` is ``http``. The URL to use to send - notifications to. MUST be an HTTPS URL with a path pointing - to ``/_matrix/push/v1/notify``. + notifications to. MUST be an HTTPS URL with a path of + ``/_matrix/push/v1/notify``. example: "https://push-gateway.location.here/_matrix/push/v1/notify" format: type: string From e7aed3da269387a28ed43b79c9d3eef66436845c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:16:39 -0600 Subject: [PATCH 028/166] Remove poll/unpoll from presence --- .../definitions/event-schemas/m.presence.yaml | 31 +------------------ specification/server_server_api.rst | 8 +---- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index bebf8211b..6d83b967a 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -16,8 +16,6 @@ type: object title: Presence EDU description: |- An EDU representing presence updates for users of the sending homeserver. - Can also be used to request additional presence updates for users of the - receiving homeserver. allOf: - $ref: ../edu.yaml - type: object @@ -35,7 +33,7 @@ allOf: type: array description: |- A list of presence updates that the receiving server is likely - to be interested in, or is subscribed to. + to be interested in. items: type: object title: User Presence Update @@ -67,31 +65,4 @@ allOf: Defaults to false. example: true required: ['user_id', 'presence', 'last_active_ago'] - poll: - type: array - description: |- - New user IDs that the sending server would like to subscribe to the - presence of. The sending server should not include users it has already - requested to be subscribed to. - - The receiving server should ensure the sending server has reasonable - interest in subscribing to the provided users. The receiver may ignore - a request to subscribe to a user the sender does not have reasonable - interest in. Reasonable interest may be residing in a room with the user, - being subscribed to a presence list, or some other requirement. - - If non-empty, the receiving server should immediately send the presence - updates to the sender for the users requested. - items: - type: string - example: ["@alice:elsewhere.org"] - unpoll: - type: array - description: |- - New user IDs the sending server is no longer interested in receiving - presence updates for. The sending server should not include users it - has previously requested to be unsubscribed from. - items: - type: string - example: ["@bob:elsewhere.org"] required: ['push'] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d5177a047..6cc04d95e 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -841,13 +841,7 @@ EDUs. There are no PDUs or Federation Queries involved. Servers should only send presence updates for users that the receiving server would be interested in. This can include the receiving server sharing a room with a given user, or a user on the receiving server has added one of the -sending server's users to their presence list. - -Servers may also request additional users by including them in the ``poll`` -array on an ``m.presence`` update. The receiving server may ignore users -requested in this array. The receiving server should maintain a list of -subscribed users for the sending server, which is appended to by the ``poll`` -array and deleted from by the ``unpoll`` array. +sending server's users to their presence list. .. TODO-doc - Explain the timing-based round-trip reduction mechanism for presence From ff1afaa8f7a4a5828d46a60d27ece5de985e8de3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:18:45 -0600 Subject: [PATCH 029/166] Add a note about how presence lists work --- specification/server_server_api.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6cc04d95e..236d90f5d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -843,6 +843,12 @@ would be interested in. This can include the receiving server sharing a room with a given user, or a user on the receiving server has added one of the sending server's users to their presence list. +Clients may define lists of users that they are interested in via "Presence +Lists" through the `Client-Server API`_. When users are added to a presence +list, a ``m.presence_invite`` EDU is sent to them. The user may then accept +or deny their involvement in the list by sending either an ``m.presence_accept`` +or ``m.presence_deny`` EDU back. + .. TODO-doc - Explain the timing-based round-trip reduction mechanism for presence messages @@ -855,7 +861,7 @@ sending server's users to their presence list. {{definition_ss_event_schemas_m_presence_accept}} -{{definition_ss_event_schemas_m_presence_accept}} +{{definition_ss_event_schemas_m_presence_deny}} Querying for information ------------------------ From 549a25cad9b73df38d84b33e1837bf161f78b18b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:33:40 -0600 Subject: [PATCH 030/166] Add a mention about how currently_active works Reference: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/presence.py#L66-L68 --- api/server-server/definitions/event-schemas/m.presence.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index 6d83b967a..011db2ca1 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -62,6 +62,8 @@ allOf: type: boolean description: |- Whether or not the user is currently using a device of theirs. + For example, if the user's ``last_active_ago`` was within the + last few minutes, they may be considered ``currently_active``. Defaults to false. example: true required: ['user_id', 'presence', 'last_active_ago'] From 766402a7025112569ddfe1efdc257513ff7d7ddf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:34:08 -0600 Subject: [PATCH 031/166] Use strict types for the presence EDUs --- .../definitions/event-schemas/m.presence.accept.yaml | 3 ++- .../definitions/event-schemas/m.presence.deny.yaml | 3 ++- .../definitions/event-schemas/m.presence.invite.yaml | 3 ++- api/server-server/definitions/event-schemas/m.presence.yaml | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.presence.accept.yaml b/api/server-server/definitions/event-schemas/m.presence.accept.yaml index 3b80ac44d..72f932d2b 100644 --- a/api/server-server/definitions/event-schemas/m.presence.accept.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.accept.yaml @@ -22,7 +22,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.presence_accept'] description: The string ``m.presence_accept`` example: "m.presence_accept" content: diff --git a/api/server-server/definitions/event-schemas/m.presence.deny.yaml b/api/server-server/definitions/event-schemas/m.presence.deny.yaml index 1383866c0..3264a5af2 100644 --- a/api/server-server/definitions/event-schemas/m.presence.deny.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.deny.yaml @@ -31,7 +31,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.presence_deny'] description: The string ``m.presence_deny`` example: "m.presence_deny" content: diff --git a/api/server-server/definitions/event-schemas/m.presence.invite.yaml b/api/server-server/definitions/event-schemas/m.presence.invite.yaml index eeb00ae32..8ae848721 100644 --- a/api/server-server/definitions/event-schemas/m.presence.invite.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.invite.yaml @@ -21,7 +21,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.presence_invite'] description: The string ``m.presence_invite`` example: "m.presence_invite" content: diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index 011db2ca1..f9e0499b1 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -21,7 +21,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.presence'] description: The string ``m.presence`` example: "m.presence" content: From 96896fe5d6515a98f196556845f4e25fe48615d7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:34:50 -0600 Subject: [PATCH 032/166] Add a strict type the m.typing EDU --- api/server-server/definitions/event-schemas/m.typing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/server-server/definitions/event-schemas/m.typing.yaml b/api/server-server/definitions/event-schemas/m.typing.yaml index d4fa2f81e..101d8d79e 100644 --- a/api/server-server/definitions/event-schemas/m.typing.yaml +++ b/api/server-server/definitions/event-schemas/m.typing.yaml @@ -20,7 +20,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.typing'] description: The string ``m.typing`` example: "m.typing" content: From 44d1f8dbe5bf5826357d1d86d01c4897d3c83acd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:46:11 -0600 Subject: [PATCH 033/166] s/timeline/event graph --- specification/server_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f19f41a61..150e69e0d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -900,8 +900,8 @@ Receipts -------- Receipts are EDUs used to communicate a marker for a given event. Currently the -only kind of receipt supported is a "read receipt", or where in the timeline a -user has read up to. +only kind of receipt supported is a "read receipt", or where in the event graph +the user has read up to. Read receipts for events events that a user sent do not need to be sent. It is implied that by sending the event the user has read up to the event. From c492fe43b5d42f7f9d14228973e12b8a744ce3b8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 09:46:31 -0600 Subject: [PATCH 034/166] Add strict typing to the m.receipt EDU; Fix description of event_ids --- api/server-server/definitions/event-schemas/m.receipt.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.receipt.yaml b/api/server-server/definitions/event-schemas/m.receipt.yaml index b8638427e..210988f98 100644 --- a/api/server-server/definitions/event-schemas/m.receipt.yaml +++ b/api/server-server/definitions/event-schemas/m.receipt.yaml @@ -24,7 +24,8 @@ allOf: - type: object properties: edu_type: - type: string + type: enum + enum: ['m.receipt'] description: The string ``m.receipt`` example: "m.receipt" content: @@ -46,8 +47,7 @@ allOf: event_ids: type: array description: |- - The event ID that the user has read up to. Must be exactly - one element in length. + The extremity event IDs that the user has read up to. minItems: 1 maxItems: 1 items: From 6612dbecf182399af05403bc60097819382476ab Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 17 Aug 2018 11:54:14 -0400 Subject: [PATCH 035/166] tweak wording for validation --- specification/client_server_api.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index d775f2c04..1a566aa03 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -242,11 +242,13 @@ specify parameter values. The flow for this method is as follows: i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. ii. Clients SHOULD validate that the URL points to a valid homeserver - before accepting it by connecting to the ``/_matrix/client/versions`` - endpoint, and parsing and validating the data. If any step in the - validation fails, then ``FAIL_ERROR``. Validation is done as a simple - check against configuration errors, before sending sensitive - information such as a user's password to the server. + before accepting it by connecting to the |/_matrix/client/versions|_ + endpoint, ensuring that it does not return an error, and parsing and + validating that the data conforms with the expected response + format. If any step in the validation fails, then + ``FAIL_ERROR``. Validation is done as a simple check against + configuration errors, in order to ensure that the discovered address + points to a valid homeserver. f. If the ``m.identity_server`` property is present, extract the ``base_url`` value for use as the base URL of the identity server. @@ -1649,5 +1651,8 @@ have to wait in milliseconds before they can try again. .. |/user//account_data/| replace:: ``/user//account_data/`` .. _/user//account_data/: #put-matrix-client-%CLIENT_MAJOR_VERSION%-user-userid-account-data-type +.. |/_matrix/client/versions| replace:: ``/_matrix/client/versions`` +.. _/_matrix/client/versions: #get-matrix-client-versions + .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 .. _`3PID Types`: ../appendices.html#pid-types From f5dc0eaed23e6e9dea0ea61f6a2ebefff5ade34a Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 17 Aug 2018 12:00:13 -0400 Subject: [PATCH 036/166] document msisdn-related endpoints in IS (#1507) * add msisdn endpoints in Identity Server spec * add in CS endpoints that use the IS msisdn endpoints --- api/client-server/administrative_contact.yaml | 20 +- api/client-server/registration.yaml | 95 +++++++- api/identity/phone_associations.yaml | 203 ++++++++++++++++++ .../client_server/newsfragments/1507.new | 1 + specification/identity_service_api.rst | 5 + 5 files changed, 316 insertions(+), 8 deletions(-) create mode 100644 api/identity/phone_associations.yaml create mode 100644 changelogs/client_server/newsfragments/1507.new diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index 8f0319d5a..1cf66fe1f 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -66,7 +66,7 @@ paths: medium: type: string description: The medium of the third party identifier. - enum: ["email"] + enum: ["email", "msisdn"] address: type: string description: The third party identifier address. @@ -143,7 +143,21 @@ paths: validation tokens when adding an email address to an account. This API's parameters and response is identical to that of the HS API |/register/email/requestToken|_ endpoint. - operationId: requestTokenTo3PID + operationId: requestTokenTo3PIDEmail responses: 200: - description: An email was sent to the given address + description: An email was sent to the given address. + "/account/3pid/msisdn/requestToken": + post: + summary: Requests a validation token be sent to the given email address for the purpose of adding a phone number to an account. + description: |- + Proxies the identity server API ``validate/msisdn/requestToken``, but + first checks that the given phone number is **not** already associated + with an account on this Home Server. This API should be used to request + validation tokens when adding a phone number to an account. This API's + parameters and response is identical to that of the HS API + |/register/msisdn/requestToken|_ endpoint. + operationId: requestTokenTo3PIDMSISDN + responses: + 200: + description: An SMS message was sent to the given phone number. diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 6ae4ddd37..56da9addb 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -196,11 +196,9 @@ paths: description: |- Proxies the identity server API ``validate/email/requestToken``, but first checks that the given email address is not already associated - with an account on this Home Server. Note that, for consistency, - this API takes JSON objects, though the Identity Server API takes - ``x-www-form-urlencoded`` parameters. See the Identity Server API for + with an account on this Home Server. See the Identity Server API for further information. - operationId: requestTokenToRegister + operationId: requestTokenToRegisterEmail parameters: - in: body name: body @@ -252,6 +250,71 @@ paths: } schema: "$ref": "definitions/errors/error.yaml" + "/register/msisdn/requestToken": + post: + summary: Requests a validation token be sent to the given phone number for the purpose of registering an account + description: |- + Proxies the identity server API ``validate/msisdn/requestToken``, but + first checks that the given phone number is not already associated + with an account on this Home Server. See the Identity Server API for + further information. + operationId: requestTokenToRegisterMSISDN + parameters: + - in: body + name: body + schema: + type: object + properties: + id_server: + type: string + description: The ID server to send the onward request to as a hostname with an appended colon and port number if the port is not the default. + example: "id.matrix.org" + client_secret: + type: string + description: Client-generated secret string used to protect this session. + example: "this_is_my_secret_string" + country: + type: string + description: |- + The two-letter uppercase ISO country code that the number in + ``phone_number`` should be parsed as if it were dialled from. + phone_number: + type: string + description: The phone number. + example: "example@example.com" + send_attempt: + type: number + description: Used to distinguish protocol level retries from requests to re-send the SMS message. + example: 1 + required: ["client_secret", "country", "phone_number", "send_attempt"] + responses: + 200: + description: |- + An SMS message has been sent to the specified phone number. + Note that this may be an SMS message containing the validation token or it may be informing + the user of an error. + examples: + application/json: {} + schema: + type: object + 400: + description: |- + Part of the request was invalid. This may include one of the following error codes: + + * ``M_THREEPID_IN_USE`` : The phone number is already registered to an account on this server. + However, if the home server has the ability to send SMS message, it is recommended that the server + instead send an SMS message to the user with instructions on how to reset their password. + This prevents malicious parties from being able to determine if a given phone number + has an account on the Home Server in question. + * ``M_SERVER_NOT_TRUSTED`` : The ``id_server`` parameter refers to an ID server + that is not trusted by this Home Server. + examples: + application/json: { + "errcode": "M_THREEPID_IN_USE", + "error": "The specified address is already in use" + } + schema: + "$ref": "definitions/errors/error.yaml" "/account/password": post: summary: "Changes a user's password." @@ -319,10 +382,32 @@ paths: .. |/register/email/requestToken| replace:: ``/register/email/requestToken`` .. _/register/email/requestToken: #post-matrix-client-%CLIENT_MAJOR_VERSION%-register-email-requesttoken - operationId: requestTokenToResetPassword + operationId: requestTokenToResetPasswordEmail responses: 200: description: An email was sent to the given address + "/account/password/msisdn/requestToken": + post: + summary: Requests a validation token be sent to the given phone number for the purpose of resetting a user's password. + description: |- + Proxies the identity server API ``validate/msisdn/requestToken``, but + first checks that the given phone number **is** associated with an account + on this Home Server. This API should be used to request + validation tokens when authenticating for the + `account/password` endpoint. This API's parameters and response are + identical to that of the HS API |/register/msisdn/requestToken|_ except that + `M_THREEPID_NOT_FOUND` may be returned if no account matching the + given email address could be found. The server may instead send an + SMS message to the given address prompting the user to create an account. + `M_THREEPID_IN_USE` may not be returned. + + .. |/register/msisdn/requestToken| replace:: ``/register/msisdn/requestToken`` + + .. _/register/msisdn/requestToken: #post-matrix-client-%CLIENT_MAJOR_VERSION%-register-email-requesttoken + operationId: requestTokenToResetPasswordMSISDN + responses: + 200: + description: An SMS message was sent to the given phone number. "/account/deactivate": post: summary: "Deactivate a user's account." diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml new file mode 100644 index 000000000..c2cc6cfe7 --- /dev/null +++ b/api/identity/phone_associations.yaml @@ -0,0 +1,203 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Identity Service Phone Number Associations API" + version: "1.0.0" +host: localhost:8090 +schemes: + - https + - http +basePath: /_matrix/identity/api/v1 +produces: + - application/json +paths: + "/validate/msisdn/requestToken": + post: + summary: Request a token for validating a phone number. + description: |- + Create a session for validating a phone number. + + The identity service will send an SMS message containing a token. If + that token is presented to the identity service in the future, it + indicates that that user was able to read the SMS for that phone + number, and so we validate ownership of the phone number. + + Note that Home Servers offer APIs that proxy this API, adding + additional behaviour on top, for example, + ``/register/msisdn/requestToken`` is designed specifically for use when + registering an account and therefore will inform the user if the phone + number given is already registered on the server. + + Note: for backwards compatibility with older versions of this + specification, the parameters may also be specified as + ``application/x-form-www-urlencoded`` data. However, this usage is + deprecated. + operationId: msisdnRequestToken + parameters: + - in: body + name: body + schema: + type: object + example: { + "client_secret": "monkeys_are_GREAT", + "country": "GB", + "phone_number": "07700900001", + "send_attempt": 1 + } + properties: + client_secret: + type: string + description: A unique string used to identify the validation attempt. + country: + type: string + description: |- + The two-letter uppercase ISO country code that the number in + ``phone_number`` should be parsed as if it were dialled from. + phone_number: + type: string + description: The phone number to validate. + send_attempt: + type: integer + description: |- + Optional. If specified, the server will only send an SMS if + the ``send_attempt`` is a number greater than the most recent + one which it has seen (or if it has never seen one), scoped + to that ``country`` + ``phone_number`` + ``client_secret`` + triple. This is to avoid repeatedly sending the same SMS in + the case of request retries between the POSTing user and the + identity service. The client should increment this value if + they desire a new SMS (e.g. a reminder) to be sent. + next_link: + type: string + description: |- + Optional. When the validation is completed, the identity + service will redirect the user to this URL. + required: ["client_secret", "country", "phone_number"] + responses: + 200: + description: + Session created. + examples: + application/json: { + "sid": "1234" + } + schema: + type: object + properties: + sid: + type: string + description: The session ID. + 400: + description: | + An error ocurred. Some possible errors are: + + - ``M_INVALID_ADDRESS``: The phone number provided was invalid. + - ``M_SEND_ERROR``: The validation SMS could not be sent. + "/validate/msisdn/submitToken": + post: + summary: Validate ownership of a phone number. + description: |- + Validate ownership of a phone number. + + If the three parameters are consistent with a set generated by a + ``requestToken`` call, ownership of the phone number is considered to + have been validated. This does not publish any information publicly, or + associate the phone number address with any Matrix user + ID. Specifically, calls to ``/lookup`` will not show a binding. + + Note: for backwards compatibility with older versions of this + specification, the parameters may also be specified as + ``application/x-form-www-urlencoded`` data. However, this usage is + deprecated. + operationId: msisdnSubmitTokenPost + parameters: + - in: body + name: body + schema: + type: object + example: { + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } + properties: + sid: + type: string + description: The session ID, generated by the ``requestToken`` call. + client_secret: + type: string + description: The client secret that was supplied to the ``requestToken`` call. + token: + type: string + description: The token generated by the ``requestToken`` call and sent to the user. + required: ["sid", "client_secret", "token"] + responses: + 200: + description: + The success of the validation. + examples: + application/json: { + "success": true + } + schema: + type: object + properties: + success: + type: boolean + description: Whether the validation was successful or not. + get: + summary: Validate ownership of a phone number. + description: |- + Validate ownership of a phone number. + + If the three parameters are consistent with a set generated by a + ``requestToken`` call, ownership of the phone number address is + considered to have been validated. This does not publish any + information publicly, or associate the phone number with any Matrix + user ID. Specifically, calls to ``/lookup`` will not show a binding. + + Note that, in contrast with the POST version, this endpoint will be + used by end-users, and so the response should be human-readable. + operationId: msisdnSubmitTokenGet + parameters: + - in: query + type: string + name: sid + required: true + description: The session ID, generated by the ``requestToken`` call. + x-example: 1234 + - in: query + type: string + name: client_secret + required: true + description: The client secret that was supplied to the ``requestToken`` call. + x-example: monkeys_are_GREAT + - in: query + type: string + name: token + required: true + description: The token generated by the ``requestToken`` call and sent to the user. + x-example: atoken + responses: + "200": + description: Phone number is validated. + "3xx": + description: |- + Phone number address is validated, and the ``next_link`` parameter + was provided to the ``requestToken`` call. The user must be + redirected to the URL provided by the ``next_link`` parameter. + "4xx": + description: + Validation failed. diff --git a/changelogs/client_server/newsfragments/1507.new b/changelogs/client_server/newsfragments/1507.new new file mode 100644 index 000000000..6cc468702 --- /dev/null +++ b/changelogs/client_server/newsfragments/1507.new @@ -0,0 +1 @@ +``POST /account/3pid/msisdn/requestToken``, ``POST /register/msisdn/requestToken``, and ``POST /account/password/msisdn/requestToken`` \ No newline at end of file diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index cb0795939..3b037caf0 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -119,6 +119,11 @@ Email associations {{email_associations_is_http_api}} +Phone number associations +~~~~~~~~~~~~~~~~~~~~~~~~~ + +{{phone_associations_is_http_api}} + General ~~~~~~~ From c2ed79bd1a37bf85b377c25722f3002dec8d075b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 17 Aug 2018 17:07:03 +0100 Subject: [PATCH 037/166] Sentence case for headings --- meta/documentation_style.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index e60c7847e..c6bb62bf4 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -32,6 +32,12 @@ complete specification to be merged correctly. These characters are: If you find yourself using ``^`` or beyond, you should rethink your document layout if possible. +Correct capitalisation for long section names +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Headings should start with a capital letter, and use lower-case otherwise. + + TODOs ----- From 750d4f9fdaaafac8b52c05828f2675d03c10df51 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 10:53:47 -0600 Subject: [PATCH 038/166] Rename the presence EDU files to be accurate to their types; Misc cleanup of titles --- api/server-server/definitions/event-schemas/m.presence.yaml | 2 +- .../{m.presence.accept.yaml => m.presence_accept.yaml} | 2 +- .../{m.presence.deny.yaml => m.presence_deny.yaml} | 2 +- .../{m.presence.invite.yaml => m.presence_invite.yaml} | 2 +- api/server-server/definitions/event-schemas/m.receipt.yaml | 2 +- api/server-server/definitions/event-schemas/m.typing.yaml | 2 +- .../matrix_templates/templates/schema-definition.tmpl | 2 +- scripts/templating/matrix_templates/units.py | 1 - 8 files changed, 7 insertions(+), 8 deletions(-) rename api/server-server/definitions/event-schemas/{m.presence.accept.yaml => m.presence_accept.yaml} (97%) rename api/server-server/definitions/event-schemas/{m.presence.deny.yaml => m.presence_deny.yaml} (98%) rename api/server-server/definitions/event-schemas/{m.presence.invite.yaml => m.presence_invite.yaml} (97%) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index f9e0499b1..8d4ef6e8d 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Presence EDU +title: ``m.presence`` EDU description: |- An EDU representing presence updates for users of the sending homeserver. allOf: diff --git a/api/server-server/definitions/event-schemas/m.presence.accept.yaml b/api/server-server/definitions/event-schemas/m.presence_accept.yaml similarity index 97% rename from api/server-server/definitions/event-schemas/m.presence.accept.yaml rename to api/server-server/definitions/event-schemas/m.presence_accept.yaml index 72f932d2b..4c39989b6 100644 --- a/api/server-server/definitions/event-schemas/m.presence.accept.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_accept.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Presence Invite Accept EDU +title: ``m.presence_accept`` EDU description: |- An EDU representing approval for the observing user to subscribe to the presence of the the observed user. diff --git a/api/server-server/definitions/event-schemas/m.presence.deny.yaml b/api/server-server/definitions/event-schemas/m.presence_deny.yaml similarity index 98% rename from api/server-server/definitions/event-schemas/m.presence.deny.yaml rename to api/server-server/definitions/event-schemas/m.presence_deny.yaml index 3264a5af2..1b9bff7dc 100644 --- a/api/server-server/definitions/event-schemas/m.presence.deny.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_deny.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Presence Invite Deny EDU +title: ``m.presence_deny`` EDU description: |- An EDU representing a declination or revocation for the observing user to subscribe to the presence of the observed user. diff --git a/api/server-server/definitions/event-schemas/m.presence.invite.yaml b/api/server-server/definitions/event-schemas/m.presence_invite.yaml similarity index 97% rename from api/server-server/definitions/event-schemas/m.presence.invite.yaml rename to api/server-server/definitions/event-schemas/m.presence_invite.yaml index 8ae848721..4cdc58eba 100644 --- a/api/server-server/definitions/event-schemas/m.presence.invite.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_invite.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Presence Invite EDU +title: ``m.presence_invite`` EDU description: |- An EDU representing a request to subscribe to a user's presence. allOf: diff --git a/api/server-server/definitions/event-schemas/m.receipt.yaml b/api/server-server/definitions/event-schemas/m.receipt.yaml index 210988f98..58f47e3a0 100644 --- a/api/server-server/definitions/event-schemas/m.receipt.yaml +++ b/api/server-server/definitions/event-schemas/m.receipt.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Receipt EDU +title: ``m.receipt`` EDU description: |- An EDU representing receipt updates for users of the sending homeserver. When receiving receipts, the server should only update entries that are diff --git a/api/server-server/definitions/event-schemas/m.typing.yaml b/api/server-server/definitions/event-schemas/m.typing.yaml index 101d8d79e..34b395297 100644 --- a/api/server-server/definitions/event-schemas/m.typing.yaml +++ b/api/server-server/definitions/event-schemas/m.typing.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: Typing Notification EDU +title: ``m.typing`` EDU description: A typing notification EDU for a user in a room. allOf: - $ref: ../edu.yaml diff --git a/scripts/templating/matrix_templates/templates/schema-definition.tmpl b/scripts/templating/matrix_templates/templates/schema-definition.tmpl index 42a7ae476..e2be12e8a 100644 --- a/scripts/templating/matrix_templates/templates/schema-definition.tmpl +++ b/scripts/templating/matrix_templates/templates/schema-definition.tmpl @@ -1,6 +1,6 @@ {% import 'tables.tmpl' as tables -%} -``{{definition.title}}`` Schema +``{{definition.title}}`` schema {{(11 + definition.title | length) * title_kind}} {% if 'description' in definition %} diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index c1481430d..90a87cd47 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -49,7 +49,6 @@ SWAGGER_DEFINITIONS = { os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", - os.path.join(matrix_doc_dir, "api/server-server/definitions/presence"): "ss_presence", } EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema") From a4015d5c273ba308475d40bb5311d279fa6cdac9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 10:58:20 -0600 Subject: [PATCH 039/166] Spelling --- api/server-server/definitions/event-schemas/m.presence.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index 8d4ef6e8d..b9c5f754f 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -56,7 +56,7 @@ allOf: type: integer format: int64 description: |- - The number of milliseconds that have ellapsed since the user + The number of milliseconds that have elapsed since the user last did something. example: 5000 currently_active: From 5b5b4cfbed2941e342dc06678d39bf00ec50d0f7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 13:38:38 -0600 Subject: [PATCH 040/166] Explicitly say how appservices should detect state events Fixes https://github.com/matrix-org/matrix-doc/issues/1014 --- api/application-service/application_service.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 42c0c0cfe..64064cbdf 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -32,6 +32,10 @@ paths: description: |- This API is called by the homeserver when it wants to push an event (or batch of events) to the application service. + + The application service should take care to ensure that it handles + state events by the presence of a ``state_key``, not by the event + type. operationId: sendTransaction parameters: - in: path @@ -44,7 +48,7 @@ paths: x-example: "35" - in: body name: body - description: A list of events + description: A list of events. schema: type: object example: { From fd101b6ac967e692e24183942f5651e4bf451a76 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 13:39:17 -0600 Subject: [PATCH 041/166] Misc language changes --- specification/application_service_api.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..3666df33f 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -39,7 +40,7 @@ This version of the specification is generated from Application Services -------------------- Application services are passive and can only observe events from a given -homeserver. They can inject events into rooms they are participating in. +homeserver (HS). They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the @@ -217,7 +218,8 @@ need to be able to adjust the ``origin_server_ts`` value to do this. Inputs: - Application service token (``as_token``) - - Desired timestamp + - Desired timestamp in milliseconds since the unix epoch + Notes: - This will only apply when sending events. From 7caad61b86adc8ba44b9711a3f6aa91e46ec392e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 13:40:05 -0600 Subject: [PATCH 042/166] Clearly state how the users namespace relates to interest in events Fixes https://github.com/matrix-org/matrix-doc/issues/1307 --- specification/application_service_api.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 3666df33f..5fe0aedeb 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -69,7 +69,13 @@ Registration Application services register "namespaces" of user IDs, room aliases and room IDs. These namespaces are represented as regular expressions. An application service is said to be "interested" in a given event if one of the IDs in the event match -the regular expression provided by the application service. An application +the regular expression provided by the application service, such as the room having +an alias or ID in the relevant namespaces. Similarly, the application service is +said to be interested in a given event if one of the application service's namespaced +users is the target of the event, or is a joined member of the room where the event +occurred. + +An application service can also state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" namespace. An exclusive namespace prevents humans and other application From 857bcc0fe7613fa8d4623ba4ee0ee509eb528953 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 13:49:24 -0600 Subject: [PATCH 043/166] Encourage appservices to use the Authorization header This also throws in a mention about how to handle a request with a lack of user_id. The request samples now encourage the use of the header over the query string, and have had their sample values added for some readability. Fixes https://github.com/matrix-org/matrix-doc/issues/1296 Fixes https://github.com/matrix-org/matrix-doc/issues/1424 --- specification/application_service_api.rst | 36 +++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..7119bd5a6 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -192,22 +192,28 @@ they wish to be acting on behalf of. For real users, this would require additional permissions granting the AS permission to masquerade as a matrix user. Inputs: - - Application service token (``access_token``) + - Application service token (``as_token``) - User ID in the AS namespace to act as. Notes: - - This will apply on all aspects of the CS API, except for Account Management. + - This will apply on all aspects of the Client-Server API, except for Account Management. - The ``as_token`` is inserted into ``access_token`` which is usually where the - client token is. This is done on purpose to allow application services to - reuse client SDKs. + client token is, such as via the query string or ``Authorization`` header. This + is done on purpose to allow application services to reuse client SDKs. + - The ``access_token`` should be supplied through the ``Authorization`` header where + possible to prevent the token appearing in HTTP request logs by accident. -:: +The application service may specify the virtual user to act as through use of a +``user_id`` query string parameter on the request. The user specified in the query +string must be covered by one of the application service's ``user`` namespaces. If +the parameter is missing, the homeserver is to assume the application service intends +to act as the user implied by the ``sender_localpart`` property of the registration. + +An example request would be:: - /path?access_token=$token&user_id=$userid + GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org + Authorization: Bearer YourApplicationServiceTokenHere - Query Parameters: - access_token: The application service token - user_id: The desired user ID to act as. Timestamp massaging +++++++++++++++++++ @@ -223,11 +229,10 @@ Notes: :: - /path?access_token=$token&ts=$timestamp + PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 + Authorization: Bearer YourApplicationServiceTokenHere - Query Parameters added to the send event APIs only: - access_token: The application service token - ts: The desired timestamp + Content: The event to send, as per the Client-Server API. Server admin style permissions ++++++++++++++++++++++++++++++ @@ -250,12 +255,13 @@ including the AS token on a ``/register`` request, along with a login type of :: - /register?access_token=$as_token + POST /_matrix/client/%CLIENT_MAJOR_VERSION%/register + Authorization: Bearer YourApplicationServiceTokenHere Content: { type: "m.login.application_service", - username: "" + username: "_irc_example" } Application services which attempt to create users or aliases *outside* of From 205b326e4a393eb5f32d72f3154f7ea33096d8ba Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:03:30 -0600 Subject: [PATCH 044/166] Add a note that application services cannot /sync normally Fixes https://github.com/matrix-org/matrix-doc/issues/1144 --- specification/application_service_api.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..1f1c8b5bf 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -264,6 +264,14 @@ normal users who attempt to create users or aliases *inside* an application service-defined namespace will receive the same ``M_EXCLUSIVE`` error code, but only if the application service has defined the namespace as ``exclusive``. +Using ``/sync`` and ``/events`` ++++++++++++++++++++++++++++++++ + +Application services wishing to use ``/sync`` or ``/events`` from the Client-Server +API MUST do so with a virtual user (provide a ``user_id`` via the query string). It +is expectected that the application service use the transactions pushed to it to +handle events rather than syncing with the user implied by ``sender_localpart``. + ID conventions ~~~~~~~~~~~~~~ .. TODO-spec From 954498bf7877af3d67b9d08096e0cddfbe3360b5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:13:56 -0600 Subject: [PATCH 045/166] Recommend that application services use an underscore for namespacing Fixes https://github.com/matrix-org/matrix-doc/issues/689 --- specification/application_service_api.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..778d2e816 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -83,7 +83,7 @@ regular expressions and look like: users: - exclusive: true - regex: @irc.freenode.net_.* + regex: @_irc.freenode.net_.* The registration is represented by a series of key-value pairs, which this @@ -105,12 +105,17 @@ traffic to the AS is: aliases: [] # Namespaces of room aliases which should be delegated to the AS rooms: [] # Namespaces of room ids which should be delegated to the AS +Exclusive user and alias namespaces should begin with an underscore after the +sigil to avoid collisions with other users on the homeserver. Application +services should additionally attempt to identify the service they represent +in the reserved namespace. For example, ``@_irc_.*`` would be a good namespace +to register for an application service which deals with IRC. + .. WARNING:: If the homeserver in question has multiple application services, each ``as_token`` and ``id`` MUST be unique per application service as these are used to identify the application service. The homeserver MUST enforce this. - Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 069a2f74811d489e763fc152cbe386f733dec83b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:22:06 -0600 Subject: [PATCH 046/166] Split the query user and room APIs out to their own files --- .../application_service.yaml | 115 ------------------ api/application-service/query_room.yaml | 86 +++++++++++++ api/application-service/query_user.yaml | 83 +++++++++++++ specification/application_service_api.rst | 4 + 4 files changed, 173 insertions(+), 115 deletions(-) create mode 100644 api/application-service/query_room.yaml create mode 100644 api/application-service/query_user.yaml diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 42c0c0cfe..e5e22c9b7 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -92,121 +92,6 @@ paths: } schema: type: object - "/rooms/{roomAlias}": - get: - summary: Query if a room alias should exist on the application service. - description: |- - This endpoint is invoked by the homeserver on an application service to query - the existence of a given room alias. The homeserver will only query room - aliases inside the application service's ``aliases`` namespace. The - homeserver will send this request when it receives a request to join a - room alias within the application service's namespace. - operationId: queryRoomByAlias - parameters: - - in: path - name: roomAlias - type: string - description: The room alias being queried. - required: true - x-example: "#magicforest:example.com" - responses: - 200: - description: |- - The application service indicates that this room alias exists. The - application service MUST have created a room and associated it with - the queried room alias using the client-server API. Additional - information about the room such as its name and topic can be set - before responding. - examples: - application/json: { - } - schema: - type: object - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: |- - The application service indicates that this room alias does not exist. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - "/users/{userId}": - get: - summary: Query if a user should exist on the application service. - description: |- - This endpoint is invoked by the homeserver on an application service to query - the existence of a given user ID. The homeserver will only query user IDs - inside the application service's ``users`` namespace. The homeserver will - send this request when it receives an event for an unknown user ID in - the application service's namespace. - operationId: queryUserById - parameters: - - in: path - name: userId - type: string - description: The user ID being queried. - required: true - x-example: "@alice:example.com" - responses: - 200: - description: |- - The application service indicates that this user exists. The application - service MUST create the user using the client-server API. - examples: - application/json: { - } - schema: - type: object - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: |- - The application service indicates that this user does not exist. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml new file mode 100644 index 000000000..e898f1035 --- /dev/null +++ b/api/application-service/query_room.yaml @@ -0,0 +1,86 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "/" +consumes: + - application/json +produces: + - application/json +paths: + "/rooms/{roomAlias}": + get: + summary: Query if a room alias should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given room alias. The homeserver will only query room + aliases inside the application service's ``aliases`` namespace. The + homeserver will send this request when it receives a request to join a + room alias within the application service's namespace. + operationId: queryRoomByAlias + parameters: + - in: path + name: roomAlias + type: string + description: The room alias being queried. + required: true + x-example: "#magicforest:example.com" + responses: + 200: + description: |- + The application service indicates that this room alias exists. The + application service MUST have created a room and associated it with + the queried room alias using the client-server API. Additional + information about the room such as its name and topic can be set + before responding. + examples: + application/json: { + } + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: |- + The application service indicates that this room alias does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml new file mode 100644 index 000000000..4f699ad6a --- /dev/null +++ b/api/application-service/query_user.yaml @@ -0,0 +1,83 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "/" +consumes: + - application/json +produces: + - application/json +paths: + "/users/{userId}": + get: + summary: Query if a user should exist on the application service. + description: |- + This endpoint is invoked by the homeserver on an application service to query + the existence of a given user ID. The homeserver will only query user IDs + inside the application service's ``users`` namespace. The homeserver will + send this request when it receives an event for an unknown user ID in + the application service's namespace. + operationId: queryUserById + parameters: + - in: path + name: userId + type: string + description: The user ID being queried. + required: true + x-example: "@alice:example.com" + responses: + 200: + description: |- + The application service indicates that this user exists. The application + service MUST create the user using the client-server API. + examples: + application/json: { + } + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: |- + The application service indicates that this user does not exist. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index b4950eac0..cde22a89a 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -172,6 +172,10 @@ application services MUST implement these APIs. These APIs are defined below. {{application_service_as_http_api}} +{{query_user_as_http_api}} + +{{query_room_as_http_api}} + .. _create the user: `sect:asapi-permissions`_ From 95b2b7c2bcf5249c190e3ee4b56fe9c34b8041f0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:30:42 -0600 Subject: [PATCH 047/166] Move query APIs to the right heading Fixes https://github.com/matrix-org/matrix-doc/issues/1325 Addresses some of https://github.com/matrix-org/matrix-doc/issues/1532 --- specification/application_service_api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index cde22a89a..824139f03 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -163,6 +163,10 @@ this request (e.g. to join a room alias). the application service about information about the entity such as room ID to room alias mappings. +{{query_user_as_http_api}} + +{{query_room_as_http_api}} + HTTP APIs +++++++++ @@ -172,10 +176,6 @@ application services MUST implement these APIs. These APIs are defined below. {{application_service_as_http_api}} -{{query_user_as_http_api}} - -{{query_room_as_http_api}} - .. _create the user: `sect:asapi-permissions`_ From 0863c5452e95518f7047d6b9ced3e7e7141885d4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:31:21 -0600 Subject: [PATCH 048/166] Take out the false third party network endpoints Fixes https://github.com/matrix-org/matrix-doc/issues/800 --- specification/application_service_api.rst | 73 +---------------------- 1 file changed, 3 insertions(+), 70 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 824139f03..534f1ac02 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -268,78 +268,11 @@ normal users who attempt to create users or aliases *inside* an application service-defined namespace will receive the same ``M_EXCLUSIVE`` error code, but only if the application service has defined the namespace as ``exclusive``. -ID conventions -~~~~~~~~~~~~~~ -.. TODO-spec - - Giving HSes the freedom to namespace still feels like the Right Thing here. - - Exposing a public API provides the consistency which was the main complaint - against namespacing. - - This may have knock-on effects for the AS registration API. E.g. why don't - we let ASes specify the *URI* regex they want? - -This concerns the well-defined conventions for mapping 3P network IDs to matrix -IDs, which we expect clients to be able to do by themselves. - -User IDs -++++++++ -Matrix users may wish to directly contact a virtual user, e.g. to send an email. -The URI format is a well-structured way to represent a number of different ID -types, including: - -- MSISDNs (``tel``) -- Email addresses (``mailto``) -- IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04) -- XMPP (XEP-0032) -- SIP URIs (RFC 3261) - -As a result, virtual user IDs SHOULD relate to their URI counterpart. This -mapping from URI to user ID can be expressed in a number of ways: - -- Expose a C-S API on the HS which takes URIs and responds with user IDs. -- Munge the URI with the user ID. - -Exposing an API would allow HSes to internally map user IDs however they like, -at the cost of an extra round trip (of which the response can be cached). -Munging the URI would allow clients to apply the mapping locally, but would force -user X on service Y to always map to the same munged user ID. Considering the -exposed API could just be applying this munging, there is more flexibility if -an API is exposed. - -:: - - GET /_matrix/app/%CLIENT_MAJOR_VERSION%/user?uri=$url_encoded_uri - - Returns 200 OK: - { - user_id: - } - -Room Aliases -++++++++++++ -We may want to expose some 3P network rooms so Matrix users can join them directly, -e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. -``mailto``, ``tel``. Rooms which are publicly accessible (e.g. IRC rooms) can be -exposed as an alias by the application service. Private rooms -(e.g. sending an email to someone) should not -be exposed in this way, but should instead operate using normal invite/join semantics. -Therefore, the ID conventions discussed below are only valid for public rooms which -expose room aliases. - -Matrix users may wish to join XMPP rooms (e.g. using XEP-0045) or IRC rooms. In both -cases, these rooms can be expressed as URIs. For consistency, these "room" URIs -SHOULD be mapped in the same way as "user" URIs. - -:: - - GET /_matrix/app/%CLIENT_MAJOR_VERSION%/alias?uri=$url_encoded_uri +Event fields +~~~~~~~~~~~~ - Returns 200 OK: - { - alias: - } +.. TODO-TravisR: Fix this section to be a general "3rd party networks" section -Event fields -++++++++++++ We recommend that any events that originated from a remote network should include an ``external_url`` field in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. From 60b8e72a67cb81b868cc74640d35455cd23c8bd1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 15:44:46 -0600 Subject: [PATCH 049/166] Minor text changes to the query APIs; Keep OpenMarket copyright The bulk of these APIs were copied from OpenMarket's work - we should preserve the copyright header. --- api/application-service/query_room.yaml | 6 +++--- api/application-service/query_user.yaml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index e898f1035..7a5a7e54a 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -1,3 +1,4 @@ +# Copyright 2016 OpenMarket Ltd # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,7 +40,7 @@ paths: - in: path name: roomAlias type: string - description: The room alias being queried. + description: The URL encoded room alias being queried. required: true x-example: "#magicforest:example.com" responses: @@ -51,8 +52,7 @@ paths: information about the room such as its name and topic can be set before responding. examples: - application/json: { - } + application/json: {} schema: type: object 401: diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 4f699ad6a..4d1e9f44b 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -1,3 +1,4 @@ +# Copyright 2016 OpenMarket Ltd # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,13 +34,13 @@ paths: the existence of a given user ID. The homeserver will only query user IDs inside the application service's ``users`` namespace. The homeserver will send this request when it receives an event for an unknown user ID in - the application service's namespace. + the application service's namespace, such as a room invite. operationId: queryUserById parameters: - in: path name: userId type: string - description: The user ID being queried. + description: The URL encoded user ID being queried. required: true x-example: "@alice:example.com" responses: @@ -48,8 +49,7 @@ paths: The application service indicates that this user exists. The application service MUST create the user using the client-server API. examples: - application/json: { - } + application/json: {} schema: type: object 401: From ab3272045e43679d42e05120ea251db7c7dda62c Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 5 Jun 2018 15:12:25 +0200 Subject: [PATCH 050/166] add missing v1 to m.olm in /keys/upload --- api/client-server/keys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index 6e995c2ce..457311cfe 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -194,7 +194,7 @@ paths: "user_id": "@alice:example.com", "device_id": "JLAFKJWSCS", "algorithms": [ - "m.olm.curve25519-aes-sha256", + "m.olm.v1.curve25519-aes-sha256", "m.megolm.v1.aes-sha" ], "keys": { From e210f8b05005f17dfe957d9bae01a2c0dd1e84cf Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 6 Jun 2018 16:08:00 +0200 Subject: [PATCH 051/166] add e2e messaging algorithms section intro This was written by Richard van der Hoff. --- .../modules/end_to_end_encryption.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 1f778bc2c..081382709 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -228,6 +228,28 @@ A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. +Messaging Algorithms +-------------------- + +Messaging Algorithm Names +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messaging algorithm names use the extensible naming scheme used throughout this +specification. Algorithm names that start with ``m.`` are reserved for +algorithms defined by this specification. Implementations wanting to experiment +with new algorithms are encouraged to pick algorithm names that start with +their domain to reduce the risk of collisions. + +Algorithm names should be short and meaningful, and should list the primitives +used by the algorithm so that it is easier to see if the algorithm is using a +broken primitive. + +A name of ``m.olm.v1`` is too short: it gives no information about the primitives +in use, and is difficult to extend for different primitives. However a name of +``m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256`` +is too long despite giving a more precise description of the algorithm: it adds +to the data transfer overhead and sacrifices clarity for human readers without +adding any useful extra information. Protocol definitions -------------------- From 33802dbbafd0be45ad5abb1e4eb5285fd57a1a04 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 6 Jun 2018 16:10:36 +0200 Subject: [PATCH 052/166] add olm messaging algorithm subsection This was written by Richard van der Hoff. --- .../modules/end_to_end_encryption.rst | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 081382709..1a58ee31e 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -251,6 +251,76 @@ is too long despite giving a more precise description of the algorithm: it adds to the data transfer overhead and sacrifices clarity for human readers without adding any useful extra information. +``m.olm.v1.curve25519-aes-sha2`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The name ``m.olm.v1.curve25519-aes-sha2`` corresponds to version 1 of the Olm +ratchet, as defined by the `Olm specification`_. This uses: + +* Curve25519 for the initial key agreement. +* HKDF-SHA-256 for ratchet key derivation. +* Curve25519 for the root key ratchet. +* HMAC-SHA-256 for the chain key ratchet. +* HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. + +Devices that support Olm must include "m.olm.v1.curve25519-aes-sha2" in their +list of supported messaging algorithms, must list a Curve25519 device key, and +must publish Curve25519 one-time keys. + +An event encrypted using Olm has the following format: + +.. code:: json + + { + "type": "m.room.encrypted", + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "", + "ciphertext": { + "": { + "type": 0, + "body": "" + } } } } + +``ciphertext`` is a mapping from device Curve25519 key to an encrypted payload +for that device. ``body`` is a Base64-encoded Olm message body. ``type`` is an +integer indicating the type of the message body: 0 for the initial pre-key +message, 1 for ordinary messages. + +Olm sessions will generate messages with a type of 0 until they receive a +message. Once a session has decrypted a message it will produce messages with +a type of 1. + +When a client receives a message with a type of 0 it must first check if it +already has a matching session. If it does then it will use that session to +try to decrypt the message. If there is no existing session then the client +must create a new session and use the new session to decrypt the message. A +client must not persist a session or remove one-time keys used by a session +until it has successfully decrypted a message using that session. + +Messages with type 1 can only be decrypted with an existing session. If there +is no matching session, the client should show this as an invalid message. + +The plaintext payload is of the form: + +.. code:: json + + { + "type": "", + "content": "", + "room_id": "", + } + +The type and content of the plaintext message event are given in the payload. + +We include the room ID in the payload, because otherwise the homeserver would +be able to change the room a message was sent in. + +.. TODO: claimed_keys + +Clients must confirm that the ``sender_key`` belongs to the user that sent the +message. TODO: how? + Protocol definitions -------------------- @@ -310,6 +380,7 @@ Example response: .. _ed25519: http://ed25519.cr.yp.to/ .. _curve25519: https://cr.yp.to/ecdh.html +.. _`Olm specification`: http://matrix.org/docs/spec/olm.html .. _`Signing JSON`: ../appendices.html#signing-json From 07e3de3c61cbb5b379477de5856829a449e33ff1 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 6 Jun 2018 16:12:31 +0200 Subject: [PATCH 053/166] add megolm messaging algorithm subsection This was written by Richard van der Hoff. --- specification/modules/end_to_end_encryption.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 1a58ee31e..e2cb54e52 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -321,6 +321,17 @@ be able to change the room a message was sent in. Clients must confirm that the ``sender_key`` belongs to the user that sent the message. TODO: how? +``m.megolm.v1.aes-sha2`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +The name ``m.megolm.v1.aes-sha2`` corresponds to version 1 of the Megolm +ratchet, as defined by the `Megolm specification`_. This uses: + +* HMAC-SHA-256 for the hash ratchet. +* HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. +* Ed25519 for message authenticity. + + Protocol definitions -------------------- @@ -381,6 +392,7 @@ Example response: .. _ed25519: http://ed25519.cr.yp.to/ .. _curve25519: https://cr.yp.to/ecdh.html .. _`Olm specification`: http://matrix.org/docs/spec/olm.html +.. _`Megolm specification`: http://matrix.org/docs/spec/megolm.html .. _`Signing JSON`: ../appendices.html#signing-json From 2686b990806e59cce7cad07fcc4d03ea7eba13d5 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 6 Jun 2018 16:23:32 +0200 Subject: [PATCH 054/166] fix json indent --- specification/modules/end_to_end_encryption.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index e2cb54e52..68e030e10 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -274,13 +274,16 @@ An event encrypted using Olm has the following format: { "type": "m.room.encrypted", "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", + "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "", "ciphertext": { "": { "type": 0, "body": "" - } } } } + } + } + } + } ``ciphertext`` is a mapping from device Curve25519 key to an encrypted payload for that device. ``body`` is a Base64-encoded Olm message body. ``type`` is an From 8afc82c14b70d23f7133470ed9304b2031e3a55b Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 11 Jun 2018 11:48:20 +0200 Subject: [PATCH 055/166] fix /keys/claim request example It didn't correspond to the example response. --- api/client-server/keys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index 457311cfe..55f8a5a53 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -247,7 +247,7 @@ paths: description: algorithm example: "signed_curve25519" example: - "@alice:example.com": { "JLAFKJWSCS": "curve25519" } + "@alice:example.com": { "JLAFKJWSCS": "signed_curve25519" } required: - one_time_keys responses: From 10c3307427808da03aed24165b17c4911486b3b0 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 11 Jun 2018 15:18:59 +0200 Subject: [PATCH 056/166] document device_one_time_keys_count in /sync/ response fix #1157 --- api/client-server/sync.yaml | 8 ++++++++ specification/modules/end_to_end_encryption.rst | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index 34659dd06..4b44c20e1 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -253,6 +253,14 @@ paths: description: |- Information on end-to-end device updates, as specified in |device_lists_sync|_. + device_one_time_keys_count: + title: One-time keys count + type: object + additionalProperties: + type: integer + description: |- + Information on end-to-end encryption keys, as specified + in |device_lists_sync|_. examples: application/json: { "next_batch": "s72595_4483_1934", diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 68e030e10..b030ab642 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -355,6 +355,9 @@ specified). The client is expected to use |/keys/query|_ or |/keys/changes|_ for the equivalent functionality after an initial sync, as documented in `Tracking the device list for a user`_. +It also adds a ``one_time_keys_count`` property. Note the spelling difference +with the ``one_time_key_counts`` property in the |/keys/upload|_ response. + .. todo: generate this from a swagger definition? .. device_lists: { changed: ["@user:server", ... ]}, @@ -364,6 +367,9 @@ Parameter Type Description ============ =========== ===================================================== device_lists DeviceLists Optional. Information on e2e device updates. Note: only present on an incremental sync. +|device_otk| {string: Optional. For each key algorithm, the number of + integer} unclaimed one-time keys currently held on the server + for this device. ============ =========== ===================================================== ``DeviceLists`` @@ -388,6 +394,10 @@ Example response: "@alice:example.com", ], }, + "device_one_time_keys_count": { + "curve25519": 10, + "signed_curve25519": 20 + } } .. References @@ -400,6 +410,7 @@ Example response: .. _`Signing JSON`: ../appendices.html#signing-json .. |m.olm.v1.curve25519-aes-sha2| replace:: ``m.olm.v1.curve25519-aes-sha2`` +.. |device_otk| replace:: device_one_time_keys_count .. |/keys/upload| replace:: ``/keys/upload`` .. _/keys/upload: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-upload From a28f243ed7fe9b4677fc5d970ff4ae494bdc2ba8 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 11 Jun 2018 15:44:17 +0200 Subject: [PATCH 057/166] document left parameter of device_lists in sync response fix #1171 --- specification/modules/end_to_end_encryption.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index b030ab642..3bcfbd2d1 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -379,6 +379,8 @@ Parameter Type Description ========= ========= ============================================= changed [string] List of users who have updated their device identity keys since the previous sync response. +left [string] List of users with whom we do not share any encrypted rooms + anymore since the previous sync response. ========= ========= ============================================= @@ -393,6 +395,9 @@ Example response: "changed": [ "@alice:example.com", ], + "left": [ + "@bob:example.com", + ], }, "device_one_time_keys_count": { "curve25519": 10, From 8274f91b0b121205584bbf53878720ea9b365cc3 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 17:35:38 +0200 Subject: [PATCH 058/166] document device verification This was written by Richard van der Hoff. --- .../modules/end_to_end_encryption.rst | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 3bcfbd2d1..dbd6cb43b 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -228,6 +228,54 @@ A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. +Device verification +------------------- + +Before Alice sends Bob encrypted data, or trusts data received from him, she +may want to verify that she is actually communicating with him, rather than a +man-in-the-middle. This verification process requires an out-of-band channel: +there is no way to do it within Matrix without trusting the administrators of +the homeservers. + +In Matrix, the basic process for device verification is for Alice to verify +that the public Ed25519 signing key she received via ``/keys/query`` for Bob's +device corresponds to the private key in use by Bob's device. For now, it is +recommended that clients provide mechanisms by which the user can see: + +1. The public part of their device's Ed25519 signing key, encoded using + `unpadded Base64`_. + +2. The list of devices in use for each user in a room, along with the public + Ed25519 signing key for each device, again encoded using unpadded Base64. + +Alice can then meet Bob in person, or contact him via some other trusted +medium, and ask him to read out the Ed25519 key shown on his device. She +compares this with the value shown for his device on her client. + +Device verification may reach one of several conclusions. For example: + +* Alice may "accept" the device. This means that she is satisfied that the + device belongs to Bob. She can then encrypt sensitive material for that + device, and knows that messages received were sent from that device. + +* Alice may "reject" the device. She will do this if she knows or suspects + that Bob does not control that device (or equivalently, does not trust + Bob). She will not send sensitive material to that device, and cannot trust + messages apparently received from it. + +* Alice may choose to skip the device verification process. She is not able + to verify that the device actually belongs to Bob, but has no reason to + suspect otherwise. The encryption protocol continues to protect against + passive eavesdroppers. + +.. NOTE:: + + Once the signing key has been verified, it is then up to the encryption + protocol to verify that a given message was sent from a device holding that + Ed25519 private key, or to encrypt a message so that it may only be + decrypted by such a device. For the Olm protocol, this is documented at + https://matrix.org/git/olm/about/docs/signing.rst. + Messaging Algorithms -------------------- From 76071bae988ff3f3dea5c7b6e7b6210f84ebe3c1 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 17:55:54 +0200 Subject: [PATCH 059/166] explain how to verify sender_key ownership --- specification/modules/end_to_end_encryption.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index dbd6cb43b..078b3f99a 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -360,6 +360,9 @@ The plaintext payload is of the form: "type": "", "content": "", "room_id": "", + "keys": { + "ed25519": "" + } } The type and content of the plaintext message event are given in the payload. @@ -367,10 +370,12 @@ The type and content of the plaintext message event are given in the payload. We include the room ID in the payload, because otherwise the homeserver would be able to change the room a message was sent in. -.. TODO: claimed_keys - -Clients must confirm that the ``sender_key`` belongs to the user that sent the -message. TODO: how? +Clients must confirm that the ``sender_key`` and the ``ed25519`` field value +under the ``keys`` property match the keys returned by |/keys/query|_ for +the given user, and must also verify the signature of the payload. Without +this check, a client cannot be sure that the sender device owns the private +part of the ed25519 key it claims to have in the Olm payload. +This is crucial when the ed25519 key corresponds to a verified device. ``m.megolm.v1.aes-sha2`` ~~~~~~~~~~~~~~~~~~~~~~~~ From 669605b24a2e09066ffaafc82ecb701737cfe508 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 19:53:05 +0200 Subject: [PATCH 060/166] add Olm missing properties --- specification/modules/end_to_end_encryption.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 078b3f99a..50a14f729 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -360,6 +360,11 @@ The plaintext payload is of the form: "type": "", "content": "", "room_id": "", + "sender": "", + "recipient": "", + "recipient_keys": { + "ed25519": "" + }, "keys": { "ed25519": "" } @@ -370,6 +375,12 @@ The type and content of the plaintext message event are given in the payload. We include the room ID in the payload, because otherwise the homeserver would be able to change the room a message was sent in. +Other properties are included in order to prevent an attacker from publishing +someone else's curve25519 keys as their own and subsequently claiming to have +sent messages which they didn't. +``sender`` must correspond to the user who sent the event, ``recipient`` to +the local user, and ``recipient_keys`` to the local ed25519 key. + Clients must confirm that the ``sender_key`` and the ``ed25519`` field value under the ``keys`` property match the keys returned by |/keys/query|_ for the given user, and must also verify the signature of the payload. Without From 68b78dc5d8dd11233a8230dd1d34c5f4cd26b17f Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 20:56:37 +0200 Subject: [PATCH 061/166] complete Megolm documentation --- .../modules/end_to_end_encryption.rst | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 50a14f729..5ea84a3f1 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -327,7 +327,7 @@ An event encrypted using Olm has the following format: "ciphertext": { "": { "type": 0, - "body": "" + "body": "" } } } @@ -398,6 +398,53 @@ ratchet, as defined by the `Megolm specification`_. This uses: * HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. * Ed25519 for message authenticity. +Devices that support Megolm must support Olm, and include "m.megolm.v1.aes-sha2" in +their list of supported messaging algorithms. + +An event encrypted using Megolm has the following format: + +.. code:: json + + { + "type": "m.room.encrypted", + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "sender_key": "", + "device_id": "", + "session_id": "", + "ciphertext": "" + } + } + +The encrypted payload can contain any message event. The plaintext is of the form: + +.. code:: json + + { + "type": "", + "content": "", + "room_id": "" + } + +Clients must guard against replay attacks by keeping track of the ratchet indices +of Megolm sessions. They should reject messages with a ratchet index that they +have already decrypted. Care should be taken in order to avoid false positives, as a +client may decrypt the same event twice as part of its normal processing. + +As with Olm events, clients must confirm that the ``sender_key`` belongs to the user +who sent the message. The same reasoning applies, but the sender ed25519 key has to be +inferred from the ``keys.ed25519`` property of the event which established the Megolm +session. + +In order to enable end-to-end encryption in a room, clients can send a +``m.room.encryption`` state event specifying ``m.megolm.v1.aes-sha2`` as its +``algorithm`` property. + +When creating a Megolm session in a room, clients must share the corresponding session +key using Olm with the intended recipients, so that they can decrypt future messages +encrypted using this session. A ``m.room_key`` event is used to do this. Clients +must also handle ``m.room_key`` events sent by other devices in order to decrypt their +messages. Protocol definitions -------------------- From e5005b2d0f6ee756dab5e80df448ce6de9afcc6a Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 20:58:18 +0200 Subject: [PATCH 062/166] document E2E events This was written by Richard van der Hoff. --- event-schemas/examples/m.room.encrypted | 17 ++++++++++ event-schemas/examples/m.room.encryption | 10 ++++++ event-schemas/examples/m.room_key | 9 +++++ event-schemas/schema/m.room.encrypted | 34 +++++++++++++++++++ event-schemas/schema/m.room.encryption | 24 +++++++++++++ event-schemas/schema/m.room_key | 24 +++++++++++++ .../modules/end_to_end_encryption.rst | 12 +++++++ 7 files changed, 130 insertions(+) create mode 100644 event-schemas/examples/m.room.encrypted create mode 100644 event-schemas/examples/m.room.encryption create mode 100644 event-schemas/examples/m.room_key create mode 100644 event-schemas/schema/m.room.encrypted create mode 100644 event-schemas/schema/m.room.encryption create mode 100644 event-schemas/schema/m.room_key diff --git a/event-schemas/examples/m.room.encrypted b/event-schemas/examples/m.room.encrypted new file mode 100644 index 000000000..a0360963d --- /dev/null +++ b/event-schemas/examples/m.room.encrypted @@ -0,0 +1,17 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg2yyOwAtHaZTwyNg37afzg8f3r9IsN9rH4RNFg7MaZencUJe4qvELiDiopUjy5wYVDAtqdBzer5bWRD9ldxp1FLgbQvBcjkkywYjCsmsq6+hArILd9oAQZnGKn/qLsK+5uNX3PaWzDRC9wZPQvWYYPCTov3jCwXKTPsLKIiTrcCXDqMvnn8m+T3zF1/I2zqxg158tnUwWWIw51UO", + "device_id": "RJYKSTBOIE", + "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ" + }, + "event_id": "$WLGTSEFSEF:localhost", + "origin_server_ts": 1476648761524, + "sender": "@example:localhost", + "type": "m.room.encrypted", + "unsigned": { + "age": 158, + "transaction_id": "m1476648745605.19" + } +} diff --git a/event-schemas/examples/m.room.encryption b/event-schemas/examples/m.room.encryption new file mode 100644 index 000000000..25b1271f9 --- /dev/null +++ b/event-schemas/examples/m.room.encryption @@ -0,0 +1,10 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2" + }, + "event_id": "$WLGTSEFJJKJ:localhost", + "origin_server_ts": 1476648761524, + "sender": "@example:localhost", + "state_key": "", + "type": "m.room.encryption" +} diff --git a/event-schemas/examples/m.room_key b/event-schemas/examples/m.room_key new file mode 100644 index 000000000..2348e47d8 --- /dev/null +++ b/event-schemas/examples/m.room_key @@ -0,0 +1,9 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!UCnwUWwIKhcpaPTHtR:sw1v.org", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8LlfJL7qNBEYbf8q5V1G7D/0GHj81JbEFsaE8JOHXJCyIqUGU9svVEi52SAGiC4lpID43TAeGfYc64rUsBx5ovhZl8WrdszLxld29I+7H9e8GZt/NVd/ZQEBBfOv3vrgoODT3WpJiWZ7VEIjL6gspKkkRTDcmwYU6Eff+A11iJ2tEC9njtCeNfTrK7XUIPoXkHWmEjPwqdSQi9pqVb1OYRKT7un7WFJzo0WEw8xjo6wyEolSikaBr3/o8FhoIMIA9KvbjR4y1WDg" + }, + "type": "m.room_key" +} diff --git a/event-schemas/schema/m.room.encrypted b/event-schemas/schema/m.room.encrypted new file mode 100644 index 000000000..cf3e4b4a1 --- /dev/null +++ b/event-schemas/schema/m.room.encrypted @@ -0,0 +1,34 @@ +--- +allOf: + # this is a bit of a lie; if the event is sent as a to-device event it won't + # have the room event fields. We really ought to use different event types :/ + - $ref: core-event-schema/room_event.yaml + +description: |- + This event type is used when sending encrypted events. It can be used either + within a room (in which case it will have all of the `Room Event fields`_), or + as a `to-device`_ event. + +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm used to encrypt this event. The + value of this field determines which other properties will be + present. + ciphertext: + type: + - object + - string + description: |- + Normally required. The encrypted content of the event. + required: + - algorithm + type: object + type: + enum: + - m.room.encrypted + type: string +type: object diff --git a/event-schemas/schema/m.room.encryption b/event-schemas/schema/m.room.encryption new file mode 100644 index 000000000..14778efa4 --- /dev/null +++ b/event-schemas/schema/m.room.encryption @@ -0,0 +1,24 @@ +--- +allOf: + - $ref: core-event-schema/state_event.yaml +description: Defines how messages sent in this room should be encrypted. +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm to be used to encrypt messages sent in this + room. For example, ``m.megolm.v1.aes-sha2``. + required: + - algorithm + type: object + state_key: + description: A zero-length string. + pattern: '^$' + type: string + type: + enum: + - m.room.encryption + type: string +type: object diff --git a/event-schemas/schema/m.room_key b/event-schemas/schema/m.room_key new file mode 100644 index 000000000..f5e4ac296 --- /dev/null +++ b/event-schemas/schema/m.room_key @@ -0,0 +1,24 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to exchange keys for end-to-end encryption. Typically + it is encrypted as an ``m.room.encrypted`` event. +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm the keys in this event are to be used + with. The value of this field determines which other properties will + be present. + required: + - algorithm + type: object + type: + enum: + - m.room_key + type: string +type: object diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 5ea84a3f1..c43f81c4e 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -449,6 +449,18 @@ messages. Protocol definitions -------------------- +Events +~~~~~~ + +{{m_room_encryption_event}} + +{{m_room_encrypted_event}} + +{{m_room_key_event}} + +Key management API +~~~~~~~~~~~~~~~~~~ + {{keys_cs_http_api}} From 3a8d13df602bb97f1c2a580c33eda487c02f0540 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 21:25:41 +0200 Subject: [PATCH 063/166] add missing m.room.encryption properties --- event-schemas/examples/m.room.encryption | 4 +++- event-schemas/schema/m.room.encryption | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/event-schemas/examples/m.room.encryption b/event-schemas/examples/m.room.encryption index 25b1271f9..4c6342bb1 100644 --- a/event-schemas/examples/m.room.encryption +++ b/event-schemas/examples/m.room.encryption @@ -1,6 +1,8 @@ { "content": { - "algorithm": "m.megolm.v1.aes-sha2" + "algorithm": "m.megolm.v1.aes-sha2", + "rotation_period_ms": 604800000, + "rotation_period_msgs": 100 }, "event_id": "$WLGTSEFJJKJ:localhost", "origin_server_ts": 1476648761524, diff --git a/event-schemas/schema/m.room.encryption b/event-schemas/schema/m.room.encryption index 14778efa4..b990a13bf 100644 --- a/event-schemas/schema/m.room.encryption +++ b/event-schemas/schema/m.room.encryption @@ -10,6 +10,16 @@ properties: description: |- The encryption algorithm to be used to encrypt messages sent in this room. For example, ``m.megolm.v1.aes-sha2``. + rotation_period_ms: + type: integer + description: |- + How long the session should be used before changing it. ``604800000`` + (a week) is the recommended default. + rotation_period_msgs: + type: integer + description: |- + How many messages should be sent before changing the session. ``100`` is the + recommended default. required: - algorithm type: object From c60109d235a1bba80f4ee003274484e3cbc8d781 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 31 Jul 2018 18:59:09 +0200 Subject: [PATCH 064/166] complete m.room_key documentation --- event-schemas/examples/m.room_key | 2 +- event-schemas/schema/m.room_key | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/event-schemas/examples/m.room_key b/event-schemas/examples/m.room_key index 2348e47d8..512491996 100644 --- a/event-schemas/examples/m.room_key +++ b/event-schemas/examples/m.room_key @@ -3,7 +3,7 @@ "algorithm": "m.megolm.v1.aes-sha2", "room_id": "!UCnwUWwIKhcpaPTHtR:sw1v.org", "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", - "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8LlfJL7qNBEYbf8q5V1G7D/0GHj81JbEFsaE8JOHXJCyIqUGU9svVEi52SAGiC4lpID43TAeGfYc64rUsBx5ovhZl8WrdszLxld29I+7H9e8GZt/NVd/ZQEBBfOv3vrgoODT3WpJiWZ7VEIjL6gspKkkRTDcmwYU6Eff+A11iJ2tEC9njtCeNfTrK7XUIPoXkHWmEjPwqdSQi9pqVb1OYRKT7un7WFJzo0WEw8xjo6wyEolSikaBr3/o8FhoIMIA9KvbjR4y1WDg" + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8LlfJL7qNBEY..." }, "type": "m.room_key" } diff --git a/event-schemas/schema/m.room_key b/event-schemas/schema/m.room_key index f5e4ac296..ef8c51c08 100644 --- a/event-schemas/schema/m.room_key +++ b/event-schemas/schema/m.room_key @@ -4,18 +4,29 @@ allOf: description: |- This event type is used to exchange keys for end-to-end encryption. Typically - it is encrypted as an ``m.room.encrypted`` event. + it is encrypted as an ``m.room.encrypted`` event, then sent as a `to-device`_ event. properties: content: properties: algorithm: type: string + enum: ["m.megolm.v1.aes-sha2"] description: |- - The encryption algorithm the keys in this event are to be used - with. The value of this field determines which other properties will - be present. + The encryption algorithm the key in this event is to be used with. + room_id: + type: string + description: The room where the key is used. + session_id: + type: string + description: The ID of the session that the key is for. + session_key: + type: string + description: The key to be exchanged. required: - algorithm + - room_id + - session_id + - session_key type: object type: enum: From 8732378da2e734dae5616e496eb932e0ffc991bc Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 22:02:53 +0200 Subject: [PATCH 065/166] add required room ids --- event-schemas/examples/m.room.encrypted | 1 + event-schemas/examples/m.room.encryption | 1 + event-schemas/examples/m.room_key | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/event-schemas/examples/m.room.encrypted b/event-schemas/examples/m.room.encrypted index a0360963d..0e7e677a3 100644 --- a/event-schemas/examples/m.room.encrypted +++ b/event-schemas/examples/m.room.encrypted @@ -7,6 +7,7 @@ "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ" }, "event_id": "$WLGTSEFSEF:localhost", + "room_id": "!Cuyf34gef24t:localhost", "origin_server_ts": 1476648761524, "sender": "@example:localhost", "type": "m.room.encrypted", diff --git a/event-schemas/examples/m.room.encryption b/event-schemas/examples/m.room.encryption index 4c6342bb1..08f152396 100644 --- a/event-schemas/examples/m.room.encryption +++ b/event-schemas/examples/m.room.encryption @@ -7,6 +7,7 @@ "event_id": "$WLGTSEFJJKJ:localhost", "origin_server_ts": 1476648761524, "sender": "@example:localhost", + "room_id": "!Cuyf34gef24t:localhost", "state_key": "", "type": "m.room.encryption" } diff --git a/event-schemas/examples/m.room_key b/event-schemas/examples/m.room_key index 512491996..53f83e522 100644 --- a/event-schemas/examples/m.room_key +++ b/event-schemas/examples/m.room_key @@ -1,7 +1,7 @@ { "content": { "algorithm": "m.megolm.v1.aes-sha2", - "room_id": "!UCnwUWwIKhcpaPTHtR:sw1v.org", + "room_id": "!Cuyf34gef24t:localhost", "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8LlfJL7qNBEY..." }, From 661176cb3a4c4952edd9c9bf022fabbf82bc4a1d Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 17 Jul 2018 22:39:45 +0200 Subject: [PATCH 066/166] Olm m.room.encrypted example --- ...room.encrypted => m.room.encrypted#megolm} | 8 ++---- event-schemas/examples/m.room.encrypted#olm | 14 ++++++++++ event-schemas/schema/m.room.encrypted | 28 ++++++++++++++----- 3 files changed, 37 insertions(+), 13 deletions(-) rename event-schemas/examples/{m.room.encrypted => m.room.encrypted#megolm} (57%) create mode 100644 event-schemas/examples/m.room.encrypted#olm diff --git a/event-schemas/examples/m.room.encrypted b/event-schemas/examples/m.room.encrypted#megolm similarity index 57% rename from event-schemas/examples/m.room.encrypted rename to event-schemas/examples/m.room.encrypted#megolm index 0e7e677a3..1f9b75208 100644 --- a/event-schemas/examples/m.room.encrypted +++ b/event-schemas/examples/m.room.encrypted#megolm @@ -1,7 +1,7 @@ { "content": { "algorithm": "m.megolm.v1.aes-sha2", - "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg2yyOwAtHaZTwyNg37afzg8f3r9IsN9rH4RNFg7MaZencUJe4qvELiDiopUjy5wYVDAtqdBzer5bWRD9ldxp1FLgbQvBcjkkywYjCsmsq6+hArILd9oAQZnGKn/qLsK+5uNX3PaWzDRC9wZPQvWYYPCTov3jCwXKTPsLKIiTrcCXDqMvnn8m+T3zF1/I2zqxg158tnUwWWIw51UO", + "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", "device_id": "RJYKSTBOIE", "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ" @@ -10,9 +10,5 @@ "room_id": "!Cuyf34gef24t:localhost", "origin_server_ts": 1476648761524, "sender": "@example:localhost", - "type": "m.room.encrypted", - "unsigned": { - "age": 158, - "transaction_id": "m1476648745605.19" - } + "type": "m.room.encrypted" } diff --git a/event-schemas/examples/m.room.encrypted#olm b/event-schemas/examples/m.room.encrypted#olm new file mode 100644 index 000000000..abb23c31e --- /dev/null +++ b/event-schemas/examples/m.room.encrypted#olm @@ -0,0 +1,14 @@ +{ + "type": "m.room.encrypted", + "sender": "@example:localhost", + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "Szl29ksW/L8yZGWAX+8dY1XyFi+i5wm+DRhTGkbMiwU", + "ciphertext": { + "7qZcfnBmbEGzxxaWfBjElJuvn7BZx+lSz/SvFrDF/z8": { + "type": 0, + "body": "AwogGJJzMhf/S3GQFXAOrCZ3iKyGU5ZScVtjI0KypTYrW..." + } + } + } +} diff --git a/event-schemas/schema/m.room.encrypted b/event-schemas/schema/m.room.encrypted index cf3e4b4a1..664b10cfb 100644 --- a/event-schemas/schema/m.room.encrypted +++ b/event-schemas/schema/m.room.encrypted @@ -1,8 +1,6 @@ --- allOf: - # this is a bit of a lie; if the event is sent as a to-device event it won't - # have the room event fields. We really ought to use different event types :/ - - $ref: core-event-schema/room_event.yaml + - $ref: core-event-schema/event.yaml description: |- This event type is used when sending encrypted events. It can be used either @@ -14,16 +12,32 @@ properties: properties: algorithm: type: string + enum: + - m.olm.curve25519-aes-sha256 + - m.megolm.v1.aes-sha description: |- The encryption algorithm used to encrypt this event. The value of this field determines which other properties will be present. ciphertext: - type: - - object - - string + oneOf: + - type: string + - type: object + additionalProperties: + type: object + title: CiphertextInfo + properties: + body: + type: string + description: The encrypted payload. + type: + type: integer + description: The Olm message type. description: |- - Normally required. The encrypted content of the event. + The encrypted content of the event. Either the encrypted payload + itself, in the case of a Megolm event, or a map from the recipient + Curve25519 identity key to ciphertext information, in the case of an + Olm event. For more details, see `Messaging Algorithms`_. required: - algorithm type: object From 248786681ec9d446481b2c529f587ace18d4274c Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 19 Jul 2018 19:50:05 +0200 Subject: [PATCH 067/166] fix typo --- specification/modules/end_to_end_encryption.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index c43f81c4e..1bc6a2322 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -159,7 +159,7 @@ It is therefore expected that each client will maintain a list of devices for a number of users (in practice, typically each user with whom we share an encrypted room). Furthermore, it is likely that this list will need to be persisted between invocations of the client application (to preserve device -verification data and to alert Alice if Bob suddently gets a new +verification data and to alert Alice if Bob suddenly gets a new device). Alice's client can maintain a list of Bob's devices via the following From 4e0f107ef7207fde8bd9feafdd3bea88caad89b8 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 19 Jul 2018 19:57:47 +0200 Subject: [PATCH 068/166] document changed field behavior in e2e sync extension --- .../modules/end_to_end_encryption.rst | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 1bc6a2322..194d25677 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -176,9 +176,10 @@ process: flag. #. During its normal processing of responses to |/sync|_, Alice's client - inspects the |device_lists|_ field. If it is tracking the device lists of - any of the listed users, then it marks the device lists for those users - outdated, and initiates another request to |/keys/query|_ for them. + inspects the ``changed`` property of the |device_lists|_ field. If it is + tracking the device lists of any of the listed users, then it marks the + device lists for those users outdated, and initiates another request to + |/keys/query|_ for them. #. Periodically, Alice's client stores the ``next_batch`` field of the result from |/sync|_ in persistent storage. If Alice later restarts her client, it @@ -214,6 +215,18 @@ process: that the first request's results are ignored (possibly by cancelling the request). +.. Note:: + + When Bob and Alice share a room, with Bob tracking Alice's devices, she may leave + the room and then add a new device. Bob will not be notified of this change, + as he doesn't share a room anymore with Alice. When they start sharing a + room again, Bob has an out-of-date list of Alice's devices. In order to address + this issue, Bob's homeserver will add Alice's user ID to the ``changed`` property of + the ``device_lists`` field, thus Bob will update his list of Alice's devices as part + of his normal processing. Note that Bob can also be notified when he stops sharing + any room with Alice by inspecting the ``left`` property of the ``device_lists`` + field, and as a result should remove her from its list of tracked users. + .. |device_lists| replace:: ``device_lists`` .. _`device_lists`: `device_lists_sync`_ @@ -500,12 +513,20 @@ device_lists DeviceLists Optional. Information on e2e device updates. Note: ========= ========= ============================================= Parameter Type Description ========= ========= ============================================= -changed [string] List of users who have updated their device identity keys - since the previous sync response. +changed [string] List of users who have updated their device identity keys, + or who now share an encrypted room with the client since + the previous sync response. left [string] List of users with whom we do not share any encrypted rooms anymore since the previous sync response. ========= ========= ============================================= +.. NOTE:: + + For optimal performance, Alice should be added to ``changed`` in Bob's sync only + when she adds a new device, or when Alice and Bob now share a room but didn't + share any room previously. However, for the sake of simpler logic, a server + may add Alice to ``changed`` when Alice and Bob share a new room, even if they + previously already shared a room. Example response: From eb8ea0e85a9771a0d4fa0d279e6700321763de47 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 26 Jul 2018 10:44:42 +0200 Subject: [PATCH 069/166] remove warning pointing at outdated doc --- specification/modules/end_to_end_encryption.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 194d25677..213d09a5b 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -21,12 +21,6 @@ Matrix optionally supports end-to-end encryption, allowing rooms to be created whose conversation contents is not decryptable or interceptable on any of the participating homeservers. -.. WARNING:: - - End to end encryption is being worked on and will be coming soon. This - section is incomplete. You can read more about what's underway at - http://matrix.org/speculator/spec/drafts%2Fe2e/client_server/unstable.html#end-to-end-encryption. - Key Distribution ---------------- Encryption and Authentication in Matrix is based around public-key From 6c44233c42e79e5f0552f6d3288fec98be1fa146 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 26 Jul 2018 11:38:17 +0200 Subject: [PATCH 070/166] require megolm algorithm in m.room.encryption --- event-schemas/schema/m.room.encryption | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/event-schemas/schema/m.room.encryption b/event-schemas/schema/m.room.encryption index b990a13bf..d7c4d4294 100644 --- a/event-schemas/schema/m.room.encryption +++ b/event-schemas/schema/m.room.encryption @@ -7,9 +7,11 @@ properties: properties: algorithm: type: string + enum: + - "m.megolm.v1.aes-sha2" description: |- The encryption algorithm to be used to encrypt messages sent in this - room. For example, ``m.megolm.v1.aes-sha2``. + room. rotation_period_ms: type: integer description: |- From f853856f2173792d1f9307ec77a460ccde708f35 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 4 Aug 2018 14:59:09 +0200 Subject: [PATCH 071/166] add missing m.room.encrypted event properties --- event-schemas/schema/m.room.encrypted | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/event-schemas/schema/m.room.encrypted b/event-schemas/schema/m.room.encrypted index 664b10cfb..6825be1d2 100644 --- a/event-schemas/schema/m.room.encrypted +++ b/event-schemas/schema/m.room.encrypted @@ -38,8 +38,21 @@ properties: itself, in the case of a Megolm event, or a map from the recipient Curve25519 identity key to ciphertext information, in the case of an Olm event. For more details, see `Messaging Algorithms`_. + sender_key: + type: string + description: The Curve25519 key of the sender. + device_id: + type: string + description: The ID of the sending device. Required with Megolm. + session_id: + type: string + description: |- + The ID of the session used to encrypt the message. Required with + Megolm. required: - algorithm + - sender_key + - ciphertext type: object type: enum: From b2316ba782c098a123c8f51261c56dbb5dee24bf Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 16 Aug 2018 13:00:32 +0200 Subject: [PATCH 072/166] enforce unique namespacing in new algorithms experiments --- specification/modules/end_to_end_encryption.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 213d09a5b..7dfe89938 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -292,8 +292,8 @@ Messaging Algorithm Names Messaging algorithm names use the extensible naming scheme used throughout this specification. Algorithm names that start with ``m.`` are reserved for algorithms defined by this specification. Implementations wanting to experiment -with new algorithms are encouraged to pick algorithm names that start with -their domain to reduce the risk of collisions. +with new algorithms must be uniquely globally namespaced following Java's package +naming conventions. Algorithm names should be short and meaningful, and should list the primitives used by the algorithm so that it is easier to see if the algorithm is using a From 8ba19b51ab36b0fdd62084e8798ddd58c4804e14 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 16 Aug 2018 13:04:29 +0200 Subject: [PATCH 073/166] complete Olm documentation --- specification/modules/end_to_end_encryption.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 7dfe89938..b224126d3 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -357,7 +357,7 @@ client must not persist a session or remove one-time keys used by a session until it has successfully decrypted a message using that session. Messages with type 1 can only be decrypted with an existing session. If there -is no matching session, the client should show this as an invalid message. +is no matching session, the client must treat this as an invalid message. The plaintext payload is of the form: From 9430f2c7f9a2b31de29ed08f8a5260706546cfe9 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Fri, 17 Aug 2018 15:12:14 +0200 Subject: [PATCH 074/166] room ID is included in Megolm plaintext, not Olm --- specification/modules/end_to_end_encryption.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index b224126d3..fa461cc28 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -366,7 +366,6 @@ The plaintext payload is of the form: { "type": "", "content": "", - "room_id": "", "sender": "", "recipient": "", "recipient_keys": { @@ -379,9 +378,6 @@ The plaintext payload is of the form: The type and content of the plaintext message event are given in the payload. -We include the room ID in the payload, because otherwise the homeserver would -be able to change the room a message was sent in. - Other properties are included in order to prevent an attacker from publishing someone else's curve25519 keys as their own and subsequently claiming to have sent messages which they didn't. @@ -433,6 +429,9 @@ The encrypted payload can contain any message event. The plaintext is of the for "room_id": "" } +We include the room ID in the payload, because otherwise the homeserver would +be able to change the room a message was sent in. + Clients must guard against replay attacks by keeping track of the ratchet indices of Megolm sessions. They should reject messages with a ratchet index that they have already decrypted. Care should be taken in order to avoid false positives, as a From 98e2e8de71e27b2394762f647d699ac7af1aa17a Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 18 Aug 2018 11:40:48 +0200 Subject: [PATCH 075/166] changelog --- changelogs/client_server/newsfragments/1284.clarification | 1 + changelogs/client_server/newsfragments/1284.feature.rst | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelogs/client_server/newsfragments/1284.clarification create mode 100644 changelogs/client_server/newsfragments/1284.feature.rst diff --git a/changelogs/client_server/newsfragments/1284.clarification b/changelogs/client_server/newsfragments/1284.clarification new file mode 100644 index 000000000..7bc92f479 --- /dev/null +++ b/changelogs/client_server/newsfragments/1284.clarification @@ -0,0 +1 @@ +Clarify ``changed`` field behaviour in device tracking process diff --git a/changelogs/client_server/newsfragments/1284.feature.rst b/changelogs/client_server/newsfragments/1284.feature.rst new file mode 100644 index 000000000..c658142e8 --- /dev/null +++ b/changelogs/client_server/newsfragments/1284.feature.rst @@ -0,0 +1,7 @@ +End-to-end encryption for group chats: + + - Olm and Megolm messaging algorithms. + - ``m.room.encrypted``, ``m.room.encryption``, ``m.room_key`` events. + - Device verification process. + - ``device_one_time_keys_count`` sync parameter. + - ``device_lists:left`` sync parameter. From 206f78cb48ba5f247a705e6d5e0bc0737c7d7e5f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 Aug 2018 09:21:15 +0100 Subject: [PATCH 076/166] Rename 1284.feature.rst to 1284.feature --- .../newsfragments/{1284.feature.rst => 1284.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelogs/client_server/newsfragments/{1284.feature.rst => 1284.feature} (100%) diff --git a/changelogs/client_server/newsfragments/1284.feature.rst b/changelogs/client_server/newsfragments/1284.feature similarity index 100% rename from changelogs/client_server/newsfragments/1284.feature.rst rename to changelogs/client_server/newsfragments/1284.feature From e712466dca3370e6ca40755e3575da46a5cac87f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 10:44:28 -0600 Subject: [PATCH 077/166] Improve description for currently_active --- .../definitions/event-schemas/m.presence.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index b9c5f754f..7a1dba600 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -62,10 +62,10 @@ allOf: currently_active: type: boolean description: |- - Whether or not the user is currently using a device of theirs. - For example, if the user's ``last_active_ago`` was within the - last few minutes, they may be considered ``currently_active``. - Defaults to false. + True if the user is likely to be interacting with their + client. This may be indicated by the user having a + ``last_active_ago`` within the last few minutes. Defaults + to false. example: true required: ['user_id', 'presence', 'last_active_ago'] required: ['push'] From fca1c0b7f874e079ab2321834e969833ec50d85b Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 20 Aug 2018 18:06:26 +0100 Subject: [PATCH 078/166] Guests should support /context and /event --- specification/modules/guest_access.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index 4e04aa0df..d579da833 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -50,6 +50,8 @@ The following API endpoints are allowed to be accessed by guest accounts for retrieving events: * `GET /rooms/:room_id/state <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-state>`_ +* `GET /rooms/:room_id/context/:event_id <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-context-eventid>`_ +* `GET /rooms/:room_id/event/:event_id <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-event-eventid>`_ * `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-state-eventtype-statekey>`_ * `GET /rooms/:room_id/messages <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-messages>`_ * `GET /rooms/:room_id/initialSync <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-initialsync>`_ From 8b65da1cf6fce5f657a2a46b5c6c8bcc24d32ae3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 11:03:26 -0600 Subject: [PATCH 079/166] Don't try and be fancy about titles --- api/server-server/definitions/event-schemas/m.presence.yaml | 2 +- .../definitions/event-schemas/m.presence_accept.yaml | 2 +- .../definitions/event-schemas/m.presence_deny.yaml | 2 +- .../definitions/event-schemas/m.presence_invite.yaml | 2 +- api/server-server/definitions/event-schemas/m.receipt.yaml | 2 +- api/server-server/definitions/event-schemas/m.typing.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.presence.yaml b/api/server-server/definitions/event-schemas/m.presence.yaml index 7a1dba600..7f47add4d 100644 --- a/api/server-server/definitions/event-schemas/m.presence.yaml +++ b/api/server-server/definitions/event-schemas/m.presence.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.presence`` EDU +title: m.presence description: |- An EDU representing presence updates for users of the sending homeserver. allOf: diff --git a/api/server-server/definitions/event-schemas/m.presence_accept.yaml b/api/server-server/definitions/event-schemas/m.presence_accept.yaml index 4c39989b6..3ba78b477 100644 --- a/api/server-server/definitions/event-schemas/m.presence_accept.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_accept.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.presence_accept`` EDU +title: m.presence_accept description: |- An EDU representing approval for the observing user to subscribe to the presence of the the observed user. diff --git a/api/server-server/definitions/event-schemas/m.presence_deny.yaml b/api/server-server/definitions/event-schemas/m.presence_deny.yaml index 1b9bff7dc..2eb6feec9 100644 --- a/api/server-server/definitions/event-schemas/m.presence_deny.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_deny.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.presence_deny`` EDU +title: m.presence_deny description: |- An EDU representing a declination or revocation for the observing user to subscribe to the presence of the observed user. diff --git a/api/server-server/definitions/event-schemas/m.presence_invite.yaml b/api/server-server/definitions/event-schemas/m.presence_invite.yaml index 4cdc58eba..a584897b7 100644 --- a/api/server-server/definitions/event-schemas/m.presence_invite.yaml +++ b/api/server-server/definitions/event-schemas/m.presence_invite.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.presence_invite`` EDU +title: m.presence_invite description: |- An EDU representing a request to subscribe to a user's presence. allOf: diff --git a/api/server-server/definitions/event-schemas/m.receipt.yaml b/api/server-server/definitions/event-schemas/m.receipt.yaml index 58f47e3a0..7f13ebee3 100644 --- a/api/server-server/definitions/event-schemas/m.receipt.yaml +++ b/api/server-server/definitions/event-schemas/m.receipt.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.receipt`` EDU +title: m.receipt description: |- An EDU representing receipt updates for users of the sending homeserver. When receiving receipts, the server should only update entries that are diff --git a/api/server-server/definitions/event-schemas/m.typing.yaml b/api/server-server/definitions/event-schemas/m.typing.yaml index 34b395297..ccbecf535 100644 --- a/api/server-server/definitions/event-schemas/m.typing.yaml +++ b/api/server-server/definitions/event-schemas/m.typing.yaml @@ -13,7 +13,7 @@ # limitations under the License. type: object -title: ``m.typing`` EDU +title: m.typing description: A typing notification EDU for a user in a room. allOf: - $ref: ../edu.yaml From 4ab64e11afea8ea92fab88b63c8b2d0baec46ab8 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 20 Aug 2018 18:20:20 +0100 Subject: [PATCH 080/166] Add news frag --- changelogs/client_server/newsfragments/1542.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1542.feature diff --git a/changelogs/client_server/newsfragments/1542.feature b/changelogs/client_server/newsfragments/1542.feature new file mode 100644 index 000000000..7e94bdc5a --- /dev/null +++ b/changelogs/client_server/newsfragments/1542.feature @@ -0,0 +1 @@ +Guests can now call /context and /event to fetch events From c8ba2e098c5ec847c6799d37fa0476ff0867dc2c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:09:17 -0600 Subject: [PATCH 081/166] Wording improvements for appservices --- api/application-service/application_service.yaml | 6 +++--- specification/application_service_api.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 64064cbdf..d8f43ed6b 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -33,9 +33,9 @@ paths: This API is called by the homeserver when it wants to push an event (or batch of events) to the application service. - The application service should take care to ensure that it handles - state events by the presence of a ``state_key``, not by the event - type. + Note that the application service should distinguish state events + from message events via the presence of a ``state_key``, rather than + via the event type. operationId: sendTransaction parameters: - in: path diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 5fe0aedeb..5f5029691 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -224,7 +224,7 @@ need to be able to adjust the ``origin_server_ts`` value to do this. Inputs: - Application service token (``as_token``) - - Desired timestamp in milliseconds since the unix epoch + - Desired timestamp (in milliseconds since the unix epoch) Notes: - This will only apply when sending events. From 7d14309b630e91054dc3dca515c03feababb2a6a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:21:19 -0600 Subject: [PATCH 082/166] Move the power level event schema to where it was --- api/client-server/create_room.yaml | 7 +-- event-schemas/power_level_content_schema.yaml | 56 ------------------- event-schemas/schema/m.room.power_levels | 44 ++++++++++++++- 3 files changed, 46 insertions(+), 61 deletions(-) delete mode 100644 event-schemas/power_level_content_schema.yaml diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index be99c4ab1..ef207f07a 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -208,10 +208,9 @@ paths: title: Power Level Event Content description: |- The power level content to override in the default power level - event. This object is applied on top of the generated power - level event prior to it being sent to the room. Defaults - to overriding nothing. - $ref: "definitions/event-schemas/power_level_content_schema.yaml" + event. This object is applied on top of the generated `m.room.power_levels`_ + event content prior to it being sent to the room. Defaults to + overriding nothing. responses: 200: description: Information about the newly created room. diff --git a/event-schemas/power_level_content_schema.yaml b/event-schemas/power_level_content_schema.yaml deleted file mode 100644 index 5859d56ee..000000000 --- a/event-schemas/power_level_content_schema.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -properties: - ban: - description: The level required to ban a user. Defaults to 50 if unspecified. - type: number - events: - additionalProperties: - type: number - description: The level required to send specific event types. This is a mapping from event type to power level required. - title: Event power levels - type: object - events_default: - description: |- - The default level required to send message events. Can be - overridden by the ``events`` key. Defaults to 0 if unspecified. - type: number - invite: - description: The level required to invite a user. Defaults to 50 if unspecified. - type: number - kick: - description: The level required to kick a user. Defaults to 50 if unspecified. - type: number - redact: - description: The level required to redact an event. Defaults to 50 if unspecified. - type: number - state_default: - description: |- - The default level required to send state events. Can be overridden - by the ``events`` key. Defaults to 50 if unspecified, but 0 if - there is no ``m.room.power_levels`` event at all. - type: number - users: - additionalProperties: - type: number - description: The power levels for specific users. This is a mapping from ``user_id`` to power level for that user. - title: User power levels - type: object - users_default: - description: |- - The default power level for every user in the room, unless their - ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if - unspecified. - type: number -type: object \ No newline at end of file diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index 174ada820..13a44c709 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -43,7 +43,49 @@ description: |- properties: content: - $ref: "../power_level_content_schema.yaml" + properties: + ban: + description: The level required to ban a user. Defaults to 50 if unspecified. + type: number + events: + additionalProperties: + type: number + description: The level required to send specific event types. This is a mapping from event type to power level required. + title: Event power levels + type: object + events_default: + description: |- + The default level required to send message events. Can be + overridden by the ``events`` key. Defaults to 0 if unspecified. + type: number + invite: + description: The level required to invite a user. Defaults to 50 if unspecified. + type: number + kick: + description: The level required to kick a user. Defaults to 50 if unspecified. + type: number + redact: + description: The level required to redact an event. Defaults to 50 if unspecified. + type: number + state_default: + description: |- + The default level required to send state events. Can be overridden + by the ``events`` key. Defaults to 50 if unspecified, but 0 if + there is no ``m.room.power_levels`` event at all. + type: number + users: + additionalProperties: + type: number + description: The power levels for specific users. This is a mapping from ``user_id`` to power level for that user. + title: User power levels + type: object + users_default: + description: |- + The default power level for every user in the room, unless their + ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if + unspecified. + type: number + type: object state_key: description: A zero-length string. pattern: '^$' From 6c7a93d2f517b99cad4cf5e5dd734b3ce181d96a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:21:43 -0600 Subject: [PATCH 083/166] Move description about which preset to use when none is specified --- api/client-server/create_room.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index ef207f07a..48325cf91 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -96,11 +96,6 @@ paths: ``private`` visibility if this key is not included. NB: This should not be confused with ``join_rules`` which also uses the word ``public``. - - If no ``preset`` is specificed, the server may use the visbility - to determine which preset to use. A visbility of ``public`` - equates to a preset of ``public_chat`` and ``private`` visibility - equates to a preset of ``private_chat``. room_alias_name: type: string description: |- @@ -198,6 +193,11 @@ paths: description: |- Convenience parameter for setting various default state events based on a preset. + + If unspecified, the server should use the ``visibility`` to determine + which preset to use. A visbility of ``public`` equates to a preset of + ``public_chat`` and ``private`` visibility equates to a preset of + ``private_chat``. is_direct: type: boolean description: |- From 1fd7c9946103665d9b87934b437cba45c8894287 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:21:55 -0600 Subject: [PATCH 084/166] Clarify that the creation event can have other keys in it --- api/client-server/create_room.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 48325cf91..576fc4c08 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -151,18 +151,10 @@ paths: title: CreationContent type: object description: |- - Extra keys to be added to the content of the ``m.room.create``. - The server will clobber the following keys: ``creator``. Future - versions of the specification may allow the server to clobber - other keys. - properties: - "m.federate": - type: boolean - description: |- - Whether users on other servers can join this room. Defaults - to ``true`` or what the server specifies. Setting this to - ``false`` may prevent users from being invited as part of - this room creation request if they reside on other servers. + Extra keys, such as ``m.federate``, to be added to the content + of the `m.room.create`_ event. The server will clobber the following + keys: ``creator``. Future versions of the specification may allow + the server to clobber other keys. initial_state: type: array description: |- From 7e6ca5fef8b267dc2c3e22b92172aece36ac5c07 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:23:17 -0600 Subject: [PATCH 085/166] Take out the room_alias response field The argument is that this isn't really needed at this time. --- api/client-server/create_room.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 576fc4c08..cb0df80ce 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -214,16 +214,10 @@ paths: type: string description: |- The created room's ID. - room_alias: - type: string - description: |- - The complete room alias for the room, if a room alias was created - for the room. required: ['room_id'] examples: application/json: { - "room_id": "!sefiuhWgwghwWgh:example.com", - "room_alias": "#thepub:example.com" + "room_id": "!sefiuhWgwghwWgh:example.com" } 400: description: |- From 8dc6f092efe90b9e5562f0bd1ac7cc196930a2ea Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:27:29 -0600 Subject: [PATCH 086/166] Define the type of the power_level_content_override field --- api/client-server/create_room.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index cb0df80ce..f9aec5197 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -198,6 +198,7 @@ paths: ``invite_3pid``. See `Direct Messaging`_ for more information. power_level_content_override: title: Power Level Event Content + type: object description: |- The power level content to override in the default power level event. This object is applied on top of the generated `m.room.power_levels`_ From 13a1628f59f8279d1a5889830a3514a6fec9c3a9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:34:52 -0600 Subject: [PATCH 087/166] Improve wording about how masquerading works --- specification/application_service_api.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 7119bd5a6..4a29a1d88 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -185,18 +185,17 @@ homeserver. Identity assertion ++++++++++++++++++ The client-server API infers the user ID from the ``access_token`` provided in -every request. It would be an annoying amount of book-keeping to maintain tokens -for every virtual user. It would be preferable if the application service could -use the CS API with its own ``as_token`` instead, and specify the virtual user -they wish to be acting on behalf of. For real users, this would require -additional permissions granting the AS permission to masquerade as a matrix user. +every request. To avoid the application service from having to keep track of each +user's access token, the application service should identify itself to the Client-Server +API by providing its ``as_token`` instead for the ``access_token`` alongside the +user the application service would like to masquerade as. Inputs: - Application service token (``as_token``) - User ID in the AS namespace to act as. Notes: - - This will apply on all aspects of the Client-Server API, except for Account Management. + - This applies to all aspects of the Client-Server API, except for Account Management. - The ``as_token`` is inserted into ``access_token`` which is usually where the client token is, such as via the query string or ``Authorization`` header. This is done on purpose to allow application services to reuse client SDKs. From a320c58e42664bbd558f0177bce7c4fa9d1c4cef Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 12:36:26 -0600 Subject: [PATCH 088/166] A path parameter is obviously URL encoded --- api/application-service/query_room.yaml | 2 +- api/application-service/query_user.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index 7a5a7e54a..b885cb860 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -40,7 +40,7 @@ paths: - in: path name: roomAlias type: string - description: The URL encoded room alias being queried. + description: The room alias being queried. required: true x-example: "#magicforest:example.com" responses: diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 4d1e9f44b..0431b5e49 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -40,7 +40,7 @@ paths: - in: path name: userId type: string - description: The URL encoded user ID being queried. + description: The user ID being queried. required: true x-example: "@alice:example.com" responses: From e500e2502a176fee35315753edf0eb688456ba92 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 21 Aug 2018 09:38:01 -0600 Subject: [PATCH 089/166] Document the maximum value for depth Implements the proposal for https://github.com/matrix-org/matrix-doc/issues/1230 --- api/server-server/definitions/unsigned_pdu.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index ab2812244..64991d227 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -78,7 +78,10 @@ properties: required: ['sha256'] depth: type: integer - description: The maximum depth of the ``prev_events``, plus one. + description: |- + The maximum depth of the ``prev_events``, plus one. Must be less than the + maximum value for an integer (2^63 - 1). If the room's depth is already at + the limit, the depth must be set to the limit. example: 12 auth_events: type: array From 339a2748e806a8a42925afd0384521cafbd57d2d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 21 Aug 2018 09:49:41 -0600 Subject: [PATCH 090/166] Take out groups for now; Move namespace mention Groups aren't landing in the spec yet, so we shouldn't include them yet. --- specification/application_service_api.rst | 31 +++++------------------ 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 9e84339f6..258f80c0d 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -75,8 +75,7 @@ said to be interested in a given event if one of the application service's names users is the target of the event, or is a joined member of the room where the event occurred. -An application -service can also state whether they should be the only ones who +An application service can also state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" namespace. An exclusive namespace prevents humans and other application services from creating/deleting entities in that namespace. Typically, @@ -91,7 +90,6 @@ regular expressions and look like: users: - exclusive: true regex: "@_irc.freenode.net_.*" - group_id: "+irc:matrix.org" Application services may define the following namespaces (with none being explicitly required): @@ -115,22 +113,11 @@ Each individual namespace MUST declare the following fields: | regex | **Required** A regular expression defining which values this namespace includes. | +------------------+-----------------------------------------------------------------------------------------------------------------------------------+ -An application service's users and regex field MUST begin with an underscore (``_``), in -order to provide a visually clear distinction between AS users and regular -users. An optional ``group_id`` field may be added to the ``users`` namespace: - -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Name | Description | -+==================+============================================================================================================================================================================================================================================================================+ -| group_id | An existing group that all matching user IDs will be considered a part of. Users who are joined to this group through an application service are not to be listed when querying for the group's members, however the group should be listed when querying a user's groups. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -.. WARNING:: - - Users that are matched by ``group_id`` should not be publicly listed by - Homeservers. The intention is to differentiate users, perhaps with a flair, - rather than having a list of people to spam. - +Exclusive user and alias namespaces should begin with an underscore after the +sigil to avoid collisions with other users on the homeserver. Application +services should additionally attempt to identify the service they represent +in the reserved namespace. For example, ``@_irc_.*`` would be a good namespace +to register for an application service which deals with IRC. The registration is represented by a series of key-value pairs, which this specification will present as YAML. See below for the possible options along @@ -174,12 +161,6 @@ An example registration file for an IRC-bridging application service is below: regex: "#_irc_bridge_.*" rooms: [] -Exclusive user and alias namespaces should begin with an underscore after the -sigil to avoid collisions with other users on the homeserver. Application -services should additionally attempt to identify the service they represent -in the reserved namespace. For example, ``@_irc_.*`` would be a good namespace -to register for an application service which deals with IRC. - .. WARNING:: If the homeserver in question has multiple application services, each ``as_token`` and ``id`` MUST be unique per application service as these are From 389fa87e6e65a39b50748817d55d4c71f2f9e965 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 21 Aug 2018 12:10:40 -0600 Subject: [PATCH 091/166] English --- specification/application_service_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 4a29a1d88..502472a01 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -187,8 +187,8 @@ Identity assertion The client-server API infers the user ID from the ``access_token`` provided in every request. To avoid the application service from having to keep track of each user's access token, the application service should identify itself to the Client-Server -API by providing its ``as_token`` instead for the ``access_token`` alongside the -user the application service would like to masquerade as. +API by providing its ``as_token`` for the ``access_token`` alongside the user the +application service would like to masquerade as. Inputs: - Application service token (``as_token``) From 1102fc59b2dd248c5f53cb8fbf08b7577928165d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 21 Aug 2018 12:13:19 -0600 Subject: [PATCH 092/166] Spelling --- specification/application_service_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 87f15b007..602aaa920 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -286,7 +286,7 @@ Using ``/sync`` and ``/events`` Application services wishing to use ``/sync`` or ``/events`` from the Client-Server API MUST do so with a virtual user (provide a ``user_id`` via the query string). It -is expectected that the application service use the transactions pushed to it to +is expected that the application service use the transactions pushed to it to handle events rather than syncing with the user implied by ``sender_localpart``. Event fields From 9835c98544249c65612072f1030908001e266687 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 21 Aug 2018 19:27:48 -0600 Subject: [PATCH 093/166] Document how mentions (pills) work Implements the proposal over at https://github.com/matrix-org/matrix-doc/issues/1067 Includes some specification for how matrix.to is structured, and how it is intended to be replaced. --- .../appendices/identifier_grammar.rst | 32 +++++++- specification/modules/mentions.rst | 73 +++++++++++++++++++ specification/targets.yaml | 1 + 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 specification/modules/mentions.rst diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index 7156c7d57..013afa799 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -1,5 +1,5 @@ .. Copyright 2016 Openmarket Ltd. -.. Copyright 2017 New Vector Ltd. +.. Copyright 2017, 2018 New Vector Ltd. .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -252,3 +252,33 @@ domain). .. TODO-spec - Need to specify precise grammar for Room Aliases. https://matrix.org/jira/browse/SPEC-391 + +matrix.to navigation +++++++++++++++++++++ + +.. NOTE: + This namespacing is in place pending a ``matrix://`` (or similar) URI scheme. + +Rooms, users, aliases, and groups may be represented as a "matrix.to" URI. +This URI can be used to reference particular objects in a given context, such +as mentioning a user in a message or linking someone to a particular point +in the room's history (a permalink). + +A matrix.to URI has the following format, based upon the specification defined +in RFC 3986: + + https://matrix.to/#// + +The identifier may be a room ID, room alias, user ID, or group ID. The extra +parameter is only used in the case of permalinks where an event ID is referenced. +The matrix.to URI, when referenced, must always start with ``https://matrix.to/#/`` +followed by the identifier. + +Examples of matrix.to URIs are: + +* Room: ``https://matrix.to/#/!somewhere:domain.com`` +* Room alias: ``https://matrix.to/#/#somewhere:domain.com`` +* Permalink by room: ``https://matrix.to/#/!somewhere:domain.com/$event:example.org`` +* Permalink by room alias: ``https://matrix.to/#/#somewhere:domain.com/$event:example.org`` +* User: ``https://matrix.to/#/@alice:example.org`` +* Group: ``https://matrix.to/#/+example:domain.com`` diff --git a/specification/modules/mentions.rst b/specification/modules/mentions.rst new file mode 100644 index 000000000..e7483ae4f --- /dev/null +++ b/specification/modules/mentions.rst @@ -0,0 +1,73 @@ +.. Copyright 2018 New Vector Ltd. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +User, room, and group mentions +============================== + +.. _module:mentions: + +This module allows users to mention other users, rooms, and groups within +a room message. This is achieved by including a `matrix.to URI`_ in the HTML +body of an `m.room.message`_ event. This module does not have any server-specific +behaviour to it. + +Mentions apply only to `m.room.message`_ events where the ``msgtype`` is ``m.text``, +``m.emote``, or ``m.notice``. The ``format`` for the event must be ``org.matrix.custom.html`` +and therefore requires a ``formatted_body``. + +To make a mention, reference the entity being mentioned in the ``formatted_body`` +using an anchor, like so:: + + { + "body": "Hello Alice!", + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": "Hello Alice!" + } + + +Client behaviour +---------------- + +In addition to using the appropriate ``matrix.to URI`` for the mention, +clients should use the following guidelines when making mentions: + +* When mentioning users, use the user's potentially ambigious display name for + the anchor's text. If the user does not have a display name, use the user's + ID. + +* When mentioning rooms, use the canonical alias for the room. If the room + does not have a canonical alias, prefer one of the aliases listed on the + room. If no alias can be found, fall back to the room ID. In all cases, + use the alias/room ID being linked to as the anchor's text. + +* When referencing groups, use the group ID as the anchor's text. + +The text component of the anchor should be used in the event's ``body`` where +the mention would normally be represented, as shown in the example above. + +Clients should display mentions differently from other elements. For example, +this may be done by changing the background color of the mention to indicate +that it is different from a normal link. + +If the current user is mentioned in a message (either by a mention as defined +in this module or by a push rule), the client should show that mention differently +from other mentions, such as by using a red background color to signify to the +user that they were mentioned. + +When clicked, the mention should navigate the user to the appropriate room, group, +or user information. + + +.. _`matrix.to URI`: ../appendices.html#matrix-to-navigation \ No newline at end of file diff --git a/specification/targets.yaml b/specification/targets.yaml index 5480bbc5a..ced204366 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -67,6 +67,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/report_content.rst - modules/third_party_networks.rst - modules/openid.rst + - modules/mentions.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From ef41b5c2bf5acf349493279ce19851de7542d04e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 12:48:37 -0600 Subject: [PATCH 094/166] Server ACLs Implements the proposal for https://github.com/matrix-org/matrix-doc/issues/1383 --- event-schemas/examples/m.room.server_acl | 14 +++++ event-schemas/schema/m.room.server_acl | 69 ++++++++++++++++++++++++ specification/modules/server_acls.rst | 66 +++++++++++++++++++++++ specification/server_server_api.rst | 26 +++++++++ specification/targets.yaml | 1 + 5 files changed, 176 insertions(+) create mode 100644 event-schemas/examples/m.room.server_acl create mode 100644 event-schemas/schema/m.room.server_acl create mode 100644 specification/modules/server_acls.rst diff --git a/event-schemas/examples/m.room.server_acl b/event-schemas/examples/m.room.server_acl new file mode 100644 index 000000000..86da20931 --- /dev/null +++ b/event-schemas/examples/m.room.server_acl @@ -0,0 +1,14 @@ +{ + "age": 242352, + "content": { + "allow_ip_literals": false, + "allow": ["*"], + "deny": ["*.evil.com", "evil.com"] + }, + "state_key": "", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.server_acl", + "room_id": "!Cuyf34gef24t:localhost", + "sender": "@example:localhost" +} diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl new file mode 100644 index 000000000..ed64038ca --- /dev/null +++ b/event-schemas/schema/m.room.server_acl @@ -0,0 +1,69 @@ +--- +title: Server ACL +description: |- + An event to indicate which servers are permitted to participate in the + room. Server ACLs may allow or deny groups of hosts. All servers participating + in the room, including those that are denied, are expected to uphold the + server ACL. Servers that do not uphold the ACLs are recommended to be + added to the denied hosts list. + + The ``allow`` and ``deny`` lists are lists of globs supporting ``?`` and ``*`` + as wildcards. When comparing against the server ACLs, the suspect server's port + number must not be considered. Therefore ``evil.com``, ``evil.com:8448``, and + ``evil.com:1234`` would all match rules that apply to ``evil.com``, for example. + + The ACLs are applied to servers when they make requests, and are applied in + the following order: + + 1. If there is no ``m.room.server_acl`` event in the room state, allow. + #. If the server name is an IP address (v4 or v6) literal, and ``allow_ip_literals`` + is present and ``false``, deny. + #. If the server name matches an entry in the ``deny`` list, deny. + #. If the server name matches an entry in the ``allow`` list, allow. + #. Otherwise, deny. + + .. WARNING:: + Failing to provide an ``allow`` rule of some kind will prevent **all** + servers from participating in the room, including the sender. This renders + the room unusable. A common allow rule is ``[ "*" ]`` which would still + permit the use of the ``deny`` list without losing the room. +allOf: + - $ref: core-event-schema/state_event.yaml +type: object +properties: + content: + properties: + allow_ip_literals: + type: boolean + description: |- + True to allow server names that are IP address literals. False to + deny. Defaults to true if missing or otherwise not a boolean. + allow: + type: array + description: |- + The server names to allow in the room, excluding any port information. + Wildcards may be used to cover a wider range of hosts, where ``*`` + matches zero or more characters and ``?`` matches one or more characters. + + **This defaults to an empty list when not provided, effectively disallowing + every server.** + items: + type: string + deny: + type: array + description: |- + The server names to disallow in the room, excluding any port information. + Wildcards may be used to cover a wider range of hosts, where ``*`` + matches zero or more characters and ``?`` matches one or more characters. + + This defaults to an empty list when not provided. + items: + type: string + type: object + state_key: + description: A zero-length string. + pattern: '^$' + type: string + type: + enum: ['m.room.server_acl'] + type: enum diff --git a/specification/modules/server_acls.rst b/specification/modules/server_acls.rst new file mode 100644 index 000000000..72892fce5 --- /dev/null +++ b/specification/modules/server_acls.rst @@ -0,0 +1,66 @@ +.. Copyright 2018 New Vector Ltd +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +Server Access Control Lists (ACLs) for rooms +============================================ + +.. _module:server-acls: + +In some scenarios room operators may wish to prevent a malicous or untrusted +server from participating in their room. Sending an `m.room.server_acl`_ state +event into a room is an effective way to prevent the server from participating +in the room at the federation level. + +Server ACLs can also be used to make rooms only federate with a limited set of +servers, or retroactively make the room no longer federate with any other server, +similar to setting the ``m.federate`` value on the `m.room.create`_ event. + +{{m_room_server_acl_event}} + +.. Note:: + Port numbers are not supported because it is unclear to parsers whether a + port number should be matched or an IP address literal. + +.. Note:: + CIDR notation is not supported for IP addresses because Matrix does not + encourage the use of IPs for identifying servers. Instead, a blanket + ``allow_ip_literals`` is provided to cover banning them. + +Client behaviour +---------------- +Clients are not expected to perform any additional duties beyond sending the +event. Clients should describe changes to the server ACLs to the user in the +user interface, such as in the timeline. + +Clients may wish to kick affected users from the room prior to denying a server +access to the room to help prevent those servers from participating. + +Server behaviour +---------------- +Servers MUST prevent blacklisted servers from sending events or participating +in the room when an `m.room.server_acl`_ event is present in the room state. +Which APIs are specifically affected are described in the Server-Server API +specification. + +Servers should still send events to denied servers if they are still residents +of the room. + + +Security considerations +----------------------- +Server ACLs are only effective if every server in the room honours them. Servers +that do not honour the ACLs may still permit events sent by denied servers into +the room, leaking them to other servers in the room. To effectively enforce an +ACL in a room, the servers that do not honour the ACLs should be denied in the +room as well. \ No newline at end of file diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index dbde8b104..439b35f90 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -929,6 +929,32 @@ described in the `Client-Server API`_, being sure to use the ``allow_remote`` parameter (set to ``false``). +Server Access Control Lists (ACLs) +---------------------------------- + +Server ACLs and their purpose are described in the `Server ACLs`_ section of the +Client-Server API. + +When a remote server makes a request, it MUST be verified to be allowed by the +server ACLs. If the server is denied access to a room, the receiving server +MUST reply with a 403 HTTP status code and an ``errcode`` of ``M_FORBIDDEN``. + +The following endpoint prefixes MUST be protected: + +* ``/_matrix/federation/v1/send`` (on a per-PDU basis) +* ``/_matrix/federation/v1/make_join`` +* ``/_matrix/federation/v1/make_leave`` +* ``/_matrix/federation/v1/send_join`` +* ``/_matrix/federation/v1/send_leave`` +* ``/_matrix/federation/v1/invite`` +* ``/_matrix/federation/v1/state`` +* ``/_matrix/federation/v1/state_ids`` +* ``/_matrix/federation/v1/backfill`` +* ``/_matrix/federation/v1/event_auth`` +* ``/_matrix/federation/v1/query_auth`` +* ``/_matrix/federation/v1/get_missing_events`` + + Signing Events -------------- diff --git a/specification/targets.yaml b/specification/targets.yaml index 5480bbc5a..acf4b6ac9 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -67,6 +67,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/report_content.rst - modules/third_party_networks.rst - modules/openid.rst + - modules/server_acls.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From 7ec3cc4343a30c41176c50bb208d57fc698663a5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 15:21:21 -0600 Subject: [PATCH 095/166] General improvements to the push rules module This commit does a few things: * Add 3 undocumented push rules to the spec for encrypted events and at-room notifications. * Require unrecognized conditions to not match, ensuring that future conditions do not cause clients to accidentally notify users. * Clarify that push rules should be enabled when created. * Document a new condition required for at-room notifications. Fixes https://github.com/matrix-org/matrix-doc/issues/1163 Fixes https://github.com/matrix-org/matrix-doc/issues/1034 Fixes https://github.com/matrix-org/matrix-doc/issues/676 Fixes https://github.com/matrix-org/matrix-doc/issues/1033 Relates to https://github.com/matrix-org/matrix-doc/issues/1101 --- api/client-server/pushrules.yaml | 4 +- specification/modules/push.rst | 118 ++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/api/client-server/pushrules.yaml b/api/client-server/pushrules.yaml index ceb9954b1..e23c9189e 100644 --- a/api/client-server/pushrules.yaml +++ b/api/client-server/pushrules.yaml @@ -343,6 +343,8 @@ paths: This endpoint allows the creation, modification and deletion of pushers for this user ID. The behaviour of this endpoint varies depending on the values in the JSON body. + + When creating push rules, they MUST be enabled by default. operationId: setPushRule security: - accessToken: [] @@ -424,7 +426,7 @@ paths: required: ["actions"] responses: 200: - description: The pusher was set. + description: The push rule was created/updated. examples: application/json: { } diff --git a/specification/modules/push.rst b/specification/modules/push.rst index e9ee8c90d..946834362 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -124,7 +124,7 @@ There are different "kinds" of push rules and each rule has an associated priority. Every push rule MUST have a ``kind`` and ``rule_id``. The ``rule_id`` is a unique string within the kind of rule and its' scope: ``rule_ids`` do not need to be unique between rules of the same kind on different devices. Rules may -have extra keys depending on the value of ``kind``.The different kinds of rule +have extra keys depending on the value of ``kind``. The different kinds of rule in descending order of priority are: Override Rules ``override`` @@ -369,6 +369,41 @@ Definition: } +``.m.rule.roomnotif`` +````````````````````````````````` +Matches any message whose content is unencrypted and contains the +text ``@room``, signifying the whole room should be notified of +the event. + +Definition: + +.. code:: json + + { + "rule_id": ".m.rule.roomnotif", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "content.body", + "pattern": "@room" + }, + { + "kind": "sender_notification_permission", + "key": "room" + } + ], + "actions": [ + "notify", + { + "set_tweak": "highlight", + "value": true + } + ] + } + + Default Content Rules ^^^^^^^^^^^^^^^^^^^^^ @@ -428,7 +463,38 @@ Definition: "value": false } ] - }, + } + +``.m.rule.encrypted_room_one_to_one`` +``````````````````````````` +Matches any encrypted event sent in a room with exactly two members. + +Definition: + +.. code:: json + + { + "rule_id": ".m.rule.encrypted_room_one_to_one", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "room_member_count", + "is": "2" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight", + "value": false + } + ] + } ``.m.rule.room_one_to_one`` ``````````````````````````` @@ -446,6 +512,11 @@ Definition: { "kind": "room_member_count", "is": "2" + }, + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.encrypted" } ], "actions": [ @@ -489,6 +560,34 @@ Definition: ] } +``.m.rule.encrypted`` +``````````````````` +Matches all encrypted events. + +Definition: + +.. code:: json + + { + "rule_id": ".m.rule.encrypted", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.encrypted" + } + ], + "actions": [ + "notify", + { + "set_tweak": "highlight", + "value": false + } + ] + } + Conditions ++++++++++ @@ -523,6 +622,21 @@ rule determines its behaviour. The following conditions are defined: count is strictly less than the given number and so forth. If no prefix is present, this parameter defaults to ``==``. +``sender_notification_permission`` + This takes into account the current power levels in the room, ensuring the + sender of the event has high enough power to trigger the notification. + + Parameters: + + * ``key``: The notification power level to require the sender to have. Refer to + the `m.room.power_levels`_ event schema for information about what the defaults + are and how to interpret the event. The ``key`` is used to look up a specific + notification type from the ``notifications`` object in the power level event + content. + +Unrecognised conditions MUST NOT match any events, effectively making the push +rule disabled. + Push Rules: API ~~~~~~~~~~~~~~~ From ab0be0457199e65d115ed3eb6fd46ba94f591d1a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 15:24:53 -0600 Subject: [PATCH 096/166] Fix titles --- specification/modules/push.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 946834362..408eabe62 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -370,7 +370,7 @@ Definition: ``.m.rule.roomnotif`` -````````````````````````````````` +````````````````````` Matches any message whose content is unencrypted and contains the text ``@room``, signifying the whole room should be notified of the event. @@ -466,7 +466,7 @@ Definition: } ``.m.rule.encrypted_room_one_to_one`` -``````````````````````````` +````````````````````````````````````` Matches any encrypted event sent in a room with exactly two members. Definition: @@ -561,7 +561,7 @@ Definition: } ``.m.rule.encrypted`` -``````````````````` +````````````````````` Matches all encrypted events. Definition: From bce324818b8e1a41e1ee313f6944ca3e03d27d40 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 15:26:27 -0600 Subject: [PATCH 097/166] Changelog --- changelogs/client_server/newsfragments/1551.clarification | 1 + changelogs/client_server/newsfragments/1551.feature | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelogs/client_server/newsfragments/1551.clarification create mode 100644 changelogs/client_server/newsfragments/1551.feature diff --git a/changelogs/client_server/newsfragments/1551.clarification b/changelogs/client_server/newsfragments/1551.clarification new file mode 100644 index 000000000..06eac4daa --- /dev/null +++ b/changelogs/client_server/newsfragments/1551.clarification @@ -0,0 +1 @@ +Clarify that new push rules should be enabled by default, and that unrecognised conditions should not match. diff --git a/changelogs/client_server/newsfragments/1551.feature b/changelogs/client_server/newsfragments/1551.feature new file mode 100644 index 000000000..dfce0f0a4 --- /dev/null +++ b/changelogs/client_server/newsfragments/1551.feature @@ -0,0 +1 @@ +Add new push rules for encrypted events and ``@room`` notifications. From a95d7092eb3bf4e02cbd1aabc31548462d6376c2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 15:27:48 -0600 Subject: [PATCH 098/166] Changelog --- changelogs/client_server/newsfragments/1550.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1550.feature diff --git a/changelogs/client_server/newsfragments/1550.feature b/changelogs/client_server/newsfragments/1550.feature new file mode 100644 index 000000000..f04fa9ae0 --- /dev/null +++ b/changelogs/client_server/newsfragments/1550.feature @@ -0,0 +1 @@ +Add server ACLs as an option for controlling federation in a room. From dc94820450bcbd907e99bbae99e7c5b1df5380d5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 22 Aug 2018 15:28:55 -0600 Subject: [PATCH 099/166] Changelog --- changelogs/client_server/newsfragments/1547.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1547.feature diff --git a/changelogs/client_server/newsfragments/1547.feature b/changelogs/client_server/newsfragments/1547.feature new file mode 100644 index 000000000..76346f230 --- /dev/null +++ b/changelogs/client_server/newsfragments/1547.feature @@ -0,0 +1 @@ +Add a common standard for user, room, and group mentions in messages. From bbd33c146131939790e6939e881bfd46b971bd62 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 23 Aug 2018 13:29:04 -0600 Subject: [PATCH 100/166] Move appservice transaction API to the right section Part of https://github.com/matrix-org/matrix-doc/issues/1532 --- .../application_service.yaml | 70 ------------- api/application-service/transactions.yaml | 98 +++++++++++++++++++ specification/application_service_api.rst | 2 + 3 files changed, 100 insertions(+), 70 deletions(-) create mode 100644 api/application-service/transactions.yaml diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index bc6e45d5d..8fcfa0066 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -26,76 +26,6 @@ consumes: produces: - application/json paths: - "/transactions/{txnId}": - put: - summary: Send some events to the application service. - description: |- - This API is called by the homeserver when it wants to push an event - (or batch of events) to the application service. - - Note that the application service should distinguish state events - from message events via the presence of a ``state_key``, rather than - via the event type. - operationId: sendTransaction - parameters: - - in: path - name: txnId - type: string - description: |- - The transaction ID for this set of events. Homeservers generate - these IDs and they are used to ensure idempotency of requests. - required: true - x-example: "35" - - in: body - name: body - description: A list of events. - schema: - type: object - example: { - "events": [ - { - "age": 32, - "content": { - "body": "incoming message", - "msgtype": "m.text" - }, - "event_id": "$14328055551tzaee:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - }, - { - "age": 1984, - "content": { - "body": "another incoming message", - "msgtype": "m.text" - }, - "event_id": "$1228055551ffsef:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - } - ] - } - description: "Transaction informations" - properties: - events: - type: array - description: A list of events - items: - type: object - title: Event - required: ["events"] - responses: - 200: - description: The transaction was processed successfully. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml new file mode 100644 index 000000000..9388d3c5b --- /dev/null +++ b/api/application-service/transactions.yaml @@ -0,0 +1,98 @@ +# Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "/" +consumes: + - application/json +produces: + - application/json +paths: + "/transactions/{txnId}": + put: + summary: Send some events to the application service. + description: |- + This API is called by the homeserver when it wants to push an event + (or batch of events) to the application service. + + Note that the application service should distinguish state events + from message events via the presence of a ``state_key``, rather than + via the event type. + operationId: sendTransaction + parameters: + - in: path + name: txnId + type: string + description: |- + The transaction ID for this set of events. Homeservers generate + these IDs and they are used to ensure idempotency of requests. + required: true + x-example: "35" + - in: body + name: body + description: A list of events. + schema: + type: object + example: { + "events": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + }, + { + "age": 1984, + "content": { + "body": "another incoming message", + "msgtype": "m.text" + }, + "event_id": "$1228055551ffsef:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + description: "Transaction informations" + properties: + events: + type: array + description: A list of events + items: + type: object + title: Event + required: ["events"] + responses: + 200: + description: The transaction was processed successfully. + examples: + application/json: { + } + schema: + type: object \ No newline at end of file diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 08e3b0626..09e38e914 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -155,6 +155,8 @@ be made without blocking other aspects of the homeserver. Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID on retries, as the AS may have already processed the events. +{{transactions_as_http_api}} + Querying ++++++++ From 6a91ea9c85a597f0ac89fce9e99f57be4be388da Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 23 Aug 2018 13:31:33 -0600 Subject: [PATCH 101/166] Specify that application services receive events in the CSAPI format Fixes https://github.com/matrix-org/matrix-doc/issues/1269 This is also supposed to fix the 'age' problem, however that is a larger problem with the event schemas that is reserved for a future PR/commit. Reference: https://github.com/matrix-org/matrix-doc/issues/1294 Reference: https://github.com/matrix-org/matrix-doc/issues/1524 --- api/application-service/transactions.yaml | 38 +++++------------------ 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 9388d3c5b..8735cc8f7 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -21,8 +21,6 @@ schemes: - https - http basePath: "/" -consumes: - - application/json produces: - application/json paths: @@ -53,37 +51,16 @@ paths: type: object example: { "events": [ - { - "age": 32, - "content": { - "body": "incoming message", - "msgtype": "m.text" - }, - "event_id": "$14328055551tzaee:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - }, - { - "age": 1984, - "content": { - "body": "another incoming message", - "msgtype": "m.text" - }, - "event_id": "$1228055551ffsef:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - } + {"$ref": "../../event-schemas/examples/m.room.member"}, + {"$ref": "../../event-schemas/examples/m.room.message#m.text"} ] } - description: "Transaction informations" + description: Transaction information properties: events: type: array - description: A list of events + description: |- + A list of events, formatted as per the Client-Server API. items: type: object title: Event @@ -92,7 +69,6 @@ paths: 200: description: The transaction was processed successfully. examples: - application/json: { - } + application/json: {} schema: - type: object \ No newline at end of file + type: object From 9eda1a697143249a03da6bbf02674263f74e5929 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 23 Aug 2018 15:07:55 -0600 Subject: [PATCH 102/166] Move the third party network API to it's own section --- .../application_service.yaml | 240 ---------------- api/application-service/protocols.yaml | 267 ++++++++++++++++++ specification/application_service_api.rst | 20 ++ 3 files changed, 287 insertions(+), 240 deletions(-) create mode 100644 api/application-service/protocols.yaml diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index bc6e45d5d..8a32c95b3 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -96,243 +96,3 @@ paths: } schema: type: object - "/_matrix/app/unstable/thirdparty/protocol/{protocol}": - get: - summary: Retrieve metadata about a specific protocol that the application service supports. - description: |- - This API is called by the homeserver when it wants to present clients - with specific information about the various third party networks that - an application service supports. - operationId: getProtocolMetadata - parameters: - - in: path - name: protocol - type: string - description: The protocol ID. - required: true - x-example: "irc" - responses: - 200: - description: The protocol was found and metadata returned. - schema: - $ref: definitions/protocol_metadata.yaml - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: No protocol was found with the given path. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user/{protocol}": - get: - summary: Retrieve the Matrix User ID of a corresponding third party user. - description: |- - This API is called by the homeserver in order to retrieve a Matrix - User ID linked to a user on the third party network, given a set of - user parameters. - operationId: queryUserByProtocol - parameters: - - in: path - name: protocol - type: string - description: The protocol ID. - required: true - x-example: irc - - in: query - name: fields... - type: string - description: |- - One or more custom fields that are passed to the application - service to help identify the user. - responses: - 200: - description: The Matrix User IDs found with the given parameters. - schema: - $ref: definitions/user_batch.yaml - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: No users were found with the given parameters. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location/{protocol}": - get: - summary: Retreive Matrix-side portal rooms leading to a third party location. - description: |- - Retrieve a list of Matrix portal rooms that lead to the matched third party location. - operationId: queryLocationByProtocol - parameters: - - in: path - name: protocol - type: string - description: The protocol ID. - required: true - x-example: irc - - in: query - name: fields... - type: string - description: |- - One or more custom fields that are passed to the application - service to help identify the third party location. - responses: - 200: - description: At least one portal room was found. - schema: - $ref: definitions/location_batch.yaml - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: No mappings were found with the given parameters. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location": - get: - summary: Reverse-lookup third party locations given a Matrix room alias. - description: |- - Retreive an array of third party network locations from a Matrix room - alias. - operationId: queryLocationByAlias - parameters: - - in: query - name: alias - type: string - description: The Matrix room alias to look up. - responses: - 200: - description: |- - All found third party locations. - schema: - $ref: definitions/location_batch.yaml - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: No mappings were found with the given parameters. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user": - get: - summary: Reverse-lookup third party users given a Matrix User ID. - description: |- - Retreive an array of third party users from a Matrix User ID. - operationId: queryUserByID - parameters: - - in: query - name: userid - type: string - description: The Matrix User ID to look up. - responses: - 200: - description: |- - An array of third party users. - schema: - $ref: definitions/user_batch.yaml - 401: - description: |- - The homeserver has not supplied credentials to the application service. - Optional error information can be included in the body of this response. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 403: - description: |- - The credentials supplied by the homeserver were rejected. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml - 404: - description: No mappings were found with the given parameters. - examples: - application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" - } - schema: - $ref: ../client-server/definitions/errors/error.yaml diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml new file mode 100644 index 000000000..376e66c0b --- /dev/null +++ b/api/application-service/protocols.yaml @@ -0,0 +1,267 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Application Service API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: "/" +consumes: + - application/json +produces: + - application/json +paths: + "/_matrix/app/unstable/thirdparty/protocol/{protocol}": + get: + summary: Retrieve metadata about a specific protocol that the application service supports. + description: |- + This API is called by the homeserver when it wants to present clients + with specific information about the various third party networks that + an application service supports. + operationId: getProtocolMetadata + parameters: + - in: path + name: protocol + type: string + description: The protocol ID. + required: true + x-example: "irc" + responses: + 200: + description: The protocol was found and metadata returned. + schema: + $ref: definitions/protocol_metadata.yaml + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: No protocol was found with the given path. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + "/_matrix/app/unstable/thirdparty/user/{protocol}": + get: + summary: Retrieve the Matrix User ID of a corresponding third party user. + description: |- + This API is called by the homeserver in order to retrieve a Matrix + User ID linked to a user on the third party network, given a set of + user parameters. + operationId: queryUserByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol ID. + required: true + x-example: irc + - in: query + name: fields... + type: string + description: |- + One or more custom fields that are passed to the application + service to help identify the user. + responses: + 200: + description: The Matrix User IDs found with the given parameters. + schema: + $ref: definitions/user_batch.yaml + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: No users were found with the given parameters. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + "/_matrix/app/unstable/thirdparty/location/{protocol}": + get: + summary: Retreive Matrix-side portal rooms leading to a third party location. + description: |- + Retrieve a list of Matrix portal rooms that lead to the matched third party location. + operationId: queryLocationByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol ID. + required: true + x-example: irc + - in: query + name: fields... + type: string + description: |- + One or more custom fields that are passed to the application + service to help identify the third party location. + responses: + 200: + description: At least one portal room was found. + schema: + $ref: definitions/location_batch.yaml + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + "/_matrix/app/unstable/thirdparty/location": + get: + summary: Reverse-lookup third party locations given a Matrix room alias. + description: |- + Retreive an array of third party network locations from a Matrix room + alias. + operationId: queryLocationByAlias + parameters: + - in: query + name: alias + type: string + description: The Matrix room alias to look up. + responses: + 200: + description: |- + All found third party locations. + schema: + $ref: definitions/location_batch.yaml + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + "/_matrix/app/unstable/thirdparty/user": + get: + summary: Reverse-lookup third party users given a Matrix User ID. + description: |- + Retreive an array of third party users from a Matrix User ID. + operationId: queryUserByID + parameters: + - in: query + name: userid + type: string + description: The Matrix User ID to look up. + responses: + 200: + description: |- + An array of third party users. + schema: + $ref: definitions/user_batch.yaml + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } + schema: + $ref: ../client-server/definitions/errors/error.yaml diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 08e3b0626..c5f8be720 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -180,6 +180,26 @@ this request (e.g. to join a room alias). {{query_room_as_http_api}} +Third party networks +++++++++++++++++++++ + +Application services may declare which protocols they support via their registration +file. These networks are generally for third party services such as IRC that the +application service is managing. Application services may populate a Matrix room +directory for their registered protocols, as defined in the Client-Server API Extensions. + +Each protocol may have several "locations". A location within a protocol is a place +in the third party network, such as an IRC channel. Users of the third party network +may also be represented by the application service. + +Locations and users can be searched by fields defined by the application service, such +as by display name or other attribute. When clients request the homeserver to search +in a particular "network" (protocol), the search fields will be passed along to the +application service for filtering. + +{{protocols_as_http_api}} + + HTTP APIs +++++++++ From 2d43ff123400a54f087a30179af7cb5077ab8cf5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 10:22:10 -0600 Subject: [PATCH 103/166] Update third party network schemas Some information was missed when this was reviewed. This commit adds some additional documentation for how these objects interact with each other. --- .../definitions/location.yaml | 8 +- .../definitions/protocol.yaml | 88 +++++++++++++------ .../definitions/protocol_metadata.yaml | 2 + api/application-service/definitions/user.yaml | 6 +- 4 files changed, 72 insertions(+), 32 deletions(-) diff --git a/api/application-service/definitions/location.yaml b/api/application-service/definitions/location.yaml index 4967ef61f..5a0f92c88 100644 --- a/api/application-service/definitions/location.yaml +++ b/api/application-service/definitions/location.yaml @@ -19,12 +19,14 @@ properties: protocol: description: The protocol ID that the third party location is a part of. type: string - example: irc + example: "irc" fields: description: Information used to identify this third party location. type: object - example: - "network": "freenode" + example: { + "network": "freenode", "channel": "#matrix" + } +required: ['alias', 'protocol', 'fields'] title: Location type: object \ No newline at end of file diff --git a/api/application-service/definitions/protocol.yaml b/api/application-service/definitions/protocol.yaml index 231e8288a..851091d69 100644 --- a/api/application-service/definitions/protocol.yaml +++ b/api/application-service/definitions/protocol.yaml @@ -11,41 +11,60 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +title: Protocol +type: object properties: user_fields: - description: Fields used to identify a third party user. + description: |- + Fields which may be used to identify a third party user. These should be + ordered to suggest the way that entities may be grouped, where higher + groupings are ordered first. For example, the name of a network should be + searched before the nickname of a user. type: array items: type: string description: Field used to identify a third party user. example: ["network", "nickname"] location_fields: - description: Fields used to identify a third party location. + description: |- + Fields which may be used to identify a third party location. These should be + ordered to suggest the way that entities may be grouped, where higher + groupings are ordered first. For example, the name of a network should be + searched before the name of a channel. type: array items: type: string description: Field used to identify a third party location. example: ["network", "channel"] icon: - description: An icon representing the third party protocol. + description: A content URI representing an icon for the third party protocol. type: string example: "mxc://example.org/aBcDeFgH" field_types: title: Field Types - description: All location or user fields should have an entry here. + description: |- + The type definitions for the fields defined in the ``user_fields`` and + ``location_fields``. Each entry in those arrays MUST have an entry here. The + ``string`` key for this object is field name itself. + + May be an empty object if no fields are defined. type: object - properties: - fieldname: - title: Field Type - description: Definition of valid values for a field. - type: object - properties: - regexp: - description: A regular expression for validation of a field's value. - type: string - placeholder: - description: An placeholder serving as a valid example of the field value. - type: string + additionalProperties: + title: Field Type + description: Definition of valid values for a field. + type: object + properties: + regexp: + description: |- + A regular expression for validation of a field's value. This may be relatively + coarse to verify the value as the application service providing this protocol + may apply additional validation or filtering. + type: string + placeholder: + description: An placeholder serving as a valid example of the field value. + type: string + required: ['regexp', 'placeholder'] + required: ['fieldname'] example: { "network": { "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", @@ -63,17 +82,32 @@ properties: instances: description: |- A list of objects representing independent instances of configuration. - For instance multiple networks on IRC if multiple are bridged by the - same bridge. + For example, multiple networks on IRC if multiple are provided by the + same application service. type: array items: type: object - example: { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } -title: Protocol -type: object \ No newline at end of file + title: Protocol Instance + properties: + desc: + type: string + description: A human-readable description for the protocol, such as the name. + example: "Freenode" + icon: + type: string + description: |- + An optional content URI representing the protocol. Overrides the one provided + at the higher level Protocol object. + example: "mxc://example.org/JkLmNoPq" + fields: + type: object + description: Preset values for ``fields`` the client may use to search by. + example: { + "network": "freenode" + } + network_id: + type: string + description: A unique identifier across all instances. + example: "freenode" + required: ['desc', 'fields', 'network_id'] +required: ['user_fields', 'location_fields', 'icon', 'field_types', 'instances'] diff --git a/api/application-service/definitions/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml index 2b2c8f4e7..e7bf45da6 100644 --- a/api/application-service/definitions/protocol_metadata.yaml +++ b/api/application-service/definitions/protocol_metadata.yaml @@ -36,6 +36,7 @@ example: { }, "instances": [ { + "network_id": "freenode", "desc": "Freenode", "icon": "mxc://example.org/JkLmNoPq", "fields": { @@ -59,6 +60,7 @@ example: { }, "instances": [ { + "network_id": "gitter", "desc": "Gitter", "icon": "mxc://example.org/zXyWvUt", "fields": {} diff --git a/api/application-service/definitions/user.yaml b/api/application-service/definitions/user.yaml index a7b2287ef..258e7c138 100644 --- a/api/application-service/definitions/user.yaml +++ b/api/application-service/definitions/user.yaml @@ -21,11 +21,13 @@ properties: protocol: description: The protocol ID that the third party location is a part of. type: string - example: gitter + example: "gitter" fields: description: Information used to identify this third party location. type: object - example: + example: { "user": "jim" + } +required: ['userid', 'protocol', 'fields'] title: User type: object \ No newline at end of file From 017d6db7370f87b5b72e9ce19d6f8962462ab525 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:07:30 -0600 Subject: [PATCH 104/166] Document third party network/protocol directories (for appservices) Fixes https://github.com/matrix-org/matrix-doc/issues/869 --- .../appservice_room_directory.yaml | 87 +++++++++++++++++++ api/client-server/list_public_rooms.yaml | 20 ++++- api/server-server/public_rooms.yaml | 14 +++ specification/application_service_api.rst | 12 +++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 api/client-server/appservice_room_directory.yaml diff --git a/api/client-server/appservice_room_directory.yaml b/api/client-server/appservice_room_directory.yaml new file mode 100644 index 000000000..0225ecd85 --- /dev/null +++ b/api/client-server/appservice_room_directory.yaml @@ -0,0 +1,87 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Application Service Room Directory API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + # Note: this is the same access_token definition used elsewhere in the client + # server API, however this expects an access token for an application service. + $ref: definitions/security.yaml +paths: + "/directory/list/appservice/{networkId}/{roomId}": + put: + summary: |- + Updates a room's visibility in the application service's room directory. + description: |- + Updates the visibility of a given room on the application service's room + directory. + + This API is similar to the room directory visibility API used by clients + to update the homeserver's more general room directory. + + This API requires the use of an application service access token (``as_token``) + instead of a typical client's access_token. This API cannot be invoked by + users who are not identified as application services. + operationId: updateAppserviceRoomDirectoryVsibility + parameters: + - in: path + type: string + name: networkId + description: |- + The protocol (network) ID to update the room list for. This would + have been provided by the application service as being listed as + a supported protocol. + required: true + x-example: "irc" + - in: path + type: string + name: roomId + description: The room ID to add to the directory. + required: true + x-example: "!somewhere:domain.com" + - in: body + name: body + schema: + type: object + properties: + visibility: + type: enum + enum: ["public", "private"] + description: |- + Whether the room should be visible (public) in the directory + or not (private). + example: "public" + required: ['visibility'] + security: + # again, this is the appservice's token - not a typical client's + - accessToken: [] + responses: + 200: + description: The room's directory visibility has been updated. + schema: + type: object + examples: + application/json: {} + tags: + - Application service room directory management diff --git a/api/client-server/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml index 72a120609..d1abc68ec 100644 --- a/api/client-server/list_public_rooms.yaml +++ b/api/client-server/list_public_rooms.yaml @@ -194,8 +194,26 @@ paths: description: |- A string to search for in the room metadata, e.g. name, topic, canonical alias etc. (Optional). + include_all_networks: + type: boolean + description: |- + Whether or not to include all known networks/protocols from + application services on the homeserver. Defaults to false. + example: false + third_party_instance_id: + type: string + description: |- + The specific third party network/protocol to request from the + homeserver. Can only be used if ``include_all_networks`` is false. + example: "irc" example: { - "limit": 10, "filter": {"generic_search_term": "foo"}} + "limit": 10, + "filter": { + "generic_search_term": "foo" + }, + "include_all_networks": false, + "third_party_instance_id": "irc" + } responses: 200: description: A list of the rooms on the server. diff --git a/api/server-server/public_rooms.yaml b/api/server-server/public_rooms.yaml index d162568f8..b76910236 100644 --- a/api/server-server/public_rooms.yaml +++ b/api/server-server/public_rooms.yaml @@ -49,6 +49,20 @@ paths: A pagination token from a previous call to this endpoint to fetch more rooms. x-example: "GetMoreRoomsTokenHere" + - in: query + name: include_all_networks + type: boolean + description: |- + Whether or not to include all networks/protocols defined by application + services on the homeserver. Defaults to false. + x-example: false + - in: query + name: third_party_instance_id + type: string + description: |- + The specific third party network/protocol to request from the homeserver. + Can only be used if ``include_all_networks`` is false. + x-example: "irc" responses: 200: description: The public room list for the homeserver. diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index c5f8be720..9d467e879 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -218,6 +218,9 @@ Application services can use a more powerful version of the client-server API by identifying itself as an application service to the homeserver. +Endpoints defined in this section MUST be supported by homeservers in the +client-server API as accessible only by application services. + Identity assertion ++++++++++++++++++ The client-server API infers the user ID from the ``access_token`` provided in @@ -314,6 +317,15 @@ API MUST do so with a virtual user (provide a ``user_id`` via the query string). is expected that the application service use the transactions pushed to it to handle events rather than syncing with the user implied by ``sender_localpart``. +Application service room directories +++++++++++++++++++++++++++++++++++++ + +Application services can maintain their own room directories for their defined +third party protocols. These room directories may be accessed by clients through +additional parameters on the ``/publicRooms`` client-server endpoint. + +{{appservice_room_directory_cs_http_api}} + Event fields ~~~~~~~~~~~~ From 9b19fc27def683ec1f64014aa1686096cf7ee716 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:09:55 -0600 Subject: [PATCH 105/166] changelog --- changelogs/client_server/newsfragments/1554.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1554.feature diff --git a/changelogs/client_server/newsfragments/1554.feature b/changelogs/client_server/newsfragments/1554.feature new file mode 100644 index 000000000..ec7ffe71c --- /dev/null +++ b/changelogs/client_server/newsfragments/1554.feature @@ -0,0 +1 @@ +Add third party network room directories, as provided by application services. From 811998735cf2af6ea12ee27427b929ffa3491d93 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:51:23 -0600 Subject: [PATCH 106/166] Define common error codes in the Identity Service API Fixes https://github.com/matrix-org/matrix-doc/issues/1407 --- specification/identity_service_api.rst | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index 3b037caf0..fb08f6377 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -56,6 +56,75 @@ is left as an exercise for the client. 3PID types are described in `3PID Types`_ Appendix. +API Standards +------------- + +The mandatory baseline for identity service communication in Matrix is exchanging +JSON objects over HTTP APIs. HTTPS is required for communication, and all API calls +use a Content-Type of ``application/json``. In addition, strings MUST be encoded as +UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard error response". +This is a JSON object which looks like: + +.. code:: json + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. There may be +additional keys depending on the error, but the keys ``error`` and ``errcode`` +MUST always be present. + +Some standard error codes are below: + +:``M_NOT_FOUND``: + The resource requested could not be located. + +:``M_MISSING_PARAMS``: + The request was missing one or more parameters. + +:``M_INVALID_PARAM``: + The request contained one or more invalid parameters. + +:``M_SESSION_NOT_VALIDATED``: + The session has not been validated. + +:``M_NO_VALID_SESSION``: + A session could not be located for the given parameters. + +:``M_SESSION_EXPIRED``: + The session has expired and must be renewed. + +:``M_INVALID_EMAIL``: + The email address provided was not valid. + +:``M_EMAIL_SEND_ERROR``: + There was an error sending an email. Typically seen when attempting to verify + ownership of a given email address. + +:``M_INVALID_ADDRESS``: + The provided third party address was not valid. + +:``M_SEND_ERROR``: + There was an error sending a notification. Typically seen when attempting to + verify ownership of a given third party address. + +:``M_UNRECOGNIZED``: + The request contained an unrecognised value, such as an unknown token or medium. + +:``M_THREEPID_IN_USE``: + The third party identifier is already in use by another user. Typically this + error will have an additional ``mxid`` property to indicate who owns the + third party identifier. + +:``M_UNKNOWN``: + An unknown error has occurred. + Privacy ------- From 4abd618147618abc73502f1bd84ee55da5b139e5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:53:00 -0600 Subject: [PATCH 107/166] Flag response fields in the Identity Service spec as required --- api/identity/associations.yaml | 9 +++++++++ api/identity/email_associations.yaml | 2 ++ api/identity/invitation_signing.yaml | 1 + api/identity/lookup.yaml | 8 ++++++++ api/identity/phone_associations.yaml | 2 ++ api/identity/pubkey.yaml | 2 ++ api/identity/store_invite.yaml | 1 + 7 files changed, 25 insertions(+) diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 784bb5d63..6d282e8dc 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -62,6 +62,7 @@ paths: validated_at: type: integer description: Timestamp indicating the time that the 3pid was validated. + required: ['medium', 'address', 'validated_at'] 400: description: |- The session has not been validated. @@ -158,6 +159,14 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures 400: description: |- The association was not published. diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index 8431c9e83..c9fb0cd70 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -93,6 +93,7 @@ paths: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: @@ -151,6 +152,7 @@ paths: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of an email address. description: |- diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index 982dbff78..c595299f5 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -71,6 +71,7 @@ paths: token: type: string description: The token for the invitation. + required: ['mxid', 'sender', 'signatures', 'token'] examples: application/json: { "mxid": "@foo:bar.com", diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index bfd2153ec..3bc58be59 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -86,6 +86,14 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures "/bulk_lookup": post: summary: Lookup Matrix user IDs for a list of 3pids. diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index c2cc6cfe7..605dadcc5 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -99,6 +99,7 @@ paths: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: @@ -157,6 +158,7 @@ paths: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of a phone number. description: |- diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 00796975d..45f8b99ac 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -80,6 +80,7 @@ paths: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] "/pubkey/ephemeral/isvalid": get: summary: Check whether a short-term public key is valid. @@ -108,3 +109,4 @@ paths: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index 6b847b5b0..d6fae7a76 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -90,6 +90,7 @@ paths: display_name: type: string description: The generated (redacted) display_name. + required: ['token', 'public_keys', 'display_name'] example: application/json: { "token": "sometoken", From dafea96621d6383f6b44d9368024bb95c8ab94a7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:53:27 -0600 Subject: [PATCH 108/166] Fix indentation and schema references in the identity service spec --- api/identity/associations.yaml | 69 +++++++++++++++----------- api/identity/email_associations.yaml | 31 +++++++----- api/identity/invitation_signing.yaml | 6 ++- api/identity/lookup.yaml | 35 +++++++------ api/identity/phone_associations.yaml | 33 +++++++----- api/identity/pubkey.yaml | 20 ++++++-- api/identity/store_invite.yaml | 34 +++++++------ schemas/server-signatures.yaml | 24 +++++++++ specification/identity_service_api.rst | 2 +- 9 files changed, 162 insertions(+), 92 deletions(-) create mode 100644 schemas/server-signatures.yaml diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 6d282e8dc..4225919b1 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -46,10 +46,10 @@ paths: description: Validation information for the session. examples: application/json: { - "medium": "email", - "validated_at": 1457622739026, - "address": "louise@bobs.burgers" - } + "medium": "email", + "validated_at": 1457622739026, + "address": "louise@bobs.burgers" + } schema: type: object properties: @@ -72,16 +72,20 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/bind": post: summary: Publish an association between a session and a Matrix user ID. @@ -102,10 +106,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "mxid": "@ears:matrix.org" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "mxid": "@ears:matrix.org" + } properties: sid: type: string @@ -122,19 +126,19 @@ paths: description: The association was published. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -159,6 +163,7 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" required: - address - medium @@ -176,13 +181,17 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index c9fb0cd70..28f5e6809 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -51,10 +51,10 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "email": "foo@example.com", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "email": "foo@example.com", + "send_attempt": 1 + } properties: client_secret: type: string @@ -85,8 +85,8 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: @@ -100,6 +100,13 @@ paths: - ``M_INVALID_EMAIL``: The email address provided was invalid. - ``M_EMAIL_SEND_ERROR``: The validation email could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_EMAIL", + "error": "The email address is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/email/submitToken": post: summary: Validate ownership of an email address. @@ -123,10 +130,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -144,8 +151,8 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index c595299f5..7de62dd41 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -68,6 +68,7 @@ paths: signatures: type: object description: The signature of the mxid, sender, and token. + $ref: "../../schemas/server-signatures.yaml" token: type: string description: The token for the invitation. @@ -85,7 +86,10 @@ paths: } 404: description: Token was not found. - example: { + examples: + application/json: { "errcode": "M_UNRECOGNIZED", "error": "Didn't recognize token" } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index 3bc58be59..6f993ac7a 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -49,19 +49,18 @@ paths: The association for that 3pid, or the empty object if no association is known. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, - - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -86,6 +85,7 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" required: - address - medium @@ -118,9 +118,11 @@ paths: items: type: array title: 3PID mappings + minItems: 2 + maxItems: 2 items: - type: string - title: 3PID medium or address + - type: 3PID Medium + - type: 3PID Address description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. required: - "threepids" @@ -142,9 +144,12 @@ paths: items: type: array title: 3PID mappings + minItems: 3 + maxItems: 3 items: - type: string - title: 3PID medium or address or the Matrix ID + - type: 3PID Medium + - type: 3PID Address + - type: Matrix User ID description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. required: - "threepids" diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index 605dadcc5..f6b1bd45b 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -51,11 +51,11 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "country": "GB", - "phone_number": "07700900001", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "country": "GB", + "phone_number": "07700900001", + "send_attempt": 1 + } properties: client_secret: type: string @@ -91,8 +91,8 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: @@ -106,6 +106,13 @@ paths: - ``M_INVALID_ADDRESS``: The phone number provided was invalid. - ``M_SEND_ERROR``: The validation SMS could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_ADDRESS", + "error": "The phone number is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/msisdn/submitToken": post: summary: Validate ownership of a phone number. @@ -129,10 +136,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -150,8 +157,8 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 45f8b99ac..9cb7c74e9 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -45,13 +45,25 @@ paths: The public key exists. examples: application/json: { - "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" - } + "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" + } schema: type: object properties: public_key: type: string + description: Unpadded Base64 encoded public key. + required: ['public_key'] + 404: + description: + The public key was not found. + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "The public key was not found" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/pubkey/isvalid": get: summary: Check whether a long-term public key is valid. @@ -72,8 +84,8 @@ paths: The validity of the public key. examples: application/json: { - "valid": true - } + "valid": true + } schema: type: object properties: diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index d6fae7a76..1eca7198e 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -54,11 +54,11 @@ paths: schema: type: object example: { - "medium": "email", - "address": "foo@bar.baz", - "room_id": "!something:example.tld", - "sender": "@bob:example.com" - } + "medium": "email", + "address": "foo@bar.baz", + "room_id": "!something:example.tld", + "sender": "@bob:example.com" + } properties: medium: type: string @@ -93,13 +93,13 @@ paths: required: ['token', 'public_keys', 'display_name'] example: application/json: { - "token": "sometoken", - "public_keys": [ - "serverpublickey", - "ephemeralpublickey" - ], - "display_name": "f...@b..." - } + "token": "sometoken", + "public_keys": [ + "serverpublickey", + "ephemeralpublickey" + ], + "display_name": "f...@b..." + } 400: description: | An error has occured. @@ -109,7 +109,9 @@ paths: error code will be ``M_UNRECOGNIZED``. examples: application/json: { - "errcode": "M_THREEPID_IN_USE", - "error": "Binding already known", - "mxid": mxid - } + "errcode": "M_THREEPID_IN_USE", + "error": "Binding already known", + "mxid": mxid + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/schemas/server-signatures.yaml b/schemas/server-signatures.yaml new file mode 100644 index 000000000..a18552566 --- /dev/null +++ b/schemas/server-signatures.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +example: { + "example.com": { + "ed25519:0": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } +} +additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string \ No newline at end of file diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index fb08f6377..7bbd8ae8c 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -23,7 +23,7 @@ user identifiers. From time to time, it is useful to refer to users by other number. This identity service specification describes how mappings between third-party identifiers and Matrix user identifiers can be established, validated, and used. This description technically may apply to any 3pid, but in -practice has only been applied specifically to email addresses. +practice has only been applied specifically to email addresses and phone numbers. .. contents:: Table of Contents .. sectnum:: From c879eb950f9a201e4bc6075b7796598dcd4e3aaf Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sat, 25 Aug 2018 20:38:06 +0900 Subject: [PATCH 109/166] client-server/openid.yaml: Fix a type'o Signed-off-by: Alexey Rusakov --- api/client-server/openid.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/openid.yaml b/api/client-server/openid.yaml index 4b89232ea..cb982fb32 100644 --- a/api/client-server/openid.yaml +++ b/api/client-server/openid.yaml @@ -90,7 +90,7 @@ paths: The homeserver domain the consumer should use when attempting to verify the user's identity. expires_in: - type: int + type: integer description: |- The number of seconds before this token expires and a new one must be generated. From 312799ae78a27112077753db9782e9193e12e2a7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 22:30:49 -0600 Subject: [PATCH 110/166] General clarification for mention and how matrix.to URIs are meant to work --- specification/appendices/identifier_grammar.rst | 5 +++++ specification/modules/mentions.rst | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index 013afa799..0412c065b 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -274,6 +274,11 @@ parameter is only used in the case of permalinks where an event ID is referenced The matrix.to URI, when referenced, must always start with ``https://matrix.to/#/`` followed by the identifier. +Clients should not rely on matrix.to URIs falling back to a web server if accessed +and instead should perform some sort of action within the client. For example, if +the user where to click on a matrix.to URI for a room alias, the client may open +a view for the user to participate in the room. + Examples of matrix.to URIs are: * Room: ``https://matrix.to/#/!somewhere:domain.com`` diff --git a/specification/modules/mentions.rst b/specification/modules/mentions.rst index e7483ae4f..4501b7766 100644 --- a/specification/modules/mentions.rst +++ b/specification/modules/mentions.rst @@ -41,7 +41,8 @@ Client behaviour ---------------- In addition to using the appropriate ``matrix.to URI`` for the mention, -clients should use the following guidelines when making mentions: +clients should use the following guidelines when making mentions in events +to be sent: * When mentioning users, use the user's potentially ambigious display name for the anchor's text. If the user does not have a display name, use the user's From aa294fac064e7bb8e36932a666c529fdd996f2bc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 22:57:52 -0600 Subject: [PATCH 111/166] Define the supported HTML subset for message events Also clarify that `m.notice` messages can support HTML. Fixes https://github.com/matrix-org/matrix-doc/issues/1559 Fixes https://github.com/matrix-org/matrix-doc/issues/1560 --- .../examples/m.room.message#m.notice | 4 +- specification/modules/instant_messaging.rst | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/event-schemas/examples/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice index 978c67e6b..876cbbb7d 100644 --- a/event-schemas/examples/m.room.message#m.notice +++ b/event-schemas/examples/m.room.message#m.notice @@ -2,7 +2,9 @@ "age": 242352, "content": { "body": "This is an example notice", - "msgtype": "m.notice" + "msgtype": "m.notice", + "format": "org.matrix.custom.html", + "formatted_body": "This is an example notice" }, "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index ff87f74be..29d33c558 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -56,6 +56,54 @@ of message being sent. Each type has their own required and optional keys, as outlined below. If a client cannot display the given ``msgtype`` then it SHOULD display the fallback plain text ``body`` key instead. +Some message types support HTML in the event content that clients should prefer +to display if available. Currently ``m.text``, ``m.emote``, and ``m.notice`` +support an additional ``format`` parameter of ``org.matrix.custom.html``. When +this field is present, a ``formatted_body`` with the HTML must be provided. The +plain text version of the HTML should be provided in the ``body``. + +Clients should limit the HTML they render to avoid Cross-Site Scripting, HTML +injection, and similar attacks. The strongly suggested set of HTML tags to permit, +denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, +``h2``, ``h3``, ``h4``, ``h5``, ``h6``, ``blockquote``, ``p``, ``a``, ``ul``, +``ol``, ``sup``, ``sub``, ``nl``, ``li``, ``b``, ``i``, ``u``, ``strong``, ``em``, +``strike``, ``code``, ``hr``, ``br``, ``div``, ``table``, ``thead``, ``tbody``, +``tr``, ``th``, ``td``, ``caption``, ``pre``, ``span``, ``img``. + +Not all attributes on those tags should be permitted as they may be avenues for +other disruption attempts, such as adding ``onclick`` handlers or excessively +large text. Clients should only permit the attributes listed for the tags below. +Where ``data-mx-bg-color`` and ``data-mx-color`` are listed, clients should +translate the value (a 6-character hex color code) to the appropriate CSS/attributes +for the tag. + + +:``font``: + ``data-mx-bg-color``, ``data-mx-color`` + +:``span``: + ``data-mx-bg-color``, ``data-mx-color`` + +:``a``: + ``name``, ``target``, ``href`` (provided the value is not relative and has a scheme + matching one of: ``https``, ``http``, ``ftp``, ``mailto``, ``magnet``) + +:``img``: + ``width``, ``height``, ``alt``, ``title``, ``src`` (provided it is a Matrix Content + URI) + +:``ol``: + ``start`` + +:``code``: + ``class`` (only classes which start with ``language-`` for syntax highlighting) + + +Additionally, clients should ensure that *all* ``a`` tags get a ``rel="noopener"`` +to prevent the target page from referencing the client's tab/window. + + + {{msgtype_events}} From 3c472f70e3aa0f29aa95c05887d50970a9cb2de5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 22:59:22 -0600 Subject: [PATCH 112/166] Changelog --- changelogs/client_server/newsfragments/1562.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1562.clarification diff --git a/changelogs/client_server/newsfragments/1562.clarification b/changelogs/client_server/newsfragments/1562.clarification new file mode 100644 index 000000000..c46e189d0 --- /dev/null +++ b/changelogs/client_server/newsfragments/1562.clarification @@ -0,0 +1 @@ +Clarify the supported HTML features for room messages. From 0a6c1c4ddaa28a6ce169c1f38f22ddcde7fa0291 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 23:18:43 -0600 Subject: [PATCH 113/166] Ensure the event examples and matrix.org assets are tested Otherwise we go nearly a week without realizing the build is failing due to bad schemas/examples. This also helps us ensure a PR is up to par. This commit is expected to cause a build failure at this time. A future commit will actually fix the project and address concerns raised by the testing. --- .circleci/config.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a0b60325..f79449f38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,9 +23,29 @@ buildswaggerui: &buildswaggerui wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/scripts/swagger-ui.patch patch api/client-server/index.html swagger-ui.patch +checkexamples: &checkexamples + name: Check Event Examples + command: | + source /env/bin/activate + cd event-schemas + ./check_examples.py + +genmatrixassets: &genmatrixassets + name: Generate/Verify matrix.org assets + command: | + source /env/bin/activate + ./scripts/generate-matrix-org-assets + version: 2 jobs: + check-docs: + docker: + - image: uhoreg/matrix-doc-build + steps: + - checkout + - run: *checkexamples + - run: *genmatrixassets # We don't actually use the assets, but we do want to make sure they build build-docs: docker: - image: uhoreg/matrix-doc-build From be9f6042e527a44cd1b4315a208380adef7e1507 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 23:21:50 -0600 Subject: [PATCH 114/166] Fix encrypted event examples --- event-schemas/examples/m.room.encrypted#megolm | 2 +- event-schemas/examples/m.room.encrypted#olm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/event-schemas/examples/m.room.encrypted#megolm b/event-schemas/examples/m.room.encrypted#megolm index 1f9b75208..726763181 100644 --- a/event-schemas/examples/m.room.encrypted#megolm +++ b/event-schemas/examples/m.room.encrypted#megolm @@ -1,6 +1,6 @@ { "content": { - "algorithm": "m.megolm.v1.aes-sha2", + "algorithm": "m.megolm.v1.aes-sha256", "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", "device_id": "RJYKSTBOIE", "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", diff --git a/event-schemas/examples/m.room.encrypted#olm b/event-schemas/examples/m.room.encrypted#olm index abb23c31e..23f657385 100644 --- a/event-schemas/examples/m.room.encrypted#olm +++ b/event-schemas/examples/m.room.encrypted#olm @@ -2,7 +2,7 @@ "type": "m.room.encrypted", "sender": "@example:localhost", "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", + "algorithm": "m.olm.v1.curve25519-aes-sha256", "sender_key": "Szl29ksW/L8yZGWAX+8dY1XyFi+i5wm+DRhTGkbMiwU", "ciphertext": { "7qZcfnBmbEGzxxaWfBjElJuvn7BZx+lSz/SvFrDF/z8": { From e9e93b0eecad5546efef33d46bdbe214c47d8c31 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 20:51:39 -0600 Subject: [PATCH 115/166] Move `invite_room_state` to the correct place in the client-server API Fixes https://github.com/matrix-org/matrix-doc/issues/1350 --- .../examples/m.room.member#invite_room_state | 32 +++++------ event-schemas/schema/m.room.member | 53 +++++++++++-------- scripts/templating/matrix_templates/units.py | 9 ---- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/event-schemas/examples/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state index 1a93b395b..965669adb 100644 --- a/event-schemas/examples/m.room.member#invite_room_state +++ b/event-schemas/examples/m.room.member#invite_room_state @@ -5,22 +5,24 @@ "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid" }, - "invite_room_state": [ - { - "type": "m.room.name", - "state_key": "", - "content": { - "name": "Forest of Magic" + "unsigned": { + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rule": "invite" + } } - }, - { - "type": "m.room.join_rules", - "state_key": "", - "content": { - "join_rule": "invite" - } - } - ], + ] + }, "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/schema/m.room.member b/event-schemas/schema/m.room.member index 4f4077a76..5fb5356d8 100644 --- a/event-schemas/schema/m.room.member +++ b/event-schemas/schema/m.room.member @@ -18,7 +18,9 @@ description: |- The ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise. - This event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a subset of state events such as the room name. + This event may also include an ``invite_room_state`` key inside the event's ``unsigned`` data. + If present, this contains an array of ``StrippedState`` Events. These events provide information + on a subset of state events such as the room name. properties: content: properties: @@ -71,32 +73,37 @@ properties: - signed title: Invite type: object + unsigned: + type: object + title: UnsignedData + description: Contains optional extra information about the event. + properties: + invite_room_state: + description: 'A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``. Note that this state is informational, and SHOULD NOT be trusted; once the client has joined the room, it SHOULD fetch the live state from the server and discard the invite_room_state. Also, clients must not rely on any particular state being present here; they SHOULD behave properly (with possibly a degraded but not a broken experience) in the absence of any particular events here. If they are set on the room, at least the state for ``m.room.avatar``, ``m.room.canonical_alias``, ``m.room.join_rules``, and ``m.room.name`` SHOULD be included.' + items: + description: 'A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.' + properties: + content: + description: The ``content`` for the event. + title: EventContent + type: object + state_key: + description: The ``state_key`` for the event. + type: string + type: + description: The ``type`` for the event. + type: string + required: + - type + - state_key + - content + title: StrippedState + type: object + type: array required: - membership title: EventContent type: object - invite_room_state: - description: 'A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``. Note that this state is informational, and SHOULD NOT be trusted; once the client has joined the room, it SHOULD fetch the live state from the server and discard the invite_room_state. Also, clients must not rely on any particular state being present here; they SHOULD behave properly (with possibly a degraded but not a broken experience) in the absence of any particular events here. If they are set on the room, at least the state for ``m.room.avatar``, ``m.room.canonical_alias``, ``m.room.join_rules``, and ``m.room.name`` SHOULD be included.' - items: - description: 'A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.' - properties: - content: - description: The ``content`` for the event. - title: EventContent - type: object - state_key: - description: The ``state_key`` for the event. - type: string - type: - description: The ``type`` for the event. - type: string - required: - - type - - state_key - - content - title: StrippedState - type: object - type: array state_key: description: The ``user_id`` this membership event relates to. type: string diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 90a87cd47..81da3f6ab 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -875,15 +875,6 @@ class MatrixUnits(Units): Units.prop(json_schema, "properties/content") ) - # This is horrible because we're special casing a key on m.room.member. - # We need to do this because we want to document a non-content object. - if schema["type"] == "m.room.member": - invite_room_state = get_tables_for_schema( - json_schema["properties"]["invite_room_state"]["items"], - ) - schema["content_fields"].extend(invite_room_state) - - # grab msgtype if it is the right kind of event msgtype = Units.prop( json_schema, "properties/content/properties/msgtype/enum" From 5ceb1321107f9ff0760902e4aa4287da59c8510d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 20:52:59 -0600 Subject: [PATCH 116/166] Changelog --- changelogs/client_server/newsfragments/1568.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1568.clarification diff --git a/changelogs/client_server/newsfragments/1568.clarification b/changelogs/client_server/newsfragments/1568.clarification new file mode 100644 index 000000000..4b7a6eafb --- /dev/null +++ b/changelogs/client_server/newsfragments/1568.clarification @@ -0,0 +1 @@ +Move the ``invite_room_state`` definition under ``unsigned`` where it actually resides. From 7d08ef73d09de2e37b9be9d602277c4f336be35b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:19:07 -0600 Subject: [PATCH 117/166] Fix naming of the Filter schemas EventFilter !== Filter Fixes https://github.com/matrix-org/matrix-doc/issues/1509 --- api/client-server/definitions/event_filter.yaml | 2 +- api/client-server/definitions/sync_filter.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/client-server/definitions/event_filter.yaml b/api/client-server/definitions/event_filter.yaml index 1cae3ea90..8c96917fd 100644 --- a/api/client-server/definitions/event_filter.yaml +++ b/api/client-server/definitions/event_filter.yaml @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -title: Filter +title: EventFilter properties: limit: description: The maximum number of events to return. diff --git a/api/client-server/definitions/sync_filter.yaml b/api/client-server/definitions/sync_filter.yaml index 69b245a31..33bead262 100644 --- a/api/client-server/definitions/sync_filter.yaml +++ b/api/client-server/definitions/sync_filter.yaml @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +type: object +title: Filter properties: event_fields: description: List of event fields to include. If this list is absent then all @@ -40,6 +42,7 @@ properties: room: title: RoomFilter description: Filters to be applied to room data. + type: object properties: not_rooms: description: A list of room IDs to exclude. If this list is absent then no rooms @@ -76,5 +79,3 @@ properties: allOf: - $ref: room_event_filter.yaml description: The per user account data to include for rooms. - type: object -type: object From 97e3dd443b5bc38a8ec166b9ae3642460f960e1c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:20:01 -0600 Subject: [PATCH 118/166] Update room_event_filter.yaml to use the OpenAPI allOf definition This is just maintenance. --- .../definitions/room_event_filter.yaml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/api/client-server/definitions/room_event_filter.yaml b/api/client-server/definitions/room_event_filter.yaml index 7d9184b58..9817db0c0 100644 --- a/api/client-server/definitions/room_event_filter.yaml +++ b/api/client-server/definitions/room_event_filter.yaml @@ -13,23 +13,23 @@ # limitations under the License. allOf: - $ref: event_filter.yaml -title: RoomEventFilter -properties: - not_rooms: - description: A list of room IDs to exclude. If this list is absent then no rooms - are excluded. A matching room will be excluded even if it is listed in the ``'rooms'`` - filter. - items: - type: string - type: array - rooms: - description: A list of room IDs to include. If this list is absent then all rooms - are included. - items: - type: string - type: array - contains_url: - type: boolean +- type: object + title: RoomEventFilter + properties: + not_rooms: + description: A list of room IDs to exclude. If this list is absent then no rooms + are excluded. A matching room will be excluded even if it is listed in the ``'rooms'`` + filter. + items: + type: string + type: array + rooms: + description: A list of room IDs to include. If this list is absent then all rooms + are included. + items: + type: string + type: array + contains_url: + type: boolean description: If ``true``, includes only events with a url key in their content. If ``false``, excludes those events. -type: object From 26a7a341f0f56e8ac16edf0b8ee7b57bae26b298 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:20:28 -0600 Subject: [PATCH 119/166] Mark the filter_id in the response of POST /filter as required --- api/client-server/filter.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/client-server/filter.yaml b/api/client-server/filter.yaml index b34da7b62..58d9e55cb 100644 --- a/api/client-server/filter.yaml +++ b/api/client-server/filter.yaml @@ -91,7 +91,12 @@ paths: filter_id: type: string description: |- - The ID of the filter that was created. + The ID of the filter that was created. Cannot start + with a ``{`` as this character is used to determine + if the filter provided is inline JSON or a previously + declared filter by homeservers on some APIs. + example: "66696p746572" + required: ['filter_id'] tags: - Room participation "/user/{userId}/filter/{filterId}": From 1cbcaba2c70fa41bc8f478113932fdcd642957af Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:21:27 -0600 Subject: [PATCH 120/166] Clean up examples in filter.yaml Indentation, excess examples. --- api/client-server/filter.yaml | 48 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/api/client-server/filter.yaml b/api/client-server/filter.yaml index 58d9e55cb..db2151960 100644 --- a/api/client-server/filter.yaml +++ b/api/client-server/filter.yaml @@ -54,37 +54,33 @@ paths: allOf: - $ref: "definitions/sync_filter.yaml" example: { - "room": { - "state": { - "types": ["m.room.*"], - "not_rooms": ["!726s6s6q:example.com"] - }, - "timeline": { - "limit": 10, - "types": ["m.room.message"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - }, - "ephemeral": { - "types": ["m.receipt", "m.typing"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - } + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] }, - "presence": { - "types": ["m.presence"], - "not_senders": ["@alice:example.com"] + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] }, - "event_format": "client", - "event_fields": ["type", "content", "sender"] - } + "ephemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } responses: 200: description: The filter was created. - examples: - application/json: { - "filter_id": "66696p746572" - } schema: type: object properties: From b68ed5d594d37fa0c76e1592805f64e3030b1ca4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:21:49 -0600 Subject: [PATCH 121/166] Define the default for the contains_url filter param Fixes https://github.com/matrix-org/matrix-doc/issues/1553 --- api/client-server/definitions/room_event_filter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/definitions/room_event_filter.yaml b/api/client-server/definitions/room_event_filter.yaml index 9817db0c0..c36b3768f 100644 --- a/api/client-server/definitions/room_event_filter.yaml +++ b/api/client-server/definitions/room_event_filter.yaml @@ -31,5 +31,5 @@ allOf: type: array contains_url: type: boolean - description: If ``true``, includes only events with a url key in their content. If - ``false``, excludes those events. + description: If ``true``, includes only events with a ``url`` key in their content. If + ``false``, excludes those events. Defaults to ``false``. From b0fbd7be7c4cb3abe80cc5b6218a6de0f6fb1ff8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:24:33 -0600 Subject: [PATCH 122/166] Changelog --- changelogs/client_server/newsfragments/1570.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1570.clarification diff --git a/changelogs/client_server/newsfragments/1570.clarification b/changelogs/client_server/newsfragments/1570.clarification new file mode 100644 index 000000000..dbf8a821c --- /dev/null +++ b/changelogs/client_server/newsfragments/1570.clarification @@ -0,0 +1 @@ +Clarify the object structures and defaults for Filters. From 667fa082af81b612748180e92e5fe8e6aad4aa55 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:30:33 -0600 Subject: [PATCH 123/166] Replace applicable types of 'number' to 'integer' `number` implies/represents a float where `integer` does not. The only remaining `type: number` in the project appear on power levels: those have been left untouched pending clarification. Fixes https://github.com/matrix-org/matrix-doc/issues/746 --- api/client-server/admin.yaml | 3 ++- api/client-server/content-repo.yaml | 9 ++++++--- api/client-server/definitions/public_rooms_response.yaml | 4 ++-- api/client-server/list_public_rooms.yaml | 8 ++++---- api/client-server/notifications.yaml | 2 +- api/client-server/registration.yaml | 4 ++-- api/client-server/search.yaml | 4 ++-- api/client-server/users.yaml | 2 +- event-schemas/schema/core-event-schema/room_event.yaml | 3 ++- 9 files changed, 22 insertions(+), 17 deletions(-) diff --git a/api/client-server/admin.yaml b/api/client-server/admin.yaml index 2fdac82b0..09942a101 100644 --- a/api/client-server/admin.yaml +++ b/api/client-server/admin.yaml @@ -105,7 +105,8 @@ paths: type: string description: Most recently seen IP address of the session. last_seen: - type: number + type: integer + format: int64 description: Unix timestamp that the session was last active. user_agent: type: string diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index b3e9517b1..5f4e9111a 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -259,7 +259,8 @@ paths: description: "The URL to get a preview of" required: true - in: query - type: number + type: integer + format: int64 x-example: 1510610716656 name: ts description: |- @@ -276,7 +277,8 @@ paths: type: object properties: "matrix:image:size": - type: number + type: integer + format: int64 description: |- The byte-size of the image. Omitted if there is no image attached. "og:image": @@ -324,7 +326,8 @@ paths: type: object properties: m.upload.size: - type: number + type: integer + format: int64 description: |- The maximum size an upload can be in bytes. Clients SHOULD use this as a guide when uploading content. diff --git a/api/client-server/definitions/public_rooms_response.yaml b/api/client-server/definitions/public_rooms_response.yaml index fc6ccb44c..ab7010516 100644 --- a/api/client-server/definitions/public_rooms_response.yaml +++ b/api/client-server/definitions/public_rooms_response.yaml @@ -45,7 +45,7 @@ properties: description: |- The name of the room, if any. num_joined_members: - type: number + type: integer description: |- The number of members joined to the room. room_id: @@ -82,7 +82,7 @@ properties: absence of this token means there are no results before this batch, i.e. this is the first batch. total_room_count_estimate: - type: number + type: integer description: |- An estimate on the total number of public rooms, if the server has an estimate. diff --git a/api/client-server/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml index 72a120609..8f0e80d55 100644 --- a/api/client-server/list_public_rooms.yaml +++ b/api/client-server/list_public_rooms.yaml @@ -123,7 +123,7 @@ paths: parameters: - in: query name: limit - type: number + type: integer description: |- Limit the number of results returned. - in: query @@ -173,7 +173,7 @@ paths: type: object properties: limit: - type: number + type: integer description: |- Limit the number of results returned. since: @@ -233,7 +233,7 @@ paths: description: |- The name of the room, if any. num_joined_members: - type: number + type: integer description: |- The number of members joined to the room. room_id: @@ -270,7 +270,7 @@ paths: absence of this token means there are no results before this batch, i.e. this is the first batch. total_room_count_estimate: - type: number + type: integer description: |- An estimate on the total number of public rooms, if the server has an estimate. diff --git a/api/client-server/notifications.yaml b/api/client-server/notifications.yaml index e10e5bfde..b450885b8 100644 --- a/api/client-server/notifications.yaml +++ b/api/client-server/notifications.yaml @@ -45,7 +45,7 @@ paths: required: false x-example: "xxxxx" - in: query - type: number + type: integer name: limit description: Limit on the number of events to return in this request. required: false diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 56da9addb..e4b056291 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -218,7 +218,7 @@ paths: description: The email address example: "example@example.com" send_attempt: - type: number + type: integer description: Used to distinguish protocol level retries from requests to re-send the email. example: 1 required: ["client_secret", "email", "send_attempt"] @@ -283,7 +283,7 @@ paths: description: The phone number. example: "example@example.com" send_attempt: - type: number + type: integer description: Used to distinguish protocol level retries from requests to re-send the SMS message. example: 1 required: ["client_secret", "country", "phone_number", "send_attempt"] diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e4118c32b..0d3a78841 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -179,7 +179,7 @@ paths: description: Mapping of category name to search criteria. properties: count: - type: number + type: integer description: An approximate count of the total number of results found. highlights: type: array @@ -197,7 +197,7 @@ paths: description: The result object. properties: rank: - type: number + type: integer description: A number that describes how closely this result matches the search. Higher is closer. diff --git a/api/client-server/users.yaml b/api/client-server/users.yaml index a682b4358..fc6d233bd 100644 --- a/api/client-server/users.yaml +++ b/api/client-server/users.yaml @@ -47,7 +47,7 @@ paths: description: The term to search for example: "foo" limit: - type: number + type: integer description: The maximum number of results to return (Defaults to 10). example: 10 required: ["search_term"] diff --git a/event-schemas/schema/core-event-schema/room_event.yaml b/event-schemas/schema/core-event-schema/room_event.yaml index a8a23f549..ebf970ad7 100644 --- a/event-schemas/schema/core-event-schema/room_event.yaml +++ b/event-schemas/schema/core-event-schema/room_event.yaml @@ -16,7 +16,8 @@ properties: origin_server_ts: description: Timestamp in milliseconds on originating homeserver when this event was sent. - type: number + type: integer + format: int64 unsigned: description: Contains optional extra information about the event. properties: From e75a1836b86b04e0670c8e648055be5d1d0cf54d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:33:43 -0600 Subject: [PATCH 124/166] Changelog --- changelogs/client_server/newsfragments/1571.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1571.clarification diff --git a/changelogs/client_server/newsfragments/1571.clarification b/changelogs/client_server/newsfragments/1571.clarification new file mode 100644 index 000000000..2410baf39 --- /dev/null +++ b/changelogs/client_server/newsfragments/1571.clarification @@ -0,0 +1 @@ +Clarify instances of ``type: number`` in the swagger/OpenAPI schema definitions. From 008ebb8c1ac1d1958a3809d3757bbb06c66a9718 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:55:30 -0600 Subject: [PATCH 125/166] Add `account_data` to left rooms in /sync Fixes https://github.com/matrix-org/matrix-doc/issues/1392 --- api/client-server/sync.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index 4b44c20e1..f1997b617 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -227,6 +227,14 @@ paths: room up to the point when the user left. allOf: - $ref: "definitions/timeline_batch.yaml" + account_data: + title: Account Data + type: object + description: |- + The private data that this user has attached to + this room. + allOf: + - $ref: "definitions/event_batch.yaml" presence: title: Presence type: object From e8edfba11438a211ec7bcc1cebf1192dffae105b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 21:57:04 -0600 Subject: [PATCH 126/166] Changelog --- changelogs/client_server/newsfragments/1572.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1572.clarification diff --git a/changelogs/client_server/newsfragments/1572.clarification b/changelogs/client_server/newsfragments/1572.clarification new file mode 100644 index 000000000..7e84098f4 --- /dev/null +++ b/changelogs/client_server/newsfragments/1572.clarification @@ -0,0 +1 @@ +Clarify that left rooms also have account data in ``/sync``. From e60b44e27fe9e83e5c6edca9e4c778fd244da0fa Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 22:27:34 -0600 Subject: [PATCH 127/166] Clean up PUT /directory/room Fixes https://github.com/matrix-org/matrix-doc/issues/933 The issue references two problems: a `roomInfo` and lack of a `room_id`. It appears the `room_id` has been fixed since reporting, however the `roomInfo` remained (and is now fixed). --- api/client-server/directory.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/client-server/directory.yaml b/api/client-server/directory.yaml index ee42cf845..78ddfa291 100644 --- a/api/client-server/directory.yaml +++ b/api/client-server/directory.yaml @@ -41,7 +41,7 @@ paths: required: true x-example: "#monkeys:matrix.org" - in: body - name: roomInfo + name: body description: Information about this room alias. required: true schema: @@ -50,24 +50,24 @@ paths: room_id: type: string description: The room ID to set. + required: ['room_id'] example: { - "room_id": "!abnjk1jdasj98:capuchins.com" - } + "room_id": "!abnjk1jdasj98:capuchins.com" + } responses: 200: description: The mapping was created. examples: - application/json: { - } + application/json: {} schema: type: object 409: description: A room alias with that name already exists. examples: application/json: { - "errcode": "M_UNKNOWN", - "error": "Room alias #monkeys:matrix.org already exists." - } + "errcode": "M_UNKNOWN", + "error": "Room alias #monkeys:matrix.org already exists." + } schema: "$ref": "definitions/errors/error.yaml" tags: From ce0befd7d059e1b91389bfa4692cbb4c6dfd2d84 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Aug 2018 22:28:52 -0600 Subject: [PATCH 128/166] Changelog --- changelogs/client_server/newsfragments/1574.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1574.clarification diff --git a/changelogs/client_server/newsfragments/1574.clarification b/changelogs/client_server/newsfragments/1574.clarification new file mode 100644 index 000000000..8d07ef56f --- /dev/null +++ b/changelogs/client_server/newsfragments/1574.clarification @@ -0,0 +1 @@ +Fix naming of the body field in ``PUT /directory/room``. From d7d28f7e5bf9c263fa0b5d9c04638ec02ae6f6dd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 09:16:30 -0600 Subject: [PATCH 129/166] Remove nl as a supported HTML tag --- specification/modules/instant_messaging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 29d33c558..6d1b1b355 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -66,7 +66,7 @@ Clients should limit the HTML they render to avoid Cross-Site Scripting, HTML injection, and similar attacks. The strongly suggested set of HTML tags to permit, denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, ``h2``, ``h3``, ``h4``, ``h5``, ``h6``, ``blockquote``, ``p``, ``a``, ``ul``, -``ol``, ``sup``, ``sub``, ``nl``, ``li``, ``b``, ``i``, ``u``, ``strong``, ``em``, +``ol``, ``sup``, ``sub``, ``li``, ``b``, ``i``, ``u``, ``strong``, ``em``, ``strike``, ``code``, ``hr``, ``br``, ``div``, ``table``, ``thead``, ``tbody``, ``tr``, ``th``, ``td``, ``caption``, ``pre``, ``span``, ``img``. From c7822cc9a8d18ddcdd17473e2369f93bd7a7307c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 09:16:53 -0600 Subject: [PATCH 130/166] Link to the content repo when referencing MXC URIs in images --- specification/modules/instant_messaging.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 6d1b1b355..88326999d 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -89,8 +89,7 @@ for the tag. matching one of: ``https``, ``http``, ``ftp``, ``mailto``, ``magnet``) :``img``: - ``width``, ``height``, ``alt``, ``title``, ``src`` (provided it is a Matrix Content - URI) + ``width``, ``height``, ``alt``, ``title``, ``src`` (provided it is a `Matrix Content (MXC) URI`_) :``ol``: ``start`` @@ -345,3 +344,4 @@ Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross- Scripting (XSS) attacks. This includes room names and topics. .. _`E2E module`: `module:e2e`_ +.. _`Matrix Content (MXC) URI`: `module:content`_ \ No newline at end of file From 5bf99aeb349f0af80e7f4b8f3a0c189304054e92 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 09:17:08 -0600 Subject: [PATCH 131/166] Add a note that formatted_body is not forever --- specification/modules/instant_messaging.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 88326999d..9de890717 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -102,6 +102,9 @@ Additionally, clients should ensure that *all* ``a`` tags get a ``rel="noopener" to prevent the target page from referencing the client's tab/window. +.. Note:: + A future iteration of the specification will support more powerful and extensible + message formatting options, such as the proposal `MSC1225 `_. {{msgtype_events}} From 89daa3c5ce0c745219d8cdb9e92e693fc24f1663 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 09:17:40 -0600 Subject: [PATCH 132/166] Clarify that clients aren't required to render all the tags This commit also includes minor clarifications to surrounding text. --- specification/modules/instant_messaging.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 9de890717..079a4801d 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -98,9 +98,13 @@ for the tag. ``class`` (only classes which start with ``language-`` for syntax highlighting) -Additionally, clients should ensure that *all* ``a`` tags get a ``rel="noopener"`` +Additionally, web clients should ensure that *all* ``a`` tags get a ``rel="noopener"`` to prevent the target page from referencing the client's tab/window. +Tags must not be nested more than 100 levels deep. Clients should only support the subset +of tags they can render, falling back to other representations of the tags where possible. +For example, a client may not be able to render tables correctly and instead could fall +back to rendering tab-delimited text. .. Note:: A future iteration of the specification will support more powerful and extensible From 17bdc0c740612e0476663129617ee11f0026fb86 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Aug 2018 23:21:50 -0600 Subject: [PATCH 133/166] Revert "Fix encrypted event examples" This reverts commit be9f6042e527a44cd1b4315a208380adef7e1507. --- event-schemas/examples/m.room.encrypted#megolm | 2 +- event-schemas/examples/m.room.encrypted#olm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/event-schemas/examples/m.room.encrypted#megolm b/event-schemas/examples/m.room.encrypted#megolm index 726763181..1f9b75208 100644 --- a/event-schemas/examples/m.room.encrypted#megolm +++ b/event-schemas/examples/m.room.encrypted#megolm @@ -1,6 +1,6 @@ { "content": { - "algorithm": "m.megolm.v1.aes-sha256", + "algorithm": "m.megolm.v1.aes-sha2", "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...", "device_id": "RJYKSTBOIE", "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA", diff --git a/event-schemas/examples/m.room.encrypted#olm b/event-schemas/examples/m.room.encrypted#olm index 23f657385..abb23c31e 100644 --- a/event-schemas/examples/m.room.encrypted#olm +++ b/event-schemas/examples/m.room.encrypted#olm @@ -2,7 +2,7 @@ "type": "m.room.encrypted", "sender": "@example:localhost", "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha256", + "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "Szl29ksW/L8yZGWAX+8dY1XyFi+i5wm+DRhTGkbMiwU", "ciphertext": { "7qZcfnBmbEGzxxaWfBjElJuvn7BZx+lSz/SvFrDF/z8": { From 438f5825ae82ff8ac454db26e3076fce6242daf5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 14:55:59 -0600 Subject: [PATCH 134/166] Add more supported encryption algorithms to message events --- event-schemas/schema/m.room.encrypted | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-schemas/schema/m.room.encrypted b/event-schemas/schema/m.room.encrypted index 6825be1d2..9c2fa7be3 100644 --- a/event-schemas/schema/m.room.encrypted +++ b/event-schemas/schema/m.room.encrypted @@ -14,7 +14,9 @@ properties: type: string enum: - m.olm.curve25519-aes-sha256 + - m.olm.v1.curve25519-aes-sha2 - m.megolm.v1.aes-sha + - m.megolm.v1.aes-sha2 description: |- The encryption algorithm used to encrypt this event. The value of this field determines which other properties will be From 72de8bec5c1703d1e425dd29c6463810d3641a0a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 15:05:56 -0600 Subject: [PATCH 135/166] Remove unused algorithms for encrypted messages --- event-schemas/schema/m.room.encrypted | 2 -- 1 file changed, 2 deletions(-) diff --git a/event-schemas/schema/m.room.encrypted b/event-schemas/schema/m.room.encrypted index 9c2fa7be3..44b09c3f6 100644 --- a/event-schemas/schema/m.room.encrypted +++ b/event-schemas/schema/m.room.encrypted @@ -13,9 +13,7 @@ properties: algorithm: type: string enum: - - m.olm.curve25519-aes-sha256 - m.olm.v1.curve25519-aes-sha2 - - m.megolm.v1.aes-sha - m.megolm.v1.aes-sha2 description: |- The encryption algorithm used to encrypt this event. The From 5fa6b493653cdf831183d994f272e7978fda341d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 15:56:47 -0600 Subject: [PATCH 136/166] Specify the type of filter the search API expects Note: This is badly named until https://github.com/matrix-org/matrix-doc/pull/1570 lands Fixes https://github.com/matrix-org/matrix-doc/issues/598 --- api/client-server/search.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e4118c32b..6feb57783 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -41,7 +41,7 @@ paths: type: string description: |- The point to return events from. If given, this should be a - `next_batch` result from a previous call to this endpoint. + ``next_batch`` result from a previous call to this endpoint. x-example: "YWxsCgpOb25lLDM1ODcwOA" - in: body name: body @@ -95,6 +95,7 @@ paths: # for now :/ description: |- This takes a `filter`_. + $ref: "definitions/room_event_filter.yaml" order_by: title: "Ordering" type: string From 7b7933327a2e66ff360858ae8a98cc754bc44ef7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 15:57:51 -0600 Subject: [PATCH 137/166] Changelog --- changelogs/client_server/newsfragments/1577.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1577.clarification diff --git a/changelogs/client_server/newsfragments/1577.clarification b/changelogs/client_server/newsfragments/1577.clarification new file mode 100644 index 000000000..aec3248fe --- /dev/null +++ b/changelogs/client_server/newsfragments/1577.clarification @@ -0,0 +1 @@ +Clarify the filter object schema used in room searching. From be2e0fc9d416348b411c21a8fc394387a1194ebc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 16:12:42 -0600 Subject: [PATCH 138/166] Clarify that ACLs are required to manually deny unsupported hosts --- event-schemas/schema/m.room.server_acl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl index ed64038ca..29a91f4b3 100644 --- a/event-schemas/schema/m.room.server_acl +++ b/event-schemas/schema/m.room.server_acl @@ -4,8 +4,8 @@ description: |- An event to indicate which servers are permitted to participate in the room. Server ACLs may allow or deny groups of hosts. All servers participating in the room, including those that are denied, are expected to uphold the - server ACL. Servers that do not uphold the ACLs are recommended to be - added to the denied hosts list. + server ACL. Servers that do not uphold the ACLs MUST be added to the denied hosts + list in order for the ACLs to remain effective. The ``allow`` and ``deny`` lists are lists of globs supporting ``?`` and ``*`` as wildcards. When comparing against the server ACLs, the suspect server's port @@ -27,6 +27,14 @@ description: |- servers from participating in the room, including the sender. This renders the room unusable. A common allow rule is ``[ "*" ]`` which would still permit the use of the ``deny`` list without losing the room. + + .. WARNING:: + Servers that do not uphold the ACLs MUST be manually appended to the denied hosts + list. To accomplish this, events should have their ``prev_events`` inspected for + denied hosts, therefore detecting servers which are not upholding the ACLs. Server + versions can also be used to detect hosts that will not uphold the ACLs, although + this is less effective. Server ACLs were added in Synapse v0.32.0 although other + server implementations and versions exist in the world. allOf: - $ref: core-event-schema/state_event.yaml type: object From 82be6077ffc941bfce0a26fb4f631618046ea8de Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 16:13:42 -0600 Subject: [PATCH 139/166] Add a note that ACLs don't operate at the auth level; Fix glob definition --- event-schemas/schema/m.room.server_acl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl index 29a91f4b3..9e7ccc3b4 100644 --- a/event-schemas/schema/m.room.server_acl +++ b/event-schemas/schema/m.room.server_acl @@ -22,6 +22,11 @@ description: |- #. If the server name matches an entry in the ``allow`` list, allow. #. Otherwise, deny. + .. Note:: + Server ACLs do not restrict the events relative to the room DAG via authorisation + rules, but instead act purely at the network layer to determine which servers are + allowed to connect and interact with a given room. + .. WARNING:: Failing to provide an ``allow`` rule of some kind will prevent **all** servers from participating in the room, including the sender. This renders @@ -51,7 +56,7 @@ properties: description: |- The server names to allow in the room, excluding any port information. Wildcards may be used to cover a wider range of hosts, where ``*`` - matches zero or more characters and ``?`` matches one or more characters. + matches zero or more characters and ``?`` matches exactly one character. **This defaults to an empty list when not provided, effectively disallowing every server.** @@ -62,7 +67,7 @@ properties: description: |- The server names to disallow in the room, excluding any port information. Wildcards may be used to cover a wider range of hosts, where ``*`` - matches zero or more characters and ``?`` matches one or more characters. + matches zero or more characters and ``?`` matches exactly one character. This defaults to an empty list when not provided. items: From 76afef79f8f09809b8dafecc37b7804f52d610e7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 16:14:37 -0600 Subject: [PATCH 140/166] Clarify the rationale and motive for blanket IP banning and port exclusion --- event-schemas/schema/m.room.server_acl | 4 ++++ specification/modules/server_acls.rst | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl index 9e7ccc3b4..317031cdc 100644 --- a/event-schemas/schema/m.room.server_acl +++ b/event-schemas/schema/m.room.server_acl @@ -51,6 +51,10 @@ properties: description: |- True to allow server names that are IP address literals. False to deny. Defaults to true if missing or otherwise not a boolean. + + This is strongly recommended to be set to ``false`` as servers running + with IP literal names are strongly discouraged in order to require + legitimate homeservers to be backed by a valid registered domain name. allow: type: array description: |- diff --git a/specification/modules/server_acls.rst b/specification/modules/server_acls.rst index 72892fce5..f26b8c6b1 100644 --- a/specification/modules/server_acls.rst +++ b/specification/modules/server_acls.rst @@ -17,7 +17,7 @@ Server Access Control Lists (ACLs) for rooms .. _module:server-acls: -In some scenarios room operators may wish to prevent a malicous or untrusted +In some scenarios room operators may wish to prevent a malicious or untrusted server from participating in their room. Sending an `m.room.server_acl`_ state event into a room is an effective way to prevent the server from participating in the room at the federation level. @@ -30,7 +30,10 @@ similar to setting the ``m.federate`` value on the `m.room.create`_ event. .. Note:: Port numbers are not supported because it is unclear to parsers whether a - port number should be matched or an IP address literal. + port number should be matched or an IP address literal. Additionally, it + is unlikely that one would trust a server running on a particular domain's + port but not a different port, especially considering the server host can + easily change ports. .. Note:: CIDR notation is not supported for IP addresses because Matrix does not From d7397ccd563fb886c48944c1f3869e0d617aadc8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 16:21:10 -0600 Subject: [PATCH 141/166] Provide additional rationale for kicking users when they are ACLd --- specification/modules/server_acls.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/server_acls.rst b/specification/modules/server_acls.rst index f26b8c6b1..2b2d8f351 100644 --- a/specification/modules/server_acls.rst +++ b/specification/modules/server_acls.rst @@ -47,7 +47,8 @@ event. Clients should describe changes to the server ACLs to the user in the user interface, such as in the timeline. Clients may wish to kick affected users from the room prior to denying a server -access to the room to help prevent those servers from participating. +access to the room to help prevent those servers from participating and to +provide feedback to the users that they have been excluded from the room. Server behaviour ---------------- From 313e6de48bb99afefa8adb18d6dd097a4d88a849 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 27 Aug 2018 23:36:48 +0100 Subject: [PATCH 142/166] tweak wording to spell out that handling legacy/noncompliant servers. --- event-schemas/schema/m.room.server_acl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl index 317031cdc..c6adaf054 100644 --- a/event-schemas/schema/m.room.server_acl +++ b/event-schemas/schema/m.room.server_acl @@ -34,12 +34,14 @@ description: |- permit the use of the ``deny`` list without losing the room. .. WARNING:: - Servers that do not uphold the ACLs MUST be manually appended to the denied hosts - list. To accomplish this, events should have their ``prev_events`` inspected for - denied hosts, therefore detecting servers which are not upholding the ACLs. Server - versions can also be used to detect hosts that will not uphold the ACLs, although - this is less effective. Server ACLs were added in Synapse v0.32.0 although other - server implementations and versions exist in the world. + All compliant servers must implement server ACLs. However, legacy or noncompliant + servers exist which do not uphold ACLs, and these MUST be manually appended to + the denied hosts list when setting an ACL to prevent them from leaking events from + banned servers into a room. Currently, the only way to determine noncompliant hosts is + to check the ``prev_events`` of leaked events, therefore detecting servers which + are not upholding the ACLs. Server versions can also be used to try to detect hosts that + will not uphold the ACLs, although this is not comprehensive. Server ACLs were added + in Synapse v0.32.0, although other server implementations and versions exist in the world. allOf: - $ref: core-event-schema/state_event.yaml type: object From bac0392a2d98662b19f32f2955daca145e42a499 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 17:28:08 -0600 Subject: [PATCH 143/166] General clarity for push rule defaults and where to get information Include moving a roaming condition that was under the wrong rule. --- specification/modules/push.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 408eabe62..1972fa17b 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -468,6 +468,10 @@ Definition: ``.m.rule.encrypted_room_one_to_one`` ````````````````````````````````````` Matches any encrypted event sent in a room with exactly two members. +Unlike other push rules, this rule cannot be matched against the content +of the event by nature of it being encrypted. This causes the rule to +be an "all or nothing" match where it either matches *all* events that +are encrypted (in 1:1 rooms) or none. Definition: @@ -492,6 +496,11 @@ Definition: { "set_tweak": "highlight", "value": false + }, + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.encrypted" } ] } @@ -512,11 +521,6 @@ Definition: { "kind": "room_member_count", "is": "2" - }, - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.encrypted" } ], "actions": [ @@ -562,7 +566,10 @@ Definition: ``.m.rule.encrypted`` ````````````````````` -Matches all encrypted events. +Matches all encrypted events. Unlike other push rules, this rule cannot +be matched against the content of the event by nature of it being encrypted. +This causes the rule to be an "all or nothing" match where it either +matches *all* events that are encrypted (in 1:1 rooms) or none. Definition: @@ -628,11 +635,11 @@ rule determines its behaviour. The following conditions are defined: Parameters: - * ``key``: The notification power level to require the sender to have. Refer to - the `m.room.power_levels`_ event schema for information about what the defaults - are and how to interpret the event. The ``key`` is used to look up a specific - notification type from the ``notifications`` object in the power level event - content. + * ``key``: A string that determines the power level the sender must have to trigger + notifications of a given type, such as ``room``. Refer to the `m.room.power_levels`_ + event schema for information about what the defaults are and how to interpret the event. + The ``key`` is used to look up the power level required to send a notification type + from the ``notifications`` object in the power level event content. Unrecognised conditions MUST NOT match any events, effectively making the push rule disabled. From 32ac81c5883ba7e7fff2b896a34d6ad7f829e4fd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 17:41:27 -0600 Subject: [PATCH 144/166] Spelling --- api/application-service/protocols.yaml | 6 +++--- api/client-server/third_party_lookup.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index 376e66c0b..e6489cc5b 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -127,7 +127,7 @@ paths: $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/location/{protocol}": get: - summary: Retreive Matrix-side portal rooms leading to a third party location. + summary: Retrieve Matrix-side portal rooms leading to a third party location. description: |- Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol @@ -180,7 +180,7 @@ paths: get: summary: Reverse-lookup third party locations given a Matrix room alias. description: |- - Retreive an array of third party network locations from a Matrix room + Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias parameters: @@ -225,7 +225,7 @@ paths: get: summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix User ID. + Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - in: query diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index cba9ce22a..3d348df2f 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -73,7 +73,7 @@ paths: $ref: definitions/errors/error.yaml "/thirdparty/location/{protocol}": get: - summary: Retreive Matrix-side portals rooms leading to a third party location. + summary: Retrieve Matrix-side portals rooms leading to a third party location. description: |- Requesting this endpoint with a valid protocol name results in a list of successful mapping results in a JSON array. Each result contains @@ -151,7 +151,7 @@ paths: get: summary: Reverse-lookup third party locations given a Matrix room alias. description: |- - Retreive an array of third party network locations from a Matrix room + Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias security: @@ -181,7 +181,7 @@ paths: get: summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix User ID. + Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID security: - accessToken: [] From 54032964fc89a356fef1d679d94e6e8bbdeabfc6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 17:41:52 -0600 Subject: [PATCH 145/166] Misc clarity for 3rd party appservice protocols/locations --- specification/application_service_api.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 9d467e879..6e3c5374c 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -184,13 +184,14 @@ Third party networks ++++++++++++++++++++ Application services may declare which protocols they support via their registration -file. These networks are generally for third party services such as IRC that the -application service is managing. Application services may populate a Matrix room -directory for their registered protocols, as defined in the Client-Server API Extensions. - -Each protocol may have several "locations". A location within a protocol is a place -in the third party network, such as an IRC channel. Users of the third party network -may also be represented by the application service. +configuration for the homeserver. These networks are generally for third party services +such as IRC that the application service is managing. Application services may populate +a Matrix room directory for their registered protocols, as defined in the Client-Server +API Extensions. + +Each protocol may have several "locations" (also known as "third party locations" or "3PLs"). +A location within a protocol is a place in the third party network, such as an IRC channel. +Users of the third party network may also be represented by the application service. Locations and users can be searched by fields defined by the application service, such as by display name or other attribute. When clients request the homeserver to search From 17e0ef4b91034b0cdb010416225ffac65b4107fc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 17:58:47 -0600 Subject: [PATCH 146/166] Remove empty file and now-empty section from the appservice spec This commit has approval under https://github.com/matrix-org/matrix-doc/pull/1555 although is being included in this branch/PR so the build passes, permitting a merge. --- .../application_service.yaml | 28 ------------------- specification/application_service_api.rst | 9 ------ 2 files changed, 37 deletions(-) delete mode 100644 api/application-service/application_service.yaml diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml deleted file mode 100644 index 67b5b1b5c..000000000 --- a/api/application-service/application_service.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -swagger: '2.0' -info: - title: "Matrix Application Service API" - version: "1.0.0" -host: localhost:8008 -schemes: - - https - - http -basePath: "/" -consumes: - - application/json -produces: - - application/json -paths: diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 48e6896d2..51280341c 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -246,15 +246,6 @@ application service for filtering. {{protocols_as_http_api}} -HTTP APIs -+++++++++ - -This contains application service APIs which are used by the homeserver. All -application services MUST implement these APIs. These APIs are defined below. - -{{application_service_as_http_api}} - - .. _create the user: `sect:asapi-permissions`_ Client-Server API Extensions From 39e674ccb355af61bcfef92be4b557f53db46ea4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 18:27:48 -0600 Subject: [PATCH 147/166] Clarify what matrix.to is and mention that room IDs are not routable Also actually render the warning saying that this scheme is temporary. --- specification/appendices/identifier_grammar.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index eba443e29..e72fdadaa 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -287,8 +287,10 @@ domain). matrix.to navigation ++++++++++++++++++++ -.. NOTE: +.. NOTE:: This namespacing is in place pending a ``matrix://`` (or similar) URI scheme. + This is **not** meant to be interpreted as an available web service - see + below for more details. Rooms, users, aliases, and groups may be represented as a "matrix.to" URI. This URI can be used to reference particular objects in a given context, such @@ -307,14 +309,19 @@ followed by the identifier. Clients should not rely on matrix.to URIs falling back to a web server if accessed and instead should perform some sort of action within the client. For example, if -the user where to click on a matrix.to URI for a room alias, the client may open +the user were to click on a matrix.to URI for a room alias, the client may open a view for the user to participate in the room. Examples of matrix.to URIs are: -* Room: ``https://matrix.to/#/!somewhere:domain.com`` * Room alias: ``https://matrix.to/#/#somewhere:domain.com`` +* Room: ``https://matrix.to/#/!somewhere:domain.com`` * Permalink by room: ``https://matrix.to/#/!somewhere:domain.com/$event:example.org`` * Permalink by room alias: ``https://matrix.to/#/#somewhere:domain.com/$event:example.org`` * User: ``https://matrix.to/#/@alice:example.org`` * Group: ``https://matrix.to/#/+example:domain.com`` + +.. Note:: + Room ID permalinks are unroutable as there is no reliable domain to send requests + to upon receipt of the permalink. Clients should do their best route Room IDs to + where they need to go, however they should also be aware of `issue #1579 `_. \ No newline at end of file From 439b9d2925b306652a75240173fb9f0aec161dd7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 18:59:35 -0600 Subject: [PATCH 148/166] Power levels are also integers --- event-schemas/schema/m.room.power_levels | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index 13a44c709..b00d86a9a 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -46,10 +46,10 @@ properties: properties: ban: description: The level required to ban a user. Defaults to 50 if unspecified. - type: number + type: integer events: additionalProperties: - type: number + type: integer description: The level required to send specific event types. This is a mapping from event type to power level required. title: Event power levels type: object @@ -57,25 +57,25 @@ properties: description: |- The default level required to send message events. Can be overridden by the ``events`` key. Defaults to 0 if unspecified. - type: number + type: integer invite: description: The level required to invite a user. Defaults to 50 if unspecified. - type: number + type: integer kick: description: The level required to kick a user. Defaults to 50 if unspecified. - type: number + type: integer redact: description: The level required to redact an event. Defaults to 50 if unspecified. - type: number + type: integer state_default: description: |- The default level required to send state events. Can be overridden by the ``events`` key. Defaults to 50 if unspecified, but 0 if there is no ``m.room.power_levels`` event at all. - type: number + type: integer users: additionalProperties: - type: number + type: integer description: The power levels for specific users. This is a mapping from ``user_id`` to power level for that user. title: User power levels type: object @@ -84,7 +84,7 @@ properties: The default power level for every user in the room, unless their ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if unspecified. - type: number + type: integer type: object state_key: description: A zero-length string. From fc1fdc95afb96d3b74844355d4ceada7a96b9bfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Aug 2018 17:29:58 +0100 Subject: [PATCH 149/166] Specify a limit on the number of EDUs and PDUs a transaction can contain --- api/server-server/definitions/transaction.yaml | 2 +- api/server-server/transactions.yaml | 4 ++-- specification/server_server_api.rst | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index 7df8b6464..9833f7852 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -31,7 +31,7 @@ properties: example: 1532991320875 pdus: type: array - description: List of persistent updates to rooms. + description: List of persistent updates to rooms. Must not include more than 50 PDUs. items: $ref: "pdu.yaml" required: ['origin', 'origin_server_ts', 'pdus'] diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 8d810ad59..ad10ec0b9 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -60,8 +60,8 @@ paths: edus: type: array description: |- - List of ephemeral messages. May be omitted if there are no ephemeral - messages to be sent. + List of ephemeral messages. May be omitted if there are no ephemeral + messages to be sent. Must not include more than 100 EDUs. items: $ref: "definitions/edu.yaml" example: { diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 439b35f90..f281c21a2 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -262,6 +262,8 @@ of Transaction messages, which are encoded as JSON objects, passed over an HTTP PUT request. A Transaction is meaningful only to the pair of homeservers that exchanged it; they are not globally-meaningful. +Transactions are limited in size; they can have at most 50 PDUs and 100 EDUs. + {{transactions_ss_http_api}} PDUs From ee3b0f42dbc566e911b6fc0c193cb160a653c7e4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 10:34:49 -0600 Subject: [PATCH 150/166] Fix server ACL schema: The type is a string It cannot be an enum otherwise the build starts screaming. --- event-schemas/schema/m.room.server_acl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/m.room.server_acl b/event-schemas/schema/m.room.server_acl index c6adaf054..86d83832e 100644 --- a/event-schemas/schema/m.room.server_acl +++ b/event-schemas/schema/m.room.server_acl @@ -85,4 +85,4 @@ properties: type: string type: enum: ['m.room.server_acl'] - type: enum + type: string From 80edda1666063f574c211a0c992ad1ff1ec1dded Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 10:35:54 -0600 Subject: [PATCH 151/166] Actually run the check-docs circle job --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f79449f38..22a5a90a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,6 +78,7 @@ workflows: jobs: - build-docs - build-swagger + - check-docs notify: webhooks: From e9579a7840692529f5fd32548cfd1beb22ee1aff Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 11:01:43 -0600 Subject: [PATCH 152/166] The `rank` in search results is actually a floating point number This was accidentally changed in https://github.com/matrix-org/matrix-doc/pull/1571 and appears to be the only instance. --- api/client-server/search.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 6594a9139..4a5f45150 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -198,7 +198,7 @@ paths: description: The result object. properties: rank: - type: integer + type: number description: A number that describes how closely this result matches the search. Higher is closer. From 17ae84d06443405e2dce679021f80f5fb158779c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 11:05:59 -0600 Subject: [PATCH 153/166] Check the API examples too --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22a5a90a3..776375834 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,8 @@ checkexamples: &checkexamples source /env/bin/activate cd event-schemas ./check_examples.py + cd ../api + ./check_examples.py genmatrixassets: &genmatrixassets name: Generate/Verify matrix.org assets From 791a2f2b171b4bd7762a7b051317d5790fd678ce Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 12:04:20 -0600 Subject: [PATCH 154/166] Run the validator on the spec --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 776375834..785941c7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,9 +38,21 @@ genmatrixassets: &genmatrixassets source /env/bin/activate ./scripts/generate-matrix-org-assets +validateapi: &validateapi + name: Validate OpenAPI specifications + command: | + cd api + npm install + node validator.js -s "client-server" version: 2 jobs: + validate-docs: + docker: + - image: node:alpine + steps: + - checkout + - run: *validateapi check-docs: docker: - image: uhoreg/matrix-doc-build @@ -81,6 +93,7 @@ workflows: - build-docs - build-swagger - check-docs + - validate-docs notify: webhooks: From ad068bcd2269df4c11fb79f7c0ce571332341251 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 12:11:36 -0600 Subject: [PATCH 155/166] Fix the appservice directory visibility type parameter type --- api/client-server/appservice_room_directory.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/appservice_room_directory.yaml b/api/client-server/appservice_room_directory.yaml index 0225ecd85..49393cd4f 100644 --- a/api/client-server/appservice_room_directory.yaml +++ b/api/client-server/appservice_room_directory.yaml @@ -62,11 +62,12 @@ paths: x-example: "!somewhere:domain.com" - in: body name: body + required: true schema: type: object properties: visibility: - type: enum + type: string enum: ["public", "private"] description: |- Whether the room should be visible (public) in the directory From 349696fc1d3ce0cfe0bb7b70aeaec3b362573512 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 12:16:29 -0600 Subject: [PATCH 156/166] Test building of the speculator and continuserv --- .circleci/config.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 785941c7d..d92b88901 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,18 @@ validateapi: &validateapi npm install node validator.js -s "client-server" +buildspeculator: &buildspeculator + name: Build Speculator + command: | + cd scripts/speculator + go build + +buildcontinuserv: &buildcontinuserv + name: Build Continuserv + command: | + cd scripts/continuserv + go build + version: 2 jobs: validate-docs: @@ -71,7 +83,6 @@ jobs: - run: name: "Doc build is available at:" command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL - build-swagger: docker: - image: uhoreg/matrix-doc-build @@ -84,6 +95,18 @@ jobs: - run: name: "Swagger UI is available at:" command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/api/client-server/index.html"; echo $DOCS_URL + build-dev-scripts: + docker: + - image: golang:1.8 + steps: + - checkout + - run: + name: Install Dependencies + command: | + go get github.com/hashicorp/golang-lru + go get gopkg.in/fsnotify/fsnotify.v1 + - run: *buildcontinuserv + - run: *buildspeculator workflows: version: 2 From e97a1b4af029a672905ba8c0ff8102ca9c50c9ae Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 12:17:41 -0600 Subject: [PATCH 157/166] Actually add the dev scripts build to the workflow --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d92b88901..4a8505b7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -117,6 +117,7 @@ workflows: - build-swagger - check-docs - validate-docs + - build-dev-scripts notify: webhooks: From 132c5b0f48d2e29bbefed62ff1108ae6aba34957 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 12:20:30 -0600 Subject: [PATCH 158/166] Verbose building for go scripts --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a8505b7e..659380b05 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,13 +49,13 @@ buildspeculator: &buildspeculator name: Build Speculator command: | cd scripts/speculator - go build + go build -v buildcontinuserv: &buildcontinuserv name: Build Continuserv command: | cd scripts/continuserv - go build + go build -v version: 2 jobs: @@ -103,8 +103,8 @@ jobs: - run: name: Install Dependencies command: | - go get github.com/hashicorp/golang-lru - go get gopkg.in/fsnotify/fsnotify.v1 + go get -v github.com/hashicorp/golang-lru + go get -v gopkg.in/fsnotify/fsnotify.v1 - run: *buildcontinuserv - run: *buildspeculator From c297c6a35d831f52e0518952725c0992858219e1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 13:06:52 -0600 Subject: [PATCH 159/166] Update schemas and auth rules to cover the @ state key restriction Fixes https://github.com/matrix-org/matrix-doc/issues/1305 Also fixes an issue regarding the `_` being restricted previously, which is false. --- event-schemas/schema/core-event-schema/state_event.yaml | 6 +++++- event-schemas/schema/m.room.member | 5 ++++- specification/server_server_api.rst | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/event-schemas/schema/core-event-schema/state_event.yaml b/event-schemas/schema/core-event-schema/state_event.yaml index 020e9087a..71c4137b7 100644 --- a/event-schemas/schema/core-event-schema/state_event.yaml +++ b/event-schemas/schema/core-event-schema/state_event.yaml @@ -11,7 +11,11 @@ properties: state_key: description: A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this - key makes this event a State Event. The key MUST NOT start with '_'. + key makes this event a State Event. + + State keys starting with an ``@`` are reserved for referencing user IDs, such + as room members. With the exception of a few events, state events set with a + given user's ID as the state key MUST only be set by that user. type: string required: - state_key diff --git a/event-schemas/schema/m.room.member b/event-schemas/schema/m.room.member index 5fb5356d8..de14644d4 100644 --- a/event-schemas/schema/m.room.member +++ b/event-schemas/schema/m.room.member @@ -105,7 +105,10 @@ properties: title: EventContent type: object state_key: - description: The ``user_id`` this membership event relates to. + description: |- + The ``user_id`` this membership event relates to. In all cases except for when ``membership`` is + ``join``, the user ID sending the event does not need to match the user ID in the ``state_key``, + unlike other events. Regular authorisation rules still apply. type: string type: enum: diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 439b35f90..a66f249c6 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -450,7 +450,10 @@ The rules are as follows: #. Otherwise, reject. -7. Otherwise, allow. +7. If the ``state_key`` starts with ``@`` and the ``state_key`` does not match + the ``sender``, reject. + +8. Otherwise, allow. .. NOTE:: From 85b9769cd9ea101f7d6b10259a983e0b620c9e61 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 14:06:55 -0600 Subject: [PATCH 160/166] Comment out the timestamp massaging section for now Pending discussion on https://github.com/matrix-org/matrix-doc/issues/1585 --- specification/application_service_api.rst | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 51280341c..0da00b87d 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -289,26 +289,27 @@ An example request would be:: GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org Authorization: Bearer YourApplicationServiceTokenHere +.. TODO-TravisR: Temporarily take out timestamp massaging while we're releasing r0. + See https://github.com/matrix-org/matrix-doc/issues/1585 +.. Timestamp massaging + +++++++++++++++++++ + The application service may want to inject events at a certain time (reflecting + the time on the network they are tracking e.g. irc, xmpp). Application services + need to be able to adjust the ``origin_server_ts`` value to do this. -Timestamp massaging -+++++++++++++++++++ -The application service may want to inject events at a certain time (reflecting -the time on the network they are tracking e.g. irc, xmpp). Application services -need to be able to adjust the ``origin_server_ts`` value to do this. + Inputs: + - Application service token (``as_token``) + - Desired timestamp (in milliseconds since the unix epoch) -Inputs: - - Application service token (``as_token``) - - Desired timestamp (in milliseconds since the unix epoch) - -Notes: - - This will only apply when sending events. + Notes: + - This will only apply when sending events. -:: + :: - PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 - Authorization: Bearer YourApplicationServiceTokenHere + PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 + Authorization: Bearer YourApplicationServiceTokenHere - Content: The event to send, as per the Client-Server API. + Content: The event to send, as per the Client-Server API. Server admin style permissions ++++++++++++++++++++++++++++++ From f15eafae7fbc2f38bc8142c7067fbbda4e654b4a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 09:26:31 +0100 Subject: [PATCH 161/166] Remove trailing spaces --- specification/server_server_api.rst | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 439b35f90..b4da56322 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -26,9 +26,9 @@ to communicate with each other. Homeservers use these APIs to push messages to each other in real-time, to retrieve historic messages from each other, and to query profile and presence information about users on each other's servers. -The APIs are implemented using HTTPS requests between each of the servers. -These HTTPS requests are strongly authenticated using public key signatures -at the TLS transport layer and using public key signatures in HTTP +The APIs are implemented using HTTPS requests between each of the servers. +These HTTPS requests are strongly authenticated using public key signatures +at the TLS transport layer and using public key signatures in HTTP Authorization headers at the HTTP layer. There are three main kinds of communication that occur between homeservers: @@ -121,7 +121,7 @@ Retrieving Server Keys Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}``. Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a -``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers +``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers query the ``/_matrix/key/v2/server/{keyId}`` API on behalf of another server and sign the response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. @@ -590,9 +590,9 @@ To cover this case, the federation API provides a server-to-server analog of the ``/messages`` client API, allowing one homeserver to fetch history from another. This is the ``/backfill`` API. -To request more history, the requesting homeserver picks another homeserver -that it thinks may have more (most likely this should be a homeserver for -some of the existing users in the room at the earliest point in history it +To request more history, the requesting homeserver picks another homeserver +that it thinks may have more (most likely this should be a homeserver for +some of the existing users in the room at the earliest point in history it has currently), and makes a ``/backfill`` request. Similar to backfilling a room's history, a server may not have all the events @@ -669,10 +669,10 @@ homeservers, though most in practice will use just two. The first part of the handshake usually involves using the directory server to request the room ID and join candidates through the |/query/directory|_ API endpoint. In the case of a new user joining a room as a result of a received -invite, the joining user's homeserver could optimise this step away by picking -the origin server of that invite message as the join candidate. However, the +invite, the joining user's homeserver could optimise this step away by picking +the origin server of that invite message as the join candidate. However, the joining server should be aware that the origin server of the invite might since -have left the room, so should be prepared to fall back on the regular join flow +have left the room, so should be prepared to fall back on the regular join flow if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs @@ -692,7 +692,7 @@ event to a resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the newly-joined room. The resident server must also send the event to other servers -participating in the room. +participating in the room. {{joins_ss_http_api}} @@ -716,8 +716,8 @@ Leaving Rooms (Rejecting Invites) Normally homeservers can send appropriate ``m.room.member`` events to have users leave the room, or to reject local invites. Remote invites from other homeservers -do not involve the server in the graph and therefore need another approach to -reject the invite. Joining the room and promptly leaving is not recommended as +do not involve the server in the graph and therefore need another approach to +reject the invite. Joining the room and promptly leaving is not recommended as clients and servers will interpret that as accepting the invite, then leaving the room rather than rejecting the invite. @@ -829,7 +829,7 @@ EDUs. There are no PDUs or Federation Queries involved. Servers should only send presence updates for users that the receiving server would be interested in. This can include the receiving server sharing a room -with a given user, or a user on the receiving server has added one of the +with a given user, or a user on the receiving server has added one of the sending server's users to their presence list. Clients may define lists of users that they are interested in via "Presence @@ -848,7 +848,7 @@ or ``m.presence_deny`` EDU back. {{definition_ss_event_schemas_m_presence_invite}} -{{definition_ss_event_schemas_m_presence_accept}} +{{definition_ss_event_schemas_m_presence_accept}} {{definition_ss_event_schemas_m_presence_deny}} @@ -881,11 +881,11 @@ that can be made. OpenID ------ -Third party services can exchange an access token previously generated by the +Third party services can exchange an access token previously generated by the `Client-Server API` for information about a user. This can help verify that a user is who they say they are without granting full access to the user's account. -Access tokens generated by the OpenID API are only good for the OpenID API and +Access tokens generated by the OpenID API are only good for the OpenID API and nothing else. {{openid_ss_http_api}} From 25b34e1d7bb06188f6ecb9bedef527082a06d4a6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 09:55:43 -0600 Subject: [PATCH 162/166] Mention that ts massaging was in a draft, but not in the release --- specification/application_service_api.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 0da00b87d..5b7abf4a5 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -311,6 +311,16 @@ An example request would be:: Content: The event to send, as per the Client-Server API. +Timestamp massaging ++++++++++++++++++++ + +Previous drafts of the Application Service API permitted application services +to alter the timestamp of their sent events by providing a ``ts`` query parameter +when sending an event. This API has been excluded from the first release due to +design concerns, however some servers may still support the feature. Please visit +`issue #1585 `_ for more +information. + Server admin style permissions ++++++++++++++++++++++++++++++ From 90fe395aeb51390c16422df1474b9c206b0a7f99 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 10:26:02 -0600 Subject: [PATCH 163/166] Take out the @ state_key restriction from the auth rules This is being handled in https://github.com/matrix-org/matrix-doc/pull/1591 --- specification/server_server_api.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index a66f249c6..439b35f90 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -450,10 +450,7 @@ The rules are as follows: #. Otherwise, reject. -7. If the ``state_key`` starts with ``@`` and the ``state_key`` does not match - the ``sender``, reject. - -8. Otherwise, allow. +7. Otherwise, allow. .. NOTE:: From 8e88d82a4b66f4b18aa32cd5b8700da786350aaf Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Aug 2018 19:15:05 +0100 Subject: [PATCH 164/166] fix typo in anchor. fixes #1603 --- specification/modules/send_to_device.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/send_to_device.rst b/specification/modules/send_to_device.rst index 232becae9..862885461 100644 --- a/specification/modules/send_to_device.rst +++ b/specification/modules/send_to_device.rst @@ -63,7 +63,7 @@ If the client sends messages to users on remote domains, those messages should be sent on to the remote servers via `federation`_. -.. _`federation`: ../server_server/latest.html#send-to-device-messages +.. _`federation`: ../server_server/latest.html#send-to-device-messaging .. TODO-spec: From c83da453b565a6f1c7c5a4a8c0e4459d5aa19795 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 14:18:41 -0600 Subject: [PATCH 165/166] s/number/integer --- event-schemas/schema/m.room.power_levels | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index ae4e5589e..9bb12993a 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -88,10 +88,10 @@ properties: notifications: properties: room: - type: number + type: integer description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. additionalProperties: - type: number + type: integer description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. From 684d80c4228f87b0268581f3f52660e8fbbbb165 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 17:52:02 -0600 Subject: [PATCH 166/166] Revert changes to 3pid lookup types in the IS spec The validator doesn't know what a "3PID Medium" is, for example, so it throws exceptions. This does reduce clarity in the spec though. --- api/identity/lookup.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index 6f993ac7a..f04436ef3 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -121,8 +121,11 @@ paths: minItems: 2 maxItems: 2 items: - - type: 3PID Medium - - type: 3PID Address + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + - type: string + - type: string description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. required: - "threepids" @@ -147,9 +150,13 @@ paths: minItems: 3 maxItems: 3 items: - - type: 3PID Medium - - type: 3PID Address - - type: Matrix User ID + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + #- type: Matrix User ID + - type: string + - type: string + - type: string description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. required: - "threepids"