From 30d56691b1068320eb7c989484c4d06cf461b032 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 01:47:55 +0100 Subject: [PATCH 01/10] document device list synchronisation over federation. untested closes MSC1212 --- .../event-schemas/m.device_list_update.yaml | 89 +++++++++++++++++++ api/server-server/user_devices.yaml | 84 +++++++++++++++++ specification/server_server_api.rst | 42 +++++++++ 3 files changed, 215 insertions(+) create mode 100644 api/server-server/definitions/event-schemas/m.device_list_update.yaml create mode 100644 api/server-server/user_devices.yaml diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml new file mode 100644 index 00000000..272cfee6 --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -0,0 +1,89 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: m.device_list_update +description: |- + An EDU that lets servers push details to each other when one of their users + adds a new device to their account, required for E2E encryption to correctly + target the current set of devices for a given user. + +# FIXME: It's very unclear why we have this API surface for synchronising +# device lists, rather than just using a room (which could also be used for +# presence lists, profile info, etc). But documenting what we have for now... + +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: enum + enum: ['m.device_list_update'] + description: The string ``m.device_list_update`` + example: "m.device_list_update" + content: + type: object + description: The description of the device whose details has changed + title: Device List Update + properties: + user_id: + type: string + description: The user ID who owns this device + example: "@john:example.com" + device_id: + type: string + description: The ID of the device whose details are changing + example: "QBUAZIFURK" + device_display_name: + type: string + description: |- + The public human-readable name of this device. Will be absent + if the device has no name + example: "Mobile" + stream_id: + type: integer + description: |- + An ID sent by the server for this update, unique for a given + user_id. Used to identify any gaps in the sequence of m.device_list_update + EDUs broadcast by a server. + example: 6 + prev_id: + type: array + description: |- + The stream_ids of any prior m.device_list_update EDUs sent for this user + which have not been referred to already in an EDU's prev_id field. If the + receiving server does not recognise any of the prev_ids, it means an EDU + has been lost and the server should query a snapshot of the devicelist + via /user/keys/query in order to correctly interpret future m.device_list_update + EDUs. May be missing or empty for the first EDU in a sequence. + items: + type: integer + description: |- + The stream_id of a prior EDU in this sequence which has not been referred + to already in an EDU's prev_id field. + example: 5 + deleted: + type: boolean + description: |- + True if the server is announcing that this device has been deleted. + keys: + description: |- + The updated identity keys (if any) for this device. May be absent if the + device has no E2E keys defined. + allOf: + - $ref: definitions/device_keys.yaml + required: + - user_id + - device_id + - stream_id diff --git a/api/server-server/user_devices.yaml b/api/server-server/user_devices.yaml new file mode 100644 index 00000000..d1644270 --- /dev/null +++ b/api/server-server/user_devices.yaml @@ -0,0 +1,84 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation User Device Management API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/user/devices/{userId}": + get: + summary: Gets all of the user's devices + description: Gets information on all of the user's devices + operationId: getUserDevices + security: + - signedRequest: [] + parameters: + - in: path + name: userId + type: string + required: true + description: |- + The user ID to retrieve devices for. Must be a user local to the + receiving homeserver. + required: true + x-example: "@alice:example.org" + responses: + 200: + description: The user's devices. + schema: + type: object + properties: + user_id: + type: string + description: The user ID devices were requested for. + example: "@alice:example.org" + stream_id: + type: integer + description: |- + An ID the requesting homeserver may use to detect changes in the + device list. This should increase as time goes on, and always + produce the same ``devices`` list if not incremented. + example: 334608 + devices: + type: array + description: The user's devices. May be empty. + items: + type: object + title: User Device + properties: + device_id: + type: string + description: The device ID. + example: "JLAFKJWSCS" + keys: + type: object + description: Identity keys for the device. + $ref: "../client-server/definitions/device_keys.yaml" + device_display_name: + type: string + description: Optional display name for the device. + example: "Alice's Mobile Phone" + required: ['device_id', 'keys'] + required: ['user_id', 'stream_id', 'devices'] \ No newline at end of file diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 36d7d5d4..e9ee476f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -996,6 +996,48 @@ nothing else. {{openid_ss_http_api}} +Device Management +----------------- + +Details of a user's devices must be efficiently published to other users and kept +up-to-date. This is critical for reliable end-to-end encryption, in order for users +to know which devices are participating in a room. It's also required for to-device +messaging to work. This section is intended to complement the `Device Management module`_ +of the Client-Server API. + +Matrix currently uses a custom pubsub system for synchronising information +about the list of devices for a given user over federation. When a server +wishes to determine a remote user's device list for the first time, +it should populate its local cache by calling the /user/keys/query API +on the remote server. However, subsequent updates to the cache should be applied +by consuming ``m.device_list_update`` EDUs, which must be sent by the remote server +whenever a user's device list changes. + +Servers send ``m.device_list_update`` EDUs in a sequence per source user, each with +a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s) +that this update is relative to in the `pprev_id`` field. To simplify implementation +for clustered servers which could send multiple EDUs at the same time, the ``prev_id`` +field should include all ``m.device_list_update`` EDUs which have not been yet been +referenced in a EDU. If EDUs are emitted in series by a server, there should only ever +be one ``prev_id`` in the EDU. + +This forms a simple directed acyclic graph of ``m.device_list_update`` EDUs, showing +which EDUs a server needs to have received in order to apply an update to its local +copy of the remote user's device list. If a server receives an EDU which refers to +a ``prev_id`` it does not recognise, it must resynchronise its list by calling the +/user/keys/query API and resume the process. + +..TODO: how do you synchronise the result of the /query API with the stream_id + in the EDUs? + +..TODO: this whole thing desperately feels like it should just be state in a room, + rather than inventing a whole different DAG. The same room could be used for + profiles, presence lists, etc. + +{{user_devices_ss_http_api}} + +{{definition_ss_event_schemas_m_device_list_update}} + End-to-End Encryption --------------------- From d377bedd76a0f67412f208f82997f0bf5c67cecc Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 01:54:13 +0100 Subject: [PATCH 02/10] fix path --- .../definitions/event-schemas/m.device_list_update.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml index 272cfee6..de2f42d7 100644 --- a/api/server-server/definitions/event-schemas/m.device_list_update.yaml +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -82,7 +82,7 @@ allOf: The updated identity keys (if any) for this device. May be absent if the device has no E2E keys defined. allOf: - - $ref: definitions/device_keys.yaml + - $ref: ../../../client-server/definitions/device_keys.yaml required: - user_id - device_id From 0afdcc59ea092f4cda10963c14dde79be531da4b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 02:10:45 +0100 Subject: [PATCH 03/10] spell out stream_id and fix todo --- api/server-server/user_devices.yaml | 9 +++++---- specification/server_server_api.rst | 8 +++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/api/server-server/user_devices.yaml b/api/server-server/user_devices.yaml index d1644270..4805deb4 100644 --- a/api/server-server/user_devices.yaml +++ b/api/server-server/user_devices.yaml @@ -57,10 +57,11 @@ paths: stream_id: type: integer description: |- - An ID the requesting homeserver may use to detect changes in the - device list. This should increase as time goes on, and always - produce the same ``devices`` list if not incremented. - example: 334608 + A unique ID for a given user_id which describes the version of + the returned device list. This is matched with the ``stream_id`` + field in ``m.device_list_update`` EDUs in order to incrementally + update the returned device_list. + example: 5 devices: type: array description: The user's devices. May be empty. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e9ee476f..45ac1c9f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1025,12 +1025,10 @@ This forms a simple directed acyclic graph of ``m.device_list_update`` EDUs, sho which EDUs a server needs to have received in order to apply an update to its local copy of the remote user's device list. If a server receives an EDU which refers to a ``prev_id`` it does not recognise, it must resynchronise its list by calling the -/user/keys/query API and resume the process. +/user/keys/query API and resume the process. The response contains a ``stream_id`` +which should be used to correlate with subsequent ``m.device_list_update`` EDUs. -..TODO: how do you synchronise the result of the /query API with the stream_id - in the EDUs? - -..TODO: this whole thing desperately feels like it should just be state in a room, +.. TODO: this whole thing desperately feels like it should just be state in a room, rather than inventing a whole different DAG. The same room could be used for profiles, presence lists, etc. From 0ee2cf628682ae93ba7314e984a82efdaf74b8f0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 02:14:36 +0100 Subject: [PATCH 04/10] fix device_list_update example hopefully --- .../definitions/event-schemas/m.device_list_update.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml index de2f42d7..d572e393 100644 --- a/api/server-server/definitions/event-schemas/m.device_list_update.yaml +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -77,6 +77,7 @@ allOf: type: boolean description: |- True if the server is announcing that this device has been deleted. + example: false keys: description: |- The updated identity keys (if any) for this device. May be absent if the From b8608495e9c8365b41e739588ae6063dce1a7a9a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 02:51:24 +0100 Subject: [PATCH 05/10] periods --- .../definitions/event-schemas/m.device_list_update.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml index d572e393..0e7f0d32 100644 --- a/api/server-server/definitions/event-schemas/m.device_list_update.yaml +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -34,22 +34,22 @@ allOf: example: "m.device_list_update" content: type: object - description: The description of the device whose details has changed + description: The description of the device whose details has changed. title: Device List Update properties: user_id: type: string - description: The user ID who owns this device + description: The user ID who owns this device. example: "@john:example.com" device_id: type: string - description: The ID of the device whose details are changing + description: The ID of the device whose details are changing. example: "QBUAZIFURK" device_display_name: type: string description: |- The public human-readable name of this device. Will be absent - if the device has no name + if the device has no name. example: "Mobile" stream_id: type: integer From 928fc5cbe5e5cfa3992ccb275d93751706fb1723 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 1 Sep 2018 03:08:53 +0100 Subject: [PATCH 06/10] incorporate review --- .../definitions/event-schemas/m.device_list_update.yaml | 9 ++++----- specification/server_server_api.rst | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml index 0e7f0d32..49b49c3b 100644 --- a/api/server-server/definitions/event-schemas/m.device_list_update.yaml +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -30,7 +30,7 @@ allOf: edu_type: type: enum enum: ['m.device_list_update'] - description: The string ``m.device_list_update`` + description: The string ``m.device_list_update``. example: "m.device_list_update" content: type: object @@ -55,7 +55,7 @@ allOf: type: integer description: |- An ID sent by the server for this update, unique for a given - user_id. Used to identify any gaps in the sequence of m.device_list_update + user_id. Used to identify any gaps in the sequence of ``m.device_list_update`` EDUs broadcast by a server. example: 6 prev_id: @@ -65,7 +65,7 @@ allOf: which have not been referred to already in an EDU's prev_id field. If the receiving server does not recognise any of the prev_ids, it means an EDU has been lost and the server should query a snapshot of the devicelist - via /user/keys/query in order to correctly interpret future m.device_list_update + via ``/user/keys/query`` in order to correctly interpret future ``m.device_list_update`` EDUs. May be missing or empty for the first EDU in a sequence. items: type: integer @@ -82,8 +82,7 @@ allOf: description: |- The updated identity keys (if any) for this device. May be absent if the device has no E2E keys defined. - allOf: - - $ref: ../../../client-server/definitions/device_keys.yaml + $ref: ../../../client-server/definitions/device_keys.yaml required: - user_id - device_id diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 45ac1c9f..ef5903ff 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1008,14 +1008,14 @@ of the Client-Server API. Matrix currently uses a custom pubsub system for synchronising information about the list of devices for a given user over federation. When a server wishes to determine a remote user's device list for the first time, -it should populate its local cache by calling the /user/keys/query API +it should populate its local cache by calling the ``/user/keys/query`` API on the remote server. However, subsequent updates to the cache should be applied by consuming ``m.device_list_update`` EDUs, which must be sent by the remote server whenever a user's device list changes. Servers send ``m.device_list_update`` EDUs in a sequence per source user, each with a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s) -that this update is relative to in the `pprev_id`` field. To simplify implementation +that this update is relative to in the ``prev_id`` field. To simplify implementation for clustered servers which could send multiple EDUs at the same time, the ``prev_id`` field should include all ``m.device_list_update`` EDUs which have not been yet been referenced in a EDU. If EDUs are emitted in series by a server, there should only ever @@ -1025,7 +1025,7 @@ This forms a simple directed acyclic graph of ``m.device_list_update`` EDUs, sho which EDUs a server needs to have received in order to apply an update to its local copy of the remote user's device list. If a server receives an EDU which refers to a ``prev_id`` it does not recognise, it must resynchronise its list by calling the -/user/keys/query API and resume the process. The response contains a ``stream_id`` +``/user/keys/query API`` and resume the process. The response contains a ``stream_id`` which should be used to correlate with subsequent ``m.device_list_update`` EDUs. .. TODO: this whole thing desperately feels like it should just be state in a room, From e9e3afbd342d9ff2e704f8f582bfe48ade04c4d9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 2 Sep 2018 13:00:29 +0100 Subject: [PATCH 07/10] clarify who servers should send m.device_list_update EDUs to --- specification/server_server_api.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index ef5903ff..4f8e2e3b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1008,12 +1008,16 @@ of the Client-Server API. Matrix currently uses a custom pubsub system for synchronising information about the list of devices for a given user over federation. When a server wishes to determine a remote user's device list for the first time, -it should populate its local cache by calling the ``/user/keys/query`` API +it should populate a local cache from the result of a ``/user/keys/query`` API on the remote server. However, subsequent updates to the cache should be applied -by consuming ``m.device_list_update`` EDUs, which must be sent by the remote server -whenever a user's device list changes. - -Servers send ``m.device_list_update`` EDUs in a sequence per source user, each with +by consuming ``m.device_list_update`` EDUs. Each new ``m.device_list_update`` EDU +describes an incremental change to one device for a given user which should replace +any existing entry in the local server's cache of that device list. Servers must send +``m.device_list_update`` EDUs to all the servers whose users participate in their rooms, +and must be sent whenever a local user's device list changes (i.e. new or deleted devices, +or changes of identity keys). + +Servers send ``m.device_list_update`` EDUs in a sequence per origin user, each with a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s) that this update is relative to in the ``prev_id`` field. To simplify implementation for clustered servers which could send multiple EDUs at the same time, the ``prev_id`` From ee5e5198859e11d87c6015590ed3f1549176e056 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 2 Sep 2018 13:05:15 +0100 Subject: [PATCH 08/10] device list is 2 words --- .../definitions/event-schemas/m.device_list_update.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/event-schemas/m.device_list_update.yaml b/api/server-server/definitions/event-schemas/m.device_list_update.yaml index 49b49c3b..f11e957e 100644 --- a/api/server-server/definitions/event-schemas/m.device_list_update.yaml +++ b/api/server-server/definitions/event-schemas/m.device_list_update.yaml @@ -64,7 +64,7 @@ allOf: The stream_ids of any prior m.device_list_update EDUs sent for this user which have not been referred to already in an EDU's prev_id field. If the receiving server does not recognise any of the prev_ids, it means an EDU - has been lost and the server should query a snapshot of the devicelist + has been lost and the server should query a snapshot of the device list via ``/user/keys/query`` in order to correctly interpret future ``m.device_list_update`` EDUs. May be missing or empty for the first EDU in a sequence. items: From ccbc88ee5d9dc794dcdf47fdba18a322acf24e26 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 3 Sep 2018 16:49:36 +0100 Subject: [PATCH 09/10] reword to spell out to when device list updates should be sent --- specification/server_server_api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4f8e2e3b..e3427b69 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1013,9 +1013,10 @@ on the remote server. However, subsequent updates to the cache should be applie by consuming ``m.device_list_update`` EDUs. Each new ``m.device_list_update`` EDU describes an incremental change to one device for a given user which should replace any existing entry in the local server's cache of that device list. Servers must send -``m.device_list_update`` EDUs to all the servers whose users participate in their rooms, -and must be sent whenever a local user's device list changes (i.e. new or deleted devices, -or changes of identity keys). +``m.device_list_update`` EDUs to all the servers who share a room with a given +local user, and must be sent whenever that user's device list changes (i.e. for new or +deleted devices, when that user joins a new room, or changes of device information such as +human-readable name). Servers send ``m.device_list_update`` EDUs in a sequence per origin user, each with a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s) From 334c69bcdaf61106a550342845e1c9d7327f0ab5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 3 Sep 2018 16:53:39 +0100 Subject: [PATCH 10/10] wording fix --- specification/server_server_api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e3427b69..16f03061 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1015,8 +1015,9 @@ describes an incremental change to one device for a given user which should repl any existing entry in the local server's cache of that device list. Servers must send ``m.device_list_update`` EDUs to all the servers who share a room with a given local user, and must be sent whenever that user's device list changes (i.e. for new or -deleted devices, when that user joins a new room, or changes of device information such as -human-readable name). +deleted devices, when that user joins a room which contains servers which are not +already receiving updates for that user's device list, or changes in device information +such as the device's human-readable name). Servers send ``m.device_list_update`` EDUs in a sequence per origin user, each with a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s)