From 09a768763d89365308152168095be862b246efcc Mon Sep 17 00:00:00 2001 From: Konstantinos Sideris Date: Sat, 14 Jan 2017 19:09:23 +0200 Subject: [PATCH 01/58] Add missing enum fields for the set_presence parameter Signed-off-by: Konstantinos Sideris --- api/client-server/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index 66f7d935..b7965da9 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -76,7 +76,7 @@ paths: - in: query name: set_presence type: string - enum: ["offline"] + enum: ["offline", "online", "unavailable"] description: |- Controls whether the client is automatically marked as online by polling this API. If this parameter is omitted then the client is From 1e656d836ead371bb395a768e35173a6cf41f5ea Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:35:44 +0100 Subject: [PATCH 02/58] spec notifications key on power level event and provide @room in example --- event-schemas/examples/m.room.power_levels | 5 ++++- event-schemas/schema/m.room.power_levels | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 0c8f8bc5..2b0aaca5 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -14,7 +14,10 @@ "users": { "@example:localhost": 100 }, - "users_default": 0 + "users_default": 0, + "notifications": { + "room": 20 + } }, "state_key": "", "origin_server_ts": 1431961217939, diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index 13a44c70..f348b52d 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -85,6 +85,14 @@ properties: ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if unspecified. type: number + notifications: + additionalProperties: + type: number + description: |- + The power level requirements for specific notification types. + This is a mapping from ``key`` to power level for that notifications key. + title: Notification power level requirements + type: object type: object state_key: description: A zero-length string. From 60ae73b179c53213ce952251c7e9f4d14c323fef Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:37:23 +0100 Subject: [PATCH 03/58] specify default --- event-schemas/schema/m.room.power_levels | 1 + 1 file changed, 1 insertion(+) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index f348b52d..a00746f0 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -91,6 +91,7 @@ properties: description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. + Defaults to 50 for unspecified keys. title: Notification power level requirements type: object type: object From c305317fa57b918d68731f69a0ab21367debde0b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Sun, 15 Apr 2018 22:41:24 +0100 Subject: [PATCH 04/58] explicitly specify @room --- event-schemas/schema/m.room.power_levels | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index a00746f0..25d33f08 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -86,13 +86,16 @@ properties: unspecified. type: number notifications: + properties: + room: + type: number + description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. additionalProperties: type: number description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. - Defaults to 50 for unspecified keys. - title: Notification power level requirements + title: Notifications type: object type: object state_key: From ba51d5960ecbc7b820235ad75765c06ccd4da379 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 16 Aug 2018 11:44:48 -0600 Subject: [PATCH 05/58] r0.1.0 release of the Push Gateway specification Because this is the first release, it has several moving parts to it: * The version variables have been defined. * The towncrier changelog has been prepared for future modifications. * The templating has been updated to better support future versions of the specification. * A release process document has been created. --- api/push-gateway/push_notifier.yaml | 2 +- changelogs/push_gateway.rst | 6 +++ .../push_gateway/newsfragments/.gitignore | 1 + changelogs/push_gateway/pyproject.toml | 30 ++++++++++++ meta/releasing_a_spec.md | 47 +++++++++++++++++++ scripts/gendoc.py | 6 +++ .../templating/matrix_templates/sections.py | 5 ++ scripts/templating/matrix_templates/units.py | 5 +- specification/push_gateway.rst | 19 +++++++- specification/targets.yaml | 2 +- 10 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 changelogs/push_gateway.rst create mode 100644 changelogs/push_gateway/newsfragments/.gitignore create mode 100644 changelogs/push_gateway/pyproject.toml create mode 100644 meta/releasing_a_spec.md diff --git a/api/push-gateway/push_notifier.yaml b/api/push-gateway/push_notifier.yaml index 4a6cb8f7..21c1ea29 100644 --- a/api/push-gateway/push_notifier.yaml +++ b/api/push-gateway/push_notifier.yaml @@ -20,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: /_matrix/push/v1 +basePath: /_matrix/push/%PUSH_GATEWAY_MAJOR_VERSION% consumes: - application/json produces: diff --git a/changelogs/push_gateway.rst b/changelogs/push_gateway.rst new file mode 100644 index 00000000..33a7683c --- /dev/null +++ b/changelogs/push_gateway.rst @@ -0,0 +1,6 @@ +r0.1.0 +====== + +The first release of the Push Gateway specification. This release contains +a single endpoint, ``/notify``, that pushers may use to send push notifications +to clients. diff --git a/changelogs/push_gateway/newsfragments/.gitignore b/changelogs/push_gateway/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/push_gateway/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/push_gateway/pyproject.toml b/changelogs/push_gateway/pyproject.toml new file mode 100644 index 00000000..dad1bc04 --- /dev/null +++ b/changelogs/push_gateway/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../push_gateway.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md new file mode 100644 index 00000000..cd7033fc --- /dev/null +++ b/meta/releasing_a_spec.md @@ -0,0 +1,47 @@ +# How to release a specification + +There are several specifications that belong to matrix, such as the client-server +specification, server-server specification, and identity server specification. Each +of these gets released independently of each other with their own version numbers. + +Once a specification is ready for release, a branch should be created to track the +changes in. This should be the name of the specification (as it appears in the directory +structure of this project) followed by a forward slash and the version being released, +followed by `_updates`. For example, if the Client-Server Specification was getting +an r0.4.0 release, the branch name would be `client_server/r0.4.0_updates`. + +*Note*: Historical releases prior to this process may or may not have an appropriate +release branch. Releases after this document came into place will have an appropriate +branch. + +The remainder of the process is as follows: +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the release instructions provided in the README.md located there. +1. Update the changelog section of the specification you're releasing to make a reference + to the new version. +1. Update any version/link references across all specifications. +1. Update the index to list the version correctly. +1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). + * This is done by making a PR to the `unstyled_docs/spec` folder for the version and + specification you're releasing. +1. Commit the changes and PR them to master. +1. Tag the release with the format `client_server/r0.4.0`. +1. Perform a release on GitHub to tag the release. +1. Yell from the mountaintop to the world about the new release. + +### Creating a release for a brand-new specification + +Some specifications may not have ever had a release, and therefore need a bit more work +to become ready. + +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the "new changelog" instructions provided in the README.md located there. +1. Open the specification RST file and make some changes: + * Using a released specification as a template, update the changelog section. + * Use the appropriate changelog variable in the RST. +1. Create/define the appropriate variables in `gendoc.py`. +1. Update `targets.yml`. +1. Update any version/link references across all specifications. +1. Follow the regular release process. diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 16c40af5..042e3d9d 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -518,6 +518,10 @@ if __name__ == '__main__': "--server_release", "-s", action="store", default="unstable", help="The server-server release tag to generate, e.g. r1.2" ) + parser.add_argument( + "--push_gateway_release", "-p", action="store", default="unstable", + help="The push gateway release tag to generate, e.g. r1.2" + ) parser.add_argument( "--list_targets", action="store_true", help="Do not update the specification. Instead print a list of targets.", @@ -542,6 +546,8 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + "%PUSH_GATEWAY_MAJOR_VERSION%": "v1", + "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 1a93c723..65ed7f6d 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -31,6 +31,11 @@ class MatrixSections(Sections): def render_client_server_changelog(self): changelogs = self.units.get("changelogs") return changelogs["client_server"] + + # TODO: We should make this a generic variable instead of having to add functions all the time. + def render_push_gateway_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["push_gateway"] def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 90a87cd4..c400b691 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -754,6 +754,7 @@ class MatrixUnits(Units): def load_apis(self, substitutions): cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") + push_gw_ver = substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable") # we abuse the typetable to return this info to the templates return TypeTable(rows=[ @@ -774,8 +775,8 @@ class MatrixUnits(Units): "unstable", "Mapping of third party IDs to Matrix IDs", ), TypeTableRow( - "`Push Gateway API `_", - "unstable", + "`Push Gateway API `_", + push_gw_ver, "Push notifications for Matrix events", ), ]) diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index e4a9d6ea..e4623887 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -21,13 +22,27 @@ the homeserver. This is managed by a distinct entity called the Push Gateway. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + +.. topic:: Version: %PUSH_GATEWAY_RELEASE_LABEL% +{{push_gateway_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/push_gateway.rst + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. +- `r0.1.0 `_ + Overview -------- diff --git a/specification/targets.yaml b/specification/targets.yaml index 53957e0a..db869330 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,7 +25,7 @@ targets: push_gateway: files: - push_gateway.rst - version_label: unstable + version_label: "%PUSH_GATEWAY_RELEASE_LABEL%" appendices: files: - appendices.rst From 17a0dcc7d396a1ce9ca55141a5491e47b62c1f62 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 16 Aug 2018 23:11:07 +0100 Subject: [PATCH 06/58] add newsfragment for #1176 --- changelogs/client_server/newsfragments/1176.new | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1176.new diff --git a/changelogs/client_server/newsfragments/1176.new b/changelogs/client_server/newsfragments/1176.new new file mode 100644 index 00000000..41e30799 --- /dev/null +++ b/changelogs/client_server/newsfragments/1176.new @@ -0,0 +1 @@ +Specify how to control the power level required for ``@room`` \ No newline at end of file From 5b73a0174dec6889d00aba7488ffd79df9c319dc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 14:05:23 -0600 Subject: [PATCH 07/58] Clarify what the release branch is used for, and how it should be named --- meta/releasing_a_spec.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index cd7033fc..078d7178 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -5,10 +5,10 @@ specification, server-server specification, and identity server specification. E of these gets released independently of each other with their own version numbers. Once a specification is ready for release, a branch should be created to track the -changes in. This should be the name of the specification (as it appears in the directory -structure of this project) followed by a forward slash and the version being released, -followed by `_updates`. For example, if the Client-Server Specification was getting -an r0.4.0 release, the branch name would be `client_server/r0.4.0_updates`. +changes in and to hold potential future hotfixes. This should be the name of the +specification (as it appears in the directory structure of this project) followed +by "release-" and the release version. For example, if the Client-Server Specification +was getting an r0.4.0 release, the branch name would be `client_server/release-r0.4.0`. *Note*: Historical releases prior to this process may or may not have an appropriate release branch. Releases after this document came into place will have an appropriate From e141f61df3a1d8305b3644ba516a1260f41744ec Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 20 Aug 2018 14:05:48 -0600 Subject: [PATCH 08/58] Update release instructions to reference symlinks, what files to update, etc --- meta/releasing_a_spec.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index 078d7178..3a9da893 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -21,12 +21,13 @@ The remainder of the process is as follows: 1. Update the changelog section of the specification you're releasing to make a reference to the new version. 1. Update any version/link references across all specifications. -1. Update the index to list the version correctly. +1. Ensure the `targets.yml` file lists the version correctly. +1. Commit the changes and PR them to master. +1. Tag the release with the format `client_server/r0.4.0`. 1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). * This is done by making a PR to the `unstyled_docs/spec` folder for the version and specification you're releasing. -1. Commit the changes and PR them to master. -1. Tag the release with the format `client_server/r0.4.0`. + * Don't forget to symlink the new release as `latest`. 1. Perform a release on GitHub to tag the release. 1. Yell from the mountaintop to the world about the new release. From 2d8a321b4629859d15b938b9bf9463dfa2569604 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 31 Jul 2018 17:27:50 +0200 Subject: [PATCH 09/58] document key sharing and m.room_key_request --- .../m.room_key_request#cancel_request | 8 +++ .../examples/m.room_key_request#request | 14 +++++ event-schemas/schema/m.room_key_request | 59 +++++++++++++++++++ .../modules/end_to_end_encryption.rst | 18 ++++++ 4 files changed, 99 insertions(+) create mode 100644 event-schemas/examples/m.room_key_request#cancel_request create mode 100644 event-schemas/examples/m.room_key_request#request create mode 100644 event-schemas/schema/m.room_key_request diff --git a/event-schemas/examples/m.room_key_request#cancel_request b/event-schemas/examples/m.room_key_request#cancel_request new file mode 100644 index 00000000..c6eb25de --- /dev/null +++ b/event-schemas/examples/m.room_key_request#cancel_request @@ -0,0 +1,8 @@ +{ + "content": { + "action": "cancel_request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/examples/m.room_key_request#request b/event-schemas/examples/m.room_key_request#request new file mode 100644 index 00000000..8557f08e --- /dev/null +++ b/event-schemas/examples/m.room_key_request#request @@ -0,0 +1,14 @@ +{ + "content": { + "body": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU" + }, + "action": "request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/schema/m.room_key_request b/event-schemas/schema/m.room_key_request new file mode 100644 index 00000000..6893dd78 --- /dev/null +++ b/event-schemas/schema/m.room_key_request @@ -0,0 +1,59 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to request keys for end-to-end encryption. It is sent as an + unencrypted `to-device`_ event. +properties: + content: + properties: + body: + description: Information about the requested key. + properties: + algorithm: + type: string + description: |- + The encryption algorithm the requested key in this event is to be used + with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session holding the key. + required: + - algorithm + - room_id + - session_id + - sender_key + type: object + title: RequestedKeyInfo + action: + enum: + - request + - cancel_request + type: string + requesting_device_id: + description: ID of the device requesting the key. + type: string + request_id: + description: |- + A random string uniquely identifying the request for a key. If the key is + requested multiple times, it should be reused. It should also reused in order + to cancel a request. + type: string + required: + - action + - requesting_device_id + - request_id + type: object + type: + enum: + - m.room_key_request + type: string +type: object diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index fa461cc2..e235e2e9 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -283,6 +283,20 @@ Device verification may reach one of several conclusions. For example: decrypted by such a device. For the Olm protocol, this is documented at https://matrix.org/git/olm/about/docs/signing.rst. +Key sharing +----------- + +If Bob has an encrypted conversation with Alice on his computer, and then logs in +through his phone for the first time, he may want to have access to the previously +exchanged messages. To address this issue, events exist for requesting and sending +keys from device to device. + +.. NOTE:: + + Key sharing can be a big attack vector, thus it must be done very carefully. + A reasonable stategy is for a user's client to only send keys requested by the + verified devices of the same user. + Messaging Algorithms -------------------- @@ -464,6 +478,10 @@ Events {{m_room_key_event}} +{{m_room_key_request_event}} + +{{m_forwarded_room_key_event}} + Key management API ~~~~~~~~~~~~~~~~~~ From 019c290fa2e4aef1e0b98aad358d5a35fef45c50 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 1 Aug 2018 00:25:31 +0200 Subject: [PATCH 10/58] document m.forwarded_room_key --- event-schemas/examples/m.forwarded_room_key | 14 +++++ event-schemas/schema/m.forwarded_room_key | 58 +++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 event-schemas/examples/m.forwarded_room_key create mode 100644 event-schemas/schema/m.forwarded_room_key diff --git a/event-schemas/examples/m.forwarded_room_key b/event-schemas/examples/m.forwarded_room_key new file mode 100644 index 00000000..8ab85c48 --- /dev/null +++ b/event-schemas/examples/m.forwarded_room_key @@ -0,0 +1,14 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8Llf...", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU", + "sender_claimed_ed25519_key": "aj40p+aw64yPIdsxoog8jhPu9i7l7NcFRecuOQblE3Y", + "forwarding_curve25519_key_chain": [ + "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" + ] + }, + "type": "m.room_key" +} diff --git a/event-schemas/schema/m.forwarded_room_key b/event-schemas/schema/m.forwarded_room_key new file mode 100644 index 00000000..02d91f4e --- /dev/null +++ b/event-schemas/schema/m.forwarded_room_key @@ -0,0 +1,58 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to forward keys for end-to-end encryption. Typically + it is encrypted as an ``m.room.encrypted`` event. +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm the key in this event is to be used with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session holding the key. + session_key: + type: string + description: The key to be exchanged. + sender_claimed_ed25519_key: + type: string + description: |- + The Ed25519 key of the device which initiated the session originally. + It is 'claimed' because the receiving device has no way to tell that the + original room_key actually came from a device which owns the private part of + this key unless they have done device verification. + forwarding_curve25519_key_chain: + type: array + items: + type: string + description: |- + Chain of Curve25519 keys. It starts out empty, but each time the + key is forwarded to another device, the previous sender in the chain is added + to the end of the list. For example, if the key is forwarded from A to B to + C, this field is empty between A and B, and contains A's Curve25519 key between + B and C. + required: + - algorithm + - room_id + - session_id + - session_key + - sender_claimed_ed25519_key + - forwarding_curve25519_key_chain + - sender_key + type: object + type: + enum: + - m.forwarded_room_key + type: string +type: object From 7355d23ea50820f120e73ea5d56b8ed17f7fcdd2 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 21 Aug 2018 21:53:02 +0200 Subject: [PATCH 11/58] add key sharing changelog --- changelogs/client_server/newsfragments/1465.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1465.feature diff --git a/changelogs/client_server/newsfragments/1465.feature b/changelogs/client_server/newsfragments/1465.feature new file mode 100644 index 00000000..61a7ed39 --- /dev/null +++ b/changelogs/client_server/newsfragments/1465.feature @@ -0,0 +1 @@ +Share room encryption keys between devices From 811998735cf2af6ea12ee27427b929ffa3491d93 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:51:23 -0600 Subject: [PATCH 12/58] Define common error codes in the Identity Service API Fixes https://github.com/matrix-org/matrix-doc/issues/1407 --- specification/identity_service_api.rst | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index 3b037caf..fb08f637 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -56,6 +56,75 @@ is left as an exercise for the client. 3PID types are described in `3PID Types`_ Appendix. +API Standards +------------- + +The mandatory baseline for identity service communication in Matrix is exchanging +JSON objects over HTTP APIs. HTTPS is required for communication, and all API calls +use a Content-Type of ``application/json``. In addition, strings MUST be encoded as +UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard error response". +This is a JSON object which looks like: + +.. code:: json + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. There may be +additional keys depending on the error, but the keys ``error`` and ``errcode`` +MUST always be present. + +Some standard error codes are below: + +:``M_NOT_FOUND``: + The resource requested could not be located. + +:``M_MISSING_PARAMS``: + The request was missing one or more parameters. + +:``M_INVALID_PARAM``: + The request contained one or more invalid parameters. + +:``M_SESSION_NOT_VALIDATED``: + The session has not been validated. + +:``M_NO_VALID_SESSION``: + A session could not be located for the given parameters. + +:``M_SESSION_EXPIRED``: + The session has expired and must be renewed. + +:``M_INVALID_EMAIL``: + The email address provided was not valid. + +:``M_EMAIL_SEND_ERROR``: + There was an error sending an email. Typically seen when attempting to verify + ownership of a given email address. + +:``M_INVALID_ADDRESS``: + The provided third party address was not valid. + +:``M_SEND_ERROR``: + There was an error sending a notification. Typically seen when attempting to + verify ownership of a given third party address. + +:``M_UNRECOGNIZED``: + The request contained an unrecognised value, such as an unknown token or medium. + +:``M_THREEPID_IN_USE``: + The third party identifier is already in use by another user. Typically this + error will have an additional ``mxid`` property to indicate who owns the + third party identifier. + +:``M_UNKNOWN``: + An unknown error has occurred. + Privacy ------- From 4abd618147618abc73502f1bd84ee55da5b139e5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:53:00 -0600 Subject: [PATCH 13/58] Flag response fields in the Identity Service spec as required --- api/identity/associations.yaml | 9 +++++++++ api/identity/email_associations.yaml | 2 ++ api/identity/invitation_signing.yaml | 1 + api/identity/lookup.yaml | 8 ++++++++ api/identity/phone_associations.yaml | 2 ++ api/identity/pubkey.yaml | 2 ++ api/identity/store_invite.yaml | 1 + 7 files changed, 25 insertions(+) diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 784bb5d6..6d282e8d 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -62,6 +62,7 @@ paths: validated_at: type: integer description: Timestamp indicating the time that the 3pid was validated. + required: ['medium', 'address', 'validated_at'] 400: description: |- The session has not been validated. @@ -158,6 +159,14 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures 400: description: |- The association was not published. diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index 8431c9e8..c9fb0cd7 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -93,6 +93,7 @@ paths: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: @@ -151,6 +152,7 @@ paths: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of an email address. description: |- diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index 982dbff7..c595299f 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -71,6 +71,7 @@ paths: token: type: string description: The token for the invitation. + required: ['mxid', 'sender', 'signatures', 'token'] examples: application/json: { "mxid": "@foo:bar.com", diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index bfd2153e..3bc58be5 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -86,6 +86,14 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures "/bulk_lookup": post: summary: Lookup Matrix user IDs for a list of 3pids. diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index c2cc6cfe..605dadcc 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -99,6 +99,7 @@ paths: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: @@ -157,6 +158,7 @@ paths: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of a phone number. description: |- diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 00796975..45f8b99a 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -80,6 +80,7 @@ paths: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] "/pubkey/ephemeral/isvalid": get: summary: Check whether a short-term public key is valid. @@ -108,3 +109,4 @@ paths: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index 6b847b5b..d6fae7a7 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -90,6 +90,7 @@ paths: display_name: type: string description: The generated (redacted) display_name. + required: ['token', 'public_keys', 'display_name'] example: application/json: { "token": "sometoken", From dafea96621d6383f6b44d9368024bb95c8ab94a7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 15:53:27 -0600 Subject: [PATCH 14/58] Fix indentation and schema references in the identity service spec --- api/identity/associations.yaml | 69 +++++++++++++++----------- api/identity/email_associations.yaml | 31 +++++++----- api/identity/invitation_signing.yaml | 6 ++- api/identity/lookup.yaml | 35 +++++++------ api/identity/phone_associations.yaml | 33 +++++++----- api/identity/pubkey.yaml | 20 ++++++-- api/identity/store_invite.yaml | 34 +++++++------ schemas/server-signatures.yaml | 24 +++++++++ specification/identity_service_api.rst | 2 +- 9 files changed, 162 insertions(+), 92 deletions(-) create mode 100644 schemas/server-signatures.yaml diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 6d282e8d..4225919b 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -46,10 +46,10 @@ paths: description: Validation information for the session. examples: application/json: { - "medium": "email", - "validated_at": 1457622739026, - "address": "louise@bobs.burgers" - } + "medium": "email", + "validated_at": 1457622739026, + "address": "louise@bobs.burgers" + } schema: type: object properties: @@ -72,16 +72,20 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/bind": post: summary: Publish an association between a session and a Matrix user ID. @@ -102,10 +106,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "mxid": "@ears:matrix.org" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "mxid": "@ears:matrix.org" + } properties: sid: type: string @@ -122,19 +126,19 @@ paths: description: The association was published. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -159,6 +163,7 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" required: - address - medium @@ -176,13 +181,17 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index c9fb0cd7..28f5e680 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -51,10 +51,10 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "email": "foo@example.com", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "email": "foo@example.com", + "send_attempt": 1 + } properties: client_secret: type: string @@ -85,8 +85,8 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: @@ -100,6 +100,13 @@ paths: - ``M_INVALID_EMAIL``: The email address provided was invalid. - ``M_EMAIL_SEND_ERROR``: The validation email could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_EMAIL", + "error": "The email address is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/email/submitToken": post: summary: Validate ownership of an email address. @@ -123,10 +130,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -144,8 +151,8 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index c595299f..7de62dd4 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -68,6 +68,7 @@ paths: signatures: type: object description: The signature of the mxid, sender, and token. + $ref: "../../schemas/server-signatures.yaml" token: type: string description: The token for the invitation. @@ -85,7 +86,10 @@ paths: } 404: description: Token was not found. - example: { + examples: + application/json: { "errcode": "M_UNRECOGNIZED", "error": "Didn't recognize token" } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index 3bc58be5..6f993ac7 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -49,19 +49,18 @@ paths: The association for that 3pid, or the empty object if no association is known. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, - - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -86,6 +85,7 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" required: - address - medium @@ -118,9 +118,11 @@ paths: items: type: array title: 3PID mappings + minItems: 2 + maxItems: 2 items: - type: string - title: 3PID medium or address + - type: 3PID Medium + - type: 3PID Address description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. required: - "threepids" @@ -142,9 +144,12 @@ paths: items: type: array title: 3PID mappings + minItems: 3 + maxItems: 3 items: - type: string - title: 3PID medium or address or the Matrix ID + - type: 3PID Medium + - type: 3PID Address + - type: Matrix User ID description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. required: - "threepids" diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index 605dadcc..f6b1bd45 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -51,11 +51,11 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "country": "GB", - "phone_number": "07700900001", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "country": "GB", + "phone_number": "07700900001", + "send_attempt": 1 + } properties: client_secret: type: string @@ -91,8 +91,8 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: @@ -106,6 +106,13 @@ paths: - ``M_INVALID_ADDRESS``: The phone number provided was invalid. - ``M_SEND_ERROR``: The validation SMS could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_ADDRESS", + "error": "The phone number is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/msisdn/submitToken": post: summary: Validate ownership of a phone number. @@ -129,10 +136,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -150,8 +157,8 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 45f8b99a..9cb7c74e 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -45,13 +45,25 @@ paths: The public key exists. examples: application/json: { - "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" - } + "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" + } schema: type: object properties: public_key: type: string + description: Unpadded Base64 encoded public key. + required: ['public_key'] + 404: + description: + The public key was not found. + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "The public key was not found" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/pubkey/isvalid": get: summary: Check whether a long-term public key is valid. @@ -72,8 +84,8 @@ paths: The validity of the public key. examples: application/json: { - "valid": true - } + "valid": true + } schema: type: object properties: diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index d6fae7a7..1eca7198 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -54,11 +54,11 @@ paths: schema: type: object example: { - "medium": "email", - "address": "foo@bar.baz", - "room_id": "!something:example.tld", - "sender": "@bob:example.com" - } + "medium": "email", + "address": "foo@bar.baz", + "room_id": "!something:example.tld", + "sender": "@bob:example.com" + } properties: medium: type: string @@ -93,13 +93,13 @@ paths: required: ['token', 'public_keys', 'display_name'] example: application/json: { - "token": "sometoken", - "public_keys": [ - "serverpublickey", - "ephemeralpublickey" - ], - "display_name": "f...@b..." - } + "token": "sometoken", + "public_keys": [ + "serverpublickey", + "ephemeralpublickey" + ], + "display_name": "f...@b..." + } 400: description: | An error has occured. @@ -109,7 +109,9 @@ paths: error code will be ``M_UNRECOGNIZED``. examples: application/json: { - "errcode": "M_THREEPID_IN_USE", - "error": "Binding already known", - "mxid": mxid - } + "errcode": "M_THREEPID_IN_USE", + "error": "Binding already known", + "mxid": mxid + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/schemas/server-signatures.yaml b/schemas/server-signatures.yaml new file mode 100644 index 00000000..a1855256 --- /dev/null +++ b/schemas/server-signatures.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +example: { + "example.com": { + "ed25519:0": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } +} +additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string \ No newline at end of file diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index fb08f637..7bbd8ae8 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -23,7 +23,7 @@ user identifiers. From time to time, it is useful to refer to users by other number. This identity service specification describes how mappings between third-party identifiers and Matrix user identifiers can be established, validated, and used. This description technically may apply to any 3pid, but in -practice has only been applied specifically to email addresses. +practice has only been applied specifically to email addresses and phone numbers. .. contents:: Table of Contents .. sectnum:: From 337316445f444e03311916dfe18316b979005123 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 16:55:00 +0200 Subject: [PATCH 15/58] fixup! document m.forwarded_room_key --- event-schemas/schema/m.forwarded_room_key | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/m.forwarded_room_key b/event-schemas/schema/m.forwarded_room_key index 02d91f4e..3b1c5900 100644 --- a/event-schemas/schema/m.forwarded_room_key +++ b/event-schemas/schema/m.forwarded_room_key @@ -21,7 +21,7 @@ properties: The Curve25519 key of the device which initiated the session originally. session_id: type: string - description: The ID of the session holding the key. + description: The ID of the session that the key is for. session_key: type: string description: The key to be exchanged. From 768f33dc2d5bd9854a932cc83f726c1fe85af8ea Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 16:55:28 +0200 Subject: [PATCH 16/58] fixup! document key sharing and m.room_key_request --- event-schemas/schema/m.room_key_request | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/m.room_key_request b/event-schemas/schema/m.room_key_request index 6893dd78..f495a11d 100644 --- a/event-schemas/schema/m.room_key_request +++ b/event-schemas/schema/m.room_key_request @@ -25,7 +25,7 @@ properties: The Curve25519 key of the device which initiated the session originally. session_id: type: string - description: The ID of the session holding the key. + description: The ID of the session that the key is for. required: - algorithm - room_id From 8ae137e82ee197414ad8f904db441ffc612b87f2 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 17:26:00 +0200 Subject: [PATCH 17/58] fixup! document m.forwarded_room_key --- event-schemas/schema/m.forwarded_room_key | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/event-schemas/schema/m.forwarded_room_key b/event-schemas/schema/m.forwarded_room_key index 3b1c5900..f0beed2b 100644 --- a/event-schemas/schema/m.forwarded_room_key +++ b/event-schemas/schema/m.forwarded_room_key @@ -4,7 +4,8 @@ allOf: description: |- This event type is used to forward keys for end-to-end encryption. Typically - it is encrypted as an ``m.room.encrypted`` event. + it is encrypted as an ``m.room.encrypted`` event, then sent as a `to-device`_ + event. properties: content: properties: From 579f5ea928f5bffb74969a9b736113529a0334e0 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 17:26:40 +0200 Subject: [PATCH 18/58] fixup! document key sharing and m.room_key_request --- event-schemas/schema/m.room_key_request | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/event-schemas/schema/m.room_key_request b/event-schemas/schema/m.room_key_request index f495a11d..007d0086 100644 --- a/event-schemas/schema/m.room_key_request +++ b/event-schemas/schema/m.room_key_request @@ -9,7 +9,9 @@ properties: content: properties: body: - description: Information about the requested key. + description: |- + Information about the requested key. Required when ``action`` is + ``request``. properties: algorithm: type: string From f6b5aee3e2e39b028b893da1745f15a56c993c73 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 17:27:18 +0200 Subject: [PATCH 19/58] explain how to use key sharing events --- specification/modules/end_to_end_encryption.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index e235e2e9..26e677dd 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -291,6 +291,17 @@ through his phone for the first time, he may want to have access to the previous exchanged messages. To address this issue, events exist for requesting and sending keys from device to device. +When a device is missing keys to decrypt messages, it can request the keys by +sending `m.room_key_request`_ to-device messages to other devices with +``action`` set to ``request``. If a device wishes to share the keys with that +device, it can forward the keys to the first device by sending an encrypted +`m.forwarded_room_key`_ to-device message. The first device should then send an +`m.room_key_request`_ to-device message with ``action`` set to +``cancel_request`` to the other devices that it had originally sent the key +request to; a device that receives a ``cancel_request`` should disregard any +previously-received ``request`` message with the same ``request_id`` and +``requesting_device_id``. + .. NOTE:: Key sharing can be a big attack vector, thus it must be done very carefully. From faa13aaa40cd0c2cf561f3e1cae5952ffcbe09d0 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Sat, 25 Aug 2018 17:28:01 +0200 Subject: [PATCH 20/58] fixup! add key sharing changelog --- changelogs/client_server/newsfragments/1465.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/client_server/newsfragments/1465.feature b/changelogs/client_server/newsfragments/1465.feature index 61a7ed39..649cf222 100644 --- a/changelogs/client_server/newsfragments/1465.feature +++ b/changelogs/client_server/newsfragments/1465.feature @@ -1 +1 @@ -Share room encryption keys between devices +Share room decryption keys between devices From fa96d8629bcce201b1ef32a8b969e0f0ae60343f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:36:04 -0600 Subject: [PATCH 21/58] Prepare the appservice spec for an r0 release This puts the scaffolding in place for an r0 release to happen, such as the changelog and version variables. --- changelogs/application_service.rst | 0 .../newsfragments/.gitignore | 1 + changelogs/application_service/pyproject.toml | 30 +++++++++++++++++++ scripts/gendoc.py | 6 ++++ .../templating/matrix_templates/sections.py | 4 +++ scripts/templating/matrix_templates/units.py | 5 ++-- specification/application_service_api.rst | 12 ++++++-- specification/targets.yaml | 2 +- 8 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 changelogs/application_service.rst create mode 100644 changelogs/application_service/newsfragments/.gitignore create mode 100644 changelogs/application_service/pyproject.toml diff --git a/changelogs/application_service.rst b/changelogs/application_service.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/application_service/newsfragments/.gitignore b/changelogs/application_service/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/application_service/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/application_service/pyproject.toml b/changelogs/application_service/pyproject.toml new file mode 100644 index 00000000..44d430e8 --- /dev/null +++ b/changelogs/application_service/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../application_service.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 16c40af5..8bd8537f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -518,6 +518,10 @@ if __name__ == '__main__': "--server_release", "-s", action="store", default="unstable", help="The server-server release tag to generate, e.g. r1.2" ) + parser.add_argument( + "--appservice_release", "-a", action="store", default="unstable", + help="The appservice release tag to generate, e.g. r1.2" + ) parser.add_argument( "--list_targets", action="store_true", help="Do not update the specification. Instead print a list of targets.", @@ -542,6 +546,8 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + "%APPSERVICE_MAJOR_VERSION%": "unstable", + "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, } exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 1a93c723..b3c7d402 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -32,6 +32,10 @@ class MatrixSections(Sections): changelogs = self.units.get("changelogs") return changelogs["client_server"] + def render_application_service_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["application_service"] + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 90a87cd4..9423dfc8 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -754,6 +754,7 @@ class MatrixUnits(Units): def load_apis(self, substitutions): cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") + as_ver = substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable") # we abuse the typetable to return this info to the templates return TypeTable(rows=[ @@ -766,8 +767,8 @@ class MatrixUnits(Units): fed_ver, "Federation between servers", ), TypeTableRow( - "`Application Service API `_", - "unstable", + "`Application Service API `_", + as_ver, "Privileged server plugins", ), TypeTableRow( "`Identity Service API `_", diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 51280341..97a2c04a 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -30,13 +30,21 @@ irrespective of the underlying homeserver implementation. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + + +.. topic:: Version: unstable +{{application_service_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst + + Application Services -------------------- Application services are passive and can only observe events from a given diff --git a/specification/targets.yaml b/specification/targets.yaml index acf4b6ac..42a50e35 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -13,7 +13,7 @@ targets: application_service: files: - application_service_api.rst - version_label: unstable + version_label: "%APPSERVICE_RELEASE_LABEL%" server_server: files: - server_server_api.rst From f01cfa2c7bdd1d43109ac3acfc7b016b661e29db Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:39:39 -0600 Subject: [PATCH 22/58] Add security definitions to hs->as endpoints --- .../definitions/security.yaml | 18 ++++++++++++++++++ api/application-service/protocols.yaml | 12 ++++++++++++ api/application-service/query_room.yaml | 4 ++++ api/application-service/query_user.yaml | 4 ++++ api/application-service/transactions.yaml | 4 ++++ 5 files changed, 42 insertions(+) create mode 100644 api/application-service/definitions/security.yaml diff --git a/api/application-service/definitions/security.yaml b/api/application-service/definitions/security.yaml new file mode 100644 index 00000000..bcfc69c0 --- /dev/null +++ b/api/application-service/definitions/security.yaml @@ -0,0 +1,18 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +homeserverAccessToken: + type: apiKey + description: The ``hs_token`` provided by the application service's registration. + name: access_token + in: query diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index e6489cc5..d58710a4 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: @@ -33,6 +35,8 @@ paths: with specific information about the various third party networks that an application service supports. operationId: getProtocolMetadata + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -80,6 +84,8 @@ paths: User ID linked to a user on the third party network, given a set of user parameters. operationId: queryUserByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -131,6 +137,8 @@ paths: description: |- Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -183,6 +191,8 @@ paths: Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - homeserverAccessToken: [] parameters: - in: query name: alias @@ -227,6 +237,8 @@ paths: description: |- Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - homeserverAccessToken: [] parameters: - in: query name: userid diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index b885cb86..f388affe 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -25,6 +25,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/rooms/{roomAlias}": get: @@ -36,6 +38,8 @@ paths: homeserver will send this request when it receives a request to join a room alias within the application service's namespace. operationId: queryRoomByAlias + security: + - homeserverAccessToken: [] parameters: - in: path name: roomAlias diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 0431b5e4..5cb8d900 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -25,6 +25,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/users/{userId}": get: @@ -36,6 +38,8 @@ paths: send this request when it receives an event for an unknown user ID in the application service's namespace, such as a room invite. operationId: queryUserById + security: + - homeserverAccessToken: [] parameters: - in: path name: userId diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 8735cc8f..53291116 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -23,6 +23,8 @@ schemes: basePath: "/" produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/transactions/{txnId}": put: @@ -35,6 +37,8 @@ paths: from message events via the presence of a ``state_key``, rather than via the event type. operationId: sendTransaction + security: + - homeserverAccessToken: [] parameters: - in: path name: txnId From 07153c22a941a8de243031b2ef661c4e6e4b6866 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 16:54:53 -0600 Subject: [PATCH 23/58] Misc. cleanup of the appservice spec --- specification/application_service_api.rst | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 97a2c04a..f3205302 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -48,12 +48,12 @@ https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_serv Application Services -------------------- Application services are passive and can only observe events from a given -homeserver (HS). They can inject events into rooms they are participating in. +homeserver. They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the application service. This is achieved by manually configuring the homeserver -with information about the application service (AS). +with information about the application service. Registration ~~~~~~~~~~~~ @@ -187,24 +187,24 @@ events. Each list of events includes a transaction ID, which works as follows: Typical HS ---> AS : Homeserver sends events with transaction ID T. - <--- : AS sends back 200 OK. + <--- : Application Service sends back 200 OK. AS ACK Lost HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. HS ---> AS : Homeserver retries with the same transaction ID of T. - <--- : AS sends back 200 OK. If the AS had processed these events - already, it can NO-OP this request (and it knows if it is the same - events based on the transaction ID). + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). The events sent to the application service should be linearised, as if they were from the event stream. The homeserver MUST maintain a queue of transactions to -send to the AS. If the application service cannot be reached, the homeserver -SHOULD backoff exponentially until the application service is reachable again. +send to the application service. If the application service cannot be reached, the +homeserver SHOULD backoff exponentially until the application service is reachable again. As application services cannot *modify* the events in any way, these requests can be made without blocking other aspects of the homeserver. Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID -on retries, as the AS may have already processed the events. +on retries, as the application service may have already processed the events. {{transactions_as_http_api}} @@ -313,7 +313,7 @@ Notes: :: - PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 + PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 Authorization: Bearer YourApplicationServiceTokenHere Content: The event to send, as per the Client-Server API. @@ -334,7 +334,7 @@ users needs API changes in order to: - Have a 'passwordless' user. This involves bypassing the registration flows entirely. This is achieved by -including the AS token on a ``/register`` request, along with a login type of +including the ``as_token`` on a ``/register`` request, along with a login type of ``m.login.application_service`` to set the desired user ID without a password. :: @@ -374,8 +374,6 @@ additional parameters on the ``/publicRooms`` client-server endpoint. Event fields ~~~~~~~~~~~~ -.. TODO-TravisR: Fix this section to be a general "3rd party networks" section - We recommend that any events that originated from a remote network should include an ``external_url`` field in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. From 6f80db5ddf1a9851cdc3b8f9e167a3e805db28c1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 17:21:12 -0600 Subject: [PATCH 24/58] Include the "other versions" section of the changelog --- specification/application_service_api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index f3205302..4b42707d 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -44,6 +44,13 @@ This version of the specification is generated from For the full historical changelog, see https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. + Application Services -------------------- From 72c6fa2aaf73af1af9da40c3bdbc31de8369ce7c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 27 Aug 2018 19:33:03 -0600 Subject: [PATCH 25/58] More versioned links --- specification/modules/push.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/push.rst b/specification/modules/push.rst index e9ee8c90..492e8291 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -623,4 +623,4 @@ should send a "sync" command to instruct the client to get new events from the homeserver directly. -.. _`Push Gateway Specification`: ../push_gateway/unstable.html +.. _`Push Gateway Specification`: ../push_gateway/%PUSH_GATEWAY_RELEASE_LABEL%.html From 436544b8f8b8b6cf2e05744fa6f9b14ef4db40ed Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 14:19:55 -0600 Subject: [PATCH 26/58] Clarify behaviour of `?set_presence=unavailable` for /sync --- api/client-server/sync.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index f20798c0..21af0d1e 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -83,7 +83,8 @@ paths: polling this API. If this parameter is omitted then the client is automatically marked as online when it uses this API. Otherwise if the parameter is set to "offline" then the client is not marked as - being online when it uses this API. + being online when it uses this API. When set to "unavailable", the + client is marked as being idle. x-example: "offline" - in: query name: timeout From 4df67d9305d53d05fcbb355d9527030d64e6dd64 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 14:20:02 -0600 Subject: [PATCH 27/58] changelog --- changelogs/client_server/newsfragments/780.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/780.feature diff --git a/changelogs/client_server/newsfragments/780.feature b/changelogs/client_server/newsfragments/780.feature new file mode 100644 index 00000000..74725754 --- /dev/null +++ b/changelogs/client_server/newsfragments/780.feature @@ -0,0 +1 @@ +Add more presence options to the ``set_presence`` parameter of ``/sync``. (Thanks @mujx!) From b402608b41ee1e6a73f8e89296ebbfed4bbff3e9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 14:40:14 -0600 Subject: [PATCH 28/58] Don't reference the major version for the push gateway specification If we ever have a v2 endpoint for the push gateway, we'd likely spec it alongside the v1 stuff, updating applicable references elsewhere. --- api/push-gateway/push_notifier.yaml | 2 +- scripts/gendoc.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/api/push-gateway/push_notifier.yaml b/api/push-gateway/push_notifier.yaml index 21c1ea29..4a6cb8f7 100644 --- a/api/push-gateway/push_notifier.yaml +++ b/api/push-gateway/push_notifier.yaml @@ -20,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: /_matrix/push/%PUSH_GATEWAY_MAJOR_VERSION% +basePath: /_matrix/push/v1 consumes: - application/json produces: diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 042e3d9d..8745d3e2 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -546,7 +546,6 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), - "%PUSH_GATEWAY_MAJOR_VERSION%": "v1", "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } From dad037170a24219aa76bfdeca2c7b5b6b577b1e8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 15:45:50 -0600 Subject: [PATCH 29/58] Clarify that the requested event is excluded from /state and /state_ids Fixes https://github.com/matrix-org/matrix-doc/issues/1564 --- api/server-server/events.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index cf3988a2..f0b30178 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -50,7 +50,7 @@ paths: 200: description: |- The fully resolved state for the room, including the authorization - chain for the events. + chain for the events, excluding the requested event. schema: type: object properties: @@ -97,7 +97,7 @@ paths: 200: description: |- The fully resolved state for the room, including the authorization - chain for the events. + chain for the events, excluding the requested event. schema: type: object properties: From 6f3b42a457c7fc7973fb07847840143ffc7164e3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 28 Aug 2018 23:56:20 +0100 Subject: [PATCH 30/58] Server names cannot be %-encoded They aren't URLs; it was a thinko to refer to the URL spec for this. --- .../appendices/identifier_grammar.rst | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index fc89f031..d068b768 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -23,13 +23,38 @@ A homeserver is uniquely identified by its server name. This value is used in a number of identifiers, as described below. The server name represents the address at which the homeserver in question can -be reached by other homeservers. The complete grammar is:: +be reached by other homeservers. All valid server names are included by the +following grammar:: - server_name = host [ ":" port] - port = *DIGIT + server_name = host [ ":" port ] -where ``host`` is as defined by `RFC3986, section 3.2.2 -`_. + port = *DIGIT + + host = IPv4address / "[" IPv6address "]" / dns-name + + IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + + IPv6address = 2*45IPv6char + + IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." + ; 0-9, A-F, a-f, :, . + + dns-name = *255dns-char + + dns-char = DIGIT / ALPHA / "-" / "." + +-- in other words, the server name is the hostname, followed by an optional +numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, +an IPv6 address literal surrounded with square brackets, or a DNS name. + +IPv4 literals must be a sequence of four decimal numbers in the +range 0 to 255, separated by ".". IPv6 literals must be as specified by +`RFC3513, section 2.2 `_. + +DNS names for use with Matrix should follow the conventional restrictions for +internet hostnames: they should consist of a series of labels separated by +dots, where each label consists of the alphanumeric characters or +hyphens. Examples of valid server names are: @@ -40,6 +65,20 @@ Examples of valid server names are: * ``[1234:5678::abcd]`` (IPv6 literal) * ``[1234:5678::abcd]:5678`` (IPv6 literal with explicit port) +.. Note:: + + This grammar is based on the standard for internet host names, as specified + by `RFC1123, section 2.1 `_, + with an extension for IPv6 literals. + +Server names must be treated case-sensitively: in other words, +``@user:matrix.org`` is a different person from ``@user:MATRIX.ORG``. + +Some recommendations for a choice of server name follow: + +* The length of the complete server name should not exceed 230 characters. +* Server names should not use upper-case characters. + Room Versions ~~~~~~~~~~~~~ @@ -51,7 +90,7 @@ not understanding the new rules. A room version is defined as a string of characters which MUST NOT exceed 32 codepoints in length. Room versions MUST NOT be empty and SHOULD contain only -the characters ``a-z``, ``0-9``, ``.``, and ``-``. +the characters ``a-z``, ``0-9``, ``.``, and ``-``. Room versions are not intended to be parsed and should be treated as opaque identifiers. Room versions consisting only of the characters ``0-9`` and ``.`` From 7f9524f8016f4cdfb35c0434cd45619859326f87 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 00:07:36 +0100 Subject: [PATCH 31/58] =?UTF-8?q?s/--/=E2=80=94/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- specification/appendices/identifier_grammar.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index d068b768..e686f0c6 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -43,7 +43,7 @@ following grammar:: dns-char = DIGIT / ALPHA / "-" / "." --- in other words, the server name is the hostname, followed by an optional +— in other words, the server name is the hostname, followed by an optional numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, an IPv6 address literal surrounded with square brackets, or a DNS name. From 8679a642b49d865c628ef6f598f374f273e6e8b1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 00:19:05 +0100 Subject: [PATCH 32/58] clarity --- specification/appendices/identifier_grammar.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index e686f0c6..92db2fec 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -48,13 +48,12 @@ numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, an IPv6 address literal surrounded with square brackets, or a DNS name. IPv4 literals must be a sequence of four decimal numbers in the -range 0 to 255, separated by ".". IPv6 literals must be as specified by +range 0 to 255, separated by ``.``. IPv6 literals must be as specified by `RFC3513, section 2.2 `_. DNS names for use with Matrix should follow the conventional restrictions for internet hostnames: they should consist of a series of labels separated by -dots, where each label consists of the alphanumeric characters or -hyphens. +``.``, where each label consists of the alphanumeric characters or hyphens. Examples of valid server names are: From d91395cf9fe0c368436fd810a11959412b2aa979 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 19:33:45 -0600 Subject: [PATCH 33/58] Document 403 error for sending state events Fixes https://github.com/matrix-org/matrix-doc/issues/1399 --- api/client-server/room_state.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/api/client-server/room_state.yaml b/api/client-server/room_state.yaml index c04fb803..bda66eb8 100644 --- a/api/client-server/room_state.yaml +++ b/api/client-server/room_state.yaml @@ -87,6 +87,16 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation "/rooms/{roomId}/state/{eventType}": @@ -142,5 +152,15 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation From d1add1f58c983c5ece0994c7d7c3ef80d0841d0a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 28 Aug 2018 19:34:58 -0600 Subject: [PATCH 34/58] Changelog --- changelogs/client_server/newsfragments/1590.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1590.clarification diff --git a/changelogs/client_server/newsfragments/1590.clarification b/changelogs/client_server/newsfragments/1590.clarification new file mode 100644 index 00000000..27999193 --- /dev/null +++ b/changelogs/client_server/newsfragments/1590.clarification @@ -0,0 +1 @@ +Document the 403 error for sending state events. From 120bb8dc8eb3261f71dfc0b27e4b42c76003f808 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 09:24:24 +0100 Subject: [PATCH 35/58] s/host/hostname/ --- specification/appendices/identifier_grammar.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index 92db2fec..1a46ae1a 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -26,11 +26,11 @@ The server name represents the address at which the homeserver in question can be reached by other homeservers. All valid server names are included by the following grammar:: - server_name = host [ ":" port ] + server_name = hostname [ ":" port ] port = *DIGIT - host = IPv4address / "[" IPv6address "]" / dns-name + hostname = IPv4address / "[" IPv6address "]" / dns-name IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT From 08fba5de51b09e3abecf6247218146fa917f9576 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 09:25:38 +0100 Subject: [PATCH 36/58] Link to grammar rather than duplicating. --- specification/server_server_api.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 439b35f9..910dab1d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -78,14 +78,7 @@ Resolving Server Names ~~~~~~~~~~~~~~~~~~~~~~ Each matrix homeserver is identified by a server name consisting of a hostname -and an optional TLS port. - -.. code:: - - server_name = hostname [ ":" tls_port] - tls_port = *DIGIT - -.. ** +and an optional port, as described by the `grammar <../appendices.html#server-name>`_. If the port is present then the server is discovered by looking up an AAAA or A record for the hostname and connecting to the specified TLS port. If the port From 78d81188468ba80e2a6d75566a70472d35a81089 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 09:36:50 +0100 Subject: [PATCH 37/58] Further server_name clarification --- specification/server_server_api.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 910dab1d..d9edfcb3 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -80,18 +80,22 @@ Resolving Server Names Each matrix homeserver is identified by a server name consisting of a hostname and an optional port, as described by the `grammar <../appendices.html#server-name>`_. -If the port is present then the server is discovered by looking up an AAAA or -A record for the hostname and connecting to the specified TLS port. If the port -is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV -record for the hostname. If this record does not exist then the server is -discovered by looking up an AAAA or A record on the hostname and taking the -default fallback port number of 8448. +If the hostname is an IP literal, then that IP address should be used, together +with the given port number, or 8448 if no port is given. + +Otherwise, if the port is present, then an IP address is discovered by looking +up an AAAA or A record for the hostname, and the specified port is used. + +If the hostname is not an IP literal and no port is given, the server is +discovered by first looking up a ``_matrix._tcp`` SRV record for the hostname, +which may give a hostname (to be looked up using AAAA or A queries) and port. +If the SRV record does not exist, then the server is discovered by looking up +an AAAA or A record on the hostname and taking the default fallback port number +of 8448. + Homeservers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. -If the DNS name is a literal IP address, the port specified or the fallback -port should be used. - When making requests to servers, use the DNS name of the target server in the ``Host`` header, regardless of the host given in the SRV record. For example, if making a request to ``example.org``, and the SRV record resolves to ``matrix. From d493c82e82c49c8f252cb06554a51f7ef136be11 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 10:04:02 +0100 Subject: [PATCH 38/58] even more clarification --- specification/server_server_api.rst | 54 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d9edfcb3..8cce792b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -71,36 +71,40 @@ This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. -Server Discovery +Server discovery ---------------- -Resolving Server Names +Resolving server names ~~~~~~~~~~~~~~~~~~~~~~ Each matrix homeserver is identified by a server name consisting of a hostname -and an optional port, as described by the `grammar <../appendices.html#server-name>`_. - -If the hostname is an IP literal, then that IP address should be used, together -with the given port number, or 8448 if no port is given. - -Otherwise, if the port is present, then an IP address is discovered by looking -up an AAAA or A record for the hostname, and the specified port is used. - -If the hostname is not an IP literal and no port is given, the server is -discovered by first looking up a ``_matrix._tcp`` SRV record for the hostname, -which may give a hostname (to be looked up using AAAA or A queries) and port. -If the SRV record does not exist, then the server is discovered by looking up -an AAAA or A record on the hostname and taking the default fallback port number -of 8448. - -Homeservers may use SRV records to load balance requests between multiple TLS -endpoints or to failover to another endpoint if an endpoint fails. - -When making requests to servers, use the DNS name of the target server in the -``Host`` header, regardless of the host given in the SRV record. For example, -if making a request to ``example.org``, and the SRV record resolves to ``matrix. -example.org``, the ``Host`` header in the request should be ``example.org``. The -port number for target server should not appear in the ``Host`` header. +and an optional port, as described by the `grammar +<../appendices.html#server-name>`_. Server names should be resolved to an IP +address and port using the following process: + +* If the hostname is an IP literal, then that IP address should be used, + together with the given port number, or 8448 if no port is given. + +* Otherwise, if the port is present, then an IP address is discovered by + looking up an AAAA or A record for the hostname, and the specified port is + used. + +* If the hostname is not an IP literal and no port is given, the server is + discovered by first looking up a ``_matrix._tcp`` SRV record for the + hostname, which may give a hostname (to be looked up using AAAA or A queries) + and port. If the SRV record does not exist, then the server is discovered by + looking up an AAAA or A record on the hostname and taking the default + fallback port number of 8448. + + Homeservers may use SRV records to load balance requests between multiple TLS + endpoints or to failover to another endpoint if an endpoint fails. + +When making requests to servers, use the hostname of the target server in the +``Host`` header, regardless of the any hostname given in the SRV record. For +example, if the server name is ``example.org``, and the SRV record resolves to +``matrix.example.org``, the ``Host`` header in the request should be +``example.org``. The port number for target server should **not** appear in the +``Host`` header. Server implementation ~~~~~~~~~~~~~~~~~~~~~~ From 688c8ebcd348736d97cad2970e2e0b7533a3e103 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 29 Aug 2018 10:17:20 -0400 Subject: [PATCH 39/58] specify how to handle multiple olm sessions with the same device --- specification/modules/end_to_end_encryption.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index fa461cc2..6f192596 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -391,6 +391,12 @@ this check, a client cannot be sure that the sender device owns the private part of the ed25519 key it claims to have in the Olm payload. This is crucial when the ed25519 key corresponds to a verified device. +If a client has multiple sessions established with another device, it should +use the session from which it last received a message. A client may expire old +sessions by defining a maximum number of olm sessions that it will maintain for +each device, and expiring sessions on a Least Recently Used basis. The maximum +number of olm sessions maintained per device should be at least 4. + ``m.megolm.v1.aes-sha2`` ~~~~~~~~~~~~~~~~~~~~~~~~ From 9d0fec3645fb83647b15eeb9a0819696674b259f Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 29 Aug 2018 10:25:24 -0400 Subject: [PATCH 40/58] add changelog --- changelogs/client_server/newsfragments/1596.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1596.clarification diff --git a/changelogs/client_server/newsfragments/1596.clarification b/changelogs/client_server/newsfragments/1596.clarification new file mode 100644 index 00000000..3dde069f --- /dev/null +++ b/changelogs/client_server/newsfragments/1596.clarification @@ -0,0 +1 @@ +specify how to handle multiple olm sessions with the same device \ No newline at end of file From de36d978397d9056c9e50e46a16998fbc308b627 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 08:53:20 -0600 Subject: [PATCH 41/58] Clarify which event is actually be excluded from /state_ids --- api/server-server/events.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index f0b30178..5a5ce717 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -49,8 +49,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization - chain for the events, excluding the requested event. + The fully resolved state for the room (excluding the requested event), + including the authorization chain for the events. schema: type: object properties: @@ -96,8 +96,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization - chain for the events, excluding the requested event. + The fully resolved state for the room (excluding the requested event), + including the authorization chain for the events. schema: type: object properties: From 5ff244f06b106a72fe209a26e7059922f096bdb1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 09:00:46 -0600 Subject: [PATCH 42/58] More clarification about how /state_ids works --- api/server-server/events.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 5a5ce717..c23163d7 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -49,8 +49,9 @@ paths: responses: 200: description: |- - The fully resolved state for the room (excluding the requested event), - including the authorization chain for the events. + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization + chain for the events. schema: type: object properties: @@ -96,8 +97,9 @@ paths: responses: 200: description: |- - The fully resolved state for the room (excluding the requested event), - including the authorization chain for the events. + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization + chain for the events. schema: type: object properties: From c83da453b565a6f1c7c5a4a8c0e4459d5aa19795 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 14:18:41 -0600 Subject: [PATCH 43/58] s/number/integer --- event-schemas/schema/m.room.power_levels | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index ae4e5589..9bb12993 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -88,10 +88,10 @@ properties: notifications: properties: room: - type: number + type: integer description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. additionalProperties: - type: number + type: integer description: |- The power level requirements for specific notification types. This is a mapping from ``key`` to power level for that notifications key. From 196159be9116f2d82626296bd7eef44ee845f473 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 29 Aug 2018 22:15:38 +0100 Subject: [PATCH 44/58] We should include the Host header if the port was explicit. --- specification/server_server_api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 8cce792b..7239c12d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -100,11 +100,12 @@ address and port using the following process: endpoints or to failover to another endpoint if an endpoint fails. When making requests to servers, use the hostname of the target server in the -``Host`` header, regardless of the any hostname given in the SRV record. For +``Host`` header, regardless of any hostname given in the SRV record. For example, if the server name is ``example.org``, and the SRV record resolves to ``matrix.example.org``, the ``Host`` header in the request should be -``example.org``. The port number for target server should **not** appear in the -``Host`` header. +``example.org``. If an explicit port was given in the server name, it should be +included in the ``Host`` header; otherwise, no port number should be given in +the ``Host`` header. Server implementation ~~~~~~~~~~~~~~~~~~~~~~ From 684d80c4228f87b0268581f3f52660e8fbbbb165 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 17:52:02 -0600 Subject: [PATCH 45/58] Revert changes to 3pid lookup types in the IS spec The validator doesn't know what a "3PID Medium" is, for example, so it throws exceptions. This does reduce clarity in the spec though. --- api/identity/lookup.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index 6f993ac7..f04436ef 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -121,8 +121,11 @@ paths: minItems: 2 maxItems: 2 items: - - type: 3PID Medium - - type: 3PID Address + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + - type: string + - type: string description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. required: - "threepids" @@ -147,9 +150,13 @@ paths: minItems: 3 maxItems: 3 items: - - type: 3PID Medium - - type: 3PID Address - - type: Matrix User ID + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + #- type: Matrix User ID + - type: string + - type: string + - type: string description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. required: - "threepids" From 2a20c11467db2e03851b456ec32bdf456273caab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 17:47:09 -0600 Subject: [PATCH 46/58] Take out the reference to the current version from the changelog --- specification/push_gateway.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index e4623887..74f9e35e 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -41,7 +41,6 @@ Other versions of this specification The following other versions are also available, in reverse chronological order: - `HEAD `_: Includes all changes since the latest versioned release. -- `r0.1.0 `_ Overview -------- From a46783eb2460198bef94ad004b99c6bfb247a0ec Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 17:47:09 -0600 Subject: [PATCH 47/58] Revert "Take out the reference to the current version from the changelog" This reverts commit 2a20c11467db2e03851b456ec32bdf456273caab. --- specification/push_gateway.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index 74f9e35e..e4623887 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -41,6 +41,7 @@ Other versions of this specification The following other versions are also available, in reverse chronological order: - `HEAD `_: Includes all changes since the latest versioned release. +- `r0.1.0 `_ Overview -------- From fd7cb22a282d2f0775212db86804d2ed334e6091 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 21:22:47 -0600 Subject: [PATCH 48/58] Fix event type example of m.forwarded_room_key --- event-schemas/examples/m.forwarded_room_key | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/examples/m.forwarded_room_key b/event-schemas/examples/m.forwarded_room_key index 8ab85c48..ef1d6180 100644 --- a/event-schemas/examples/m.forwarded_room_key +++ b/event-schemas/examples/m.forwarded_room_key @@ -10,5 +10,5 @@ "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" ] }, - "type": "m.room_key" + "type": "m.forwarded_room_key" } From b2092922e85ecf633092498dfa840c912d34c7f8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 21:51:26 -0600 Subject: [PATCH 49/58] Don't define a variable for the appservice major version --- scripts/gendoc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 7b900562..00febdf2 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -550,7 +550,6 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), - "%APPSERVICE_MAJOR_VERSION%": "unstable", "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } From 78487a01ecae59439fec57a078ee29abeaf9362f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 23:11:00 -0600 Subject: [PATCH 50/58] Remove references to initialSync from account data and receipts module Fixes https://github.com/matrix-org/matrix-doc/issues/695 Note: This commit leaves the /join endpoints alone because they say "and /sync" rather than just initialSync --- specification/modules/account_data.rst | 2 +- specification/modules/receipts.rst | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 1c031ee1..d0ed201a 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -31,7 +31,7 @@ The client recieves the account data as events in the ``account_data`` sections of a ``/sync``. These events can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` +``account_data`` section of a room in ``/sync``. ``m.tag`` events appearing in ``/events`` will have a ``room_id`` with the room the tags are for. diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a6d8cbf7..faba7b62 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -38,11 +38,10 @@ single ``event_id``. Client behaviour ---------------- -In ``/initialSync``, receipts are listed in a separate top level ``receipts`` -key. In ``/sync``, receipts are contained in the ``ephemeral`` block for a -room. New receipts that come down the event streams are deltas which update -existing mappings. Clients should replace older receipt acknowledgements based -on ``user_id`` and ``receipt_type`` pairs. For example:: +In ``/sync``, receipts are listed under the ``ephemeral`` array of events +for a given room. New receipts that come down the event streams are deltas +which update existing mappings. Clients should replace older receipt acknowledgements +based on ``user_id`` and ``receipt_type`` pairs. For example:: Client receives m.receipt: user = @alice:example.com From ab14f33060e372d176dc7c3ed5440193f46e4b81 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 07:58:10 -0600 Subject: [PATCH 51/58] Add link to the client-server ACLs module --- specification/server_server_api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 244a8b82..983da62f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1111,3 +1111,4 @@ that are too long. .. _`Inviting to a room`: #inviting-to-a-room .. _`Canonical JSON`: ../appendices.html#canonical-json .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 +.. _`Server ACLs`: ../client_server/unstable.html#module-server-acls From 4e90b32348155d6d09864cc66cf8c3b31abc809a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 30 Aug 2018 17:18:07 +0100 Subject: [PATCH 52/58] Set the encoding to UTF-8 on all file I/O Hopefully this will resolve issues with building the spec on systems where the default encoding is somthing other than UTF-8. --- scripts/gendoc.py | 21 ++++++++++---------- scripts/templating/matrix_templates/units.py | 18 ++++++++--------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 8745d3e2..0719b48f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) - with open(os.path.join(spec_dir, file_info), "r") as f: + with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f: rst = None if adjust_titles: rst = load_with_adjusted_titles( @@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): def build_spec(target, out_filename): log("Building templated file %s" % out_filename) - with open(out_filename, "wb") as outfile: + with open(out_filename, "w", encoding="utf-8") as outfile: for file_info in target["files"]: section = get_rst( file_info=file_info, @@ -195,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section.encode('UTF-8')) + outfile.write(section) """ @@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename): "^[" + re.escape("".join(title_styles)) + "]{3,}$" ) current_title_style = None - with open(filename, "r") as infile: - with open(out_filename, "w") as outfile: + with open(filename, "r", encoding="utf-8") as infile: + with open(out_filename, "w", encoding="utf-8") as outfile: for line in infile.readlines(): if not relative_title_matcher.match(line): if title_matcher.match(line): @@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename): def rst2html(i, o, stylesheets): log("rst2html %s -> %s" % (i, o)) - with open(i, "r") as in_file: - with open(o, "w") as out_file: + with open(i, "r", encoding="utf-8") as in_file: + with open(o, "w", encoding="utf-8") as out_file: publish_file( source=in_file, destination=out_file, @@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "rb") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() replacement = r'

\n\1' - with open(path, "wb") as f: + with open(path, "w", encoding="utf-8") as f: for line in lines: - line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write((line + "\n").encode('UTF-8')) + f.write(line + "\n") def run_through_template(input_files, set_verbose, substitutions): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 219299cd..6f9eeaa2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -125,7 +125,7 @@ def resolve_references(path, schema): if '$ref' in schema: value = schema['$ref'] path = os.path.join(os.path.dirname(path), value) - with open(path) as f: + with open(path, encoding="utf-8") as f: ref = yaml.load(f, OrderedLoader) result = resolve_references(path, ref) del schema['$ref'] @@ -664,11 +664,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger API: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") group_name = "%s_%s" % (group_name, suffix) - api = yaml.load(f.read(), OrderedLoader) + api = yaml.load(f, OrderedLoader) api = resolve_references(filepath, api) api["__meta"] = self._load_swagger_meta( api, group_name @@ -698,11 +698,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger definition: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) group_name = "%s_%s" % (prefix, group_name) - definition = yaml.load(f.read(), OrderedLoader) + definition = yaml.load(f, OrderedLoader) definition = resolve_references(filepath, definition) if 'type' not in definition: continue @@ -741,7 +741,7 @@ class MatrixUnits(Units): event_type = filename[:-5] # strip the ".yaml" logger.info("Reading event schema: %s" % filepath) - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: event_schema = yaml.load(f, OrderedLoader) schema_info = process_data_type( @@ -792,7 +792,7 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) logger.info("Reading event example: %s" % filepath) try: - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: example = json.load(f) examples[filename] = examples.get(filename, []) examples[filename].append(example) @@ -830,7 +830,7 @@ class MatrixUnits(Units): def read_event_schema(self, filepath): logger.info("Reading %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: json_schema = yaml.load(f, OrderedLoader) schema = { @@ -942,7 +942,7 @@ class MatrixUnits(Units): title_part = None changelog_lines = [] - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() prev_line = None for line in (tc_lines + lines): From 05b017276c81b2a0ce0edb23b6c5c8990b86557e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 10:23:12 -0600 Subject: [PATCH 53/58] Don't accidentally limit application services to one homeserver --- specification/application_service_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 09ecd4ff..45ee75a9 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -54,8 +54,8 @@ The following other versions are also available, in reverse chronological order: Application Services -------------------- -Application services are passive and can only observe events from a given -homeserver. They can inject events into rooms they are participating in. +Application services are passive and can only observe events from homeserver. +They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the From ecd3ce39e54dd35eab60b2c8c8da900bfbf34b8b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 10:29:59 -0600 Subject: [PATCH 54/58] Don't generate proposals.rst in Circle CI --- scripts/generate-matrix-org-assets | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index ed08f81d..76032850 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,8 +8,11 @@ cd `dirname $0`/.. mkdir -p assets -# generate specification/proposals.rst -./scripts/proposals.py +if [ "$CIRCLECI" != "true" ] +then + # generate specification/proposals.rst + ./scripts/proposals.py +fi # generate the spec docs ./scripts/gendoc.py -d assets/spec From a48f7b9278d7810b8727e266e566b1a81b9c24c8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 10:49:05 -0600 Subject: [PATCH 55/58] Clarify how external_url is supposed to work Fixes https://github.com/matrix-org/matrix-doc/issues/1624 --- specification/application_service_api.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 5b7abf4a..127b3ade 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -374,13 +374,14 @@ additional parameters on the ``/publicRooms`` client-server endpoint. {{appservice_room_directory_cs_http_api}} -Event fields -~~~~~~~~~~~~ +Referencing messages from a third party network +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. TODO-TravisR: Fix this section to be a general "3rd party networks" section +Application services should include an ``external_url`` in the ``content`` of +events it emits to indicate where the message came from. This typically applies +to application services that bridge other networks into Matrix, such as IRC, +where an HTTP URL may be available to reference. -We recommend that any events that originated from a remote network should -include an ``external_url`` field in their content to provide a way for Matrix -clients to link into the 'native' client from which the event originated. -For instance, this could contain the message-ID for emails/nntp posts, or a link -to a blog comment when bridging blog comment traffic in & out of Matrix. +Clients should provide users with a way to access the ``external_url`` if it +is present. Clients should additionally ensure the URL has a scheme of ``https`` +or ``http`` before making use of it. From be5c5660282a74128047aa3ec98df1fcbffe0ca5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 11:03:42 -0600 Subject: [PATCH 56/58] Version all appservice endpoints and provide a fallback Fixes https://github.com/matrix-org/matrix-doc/issues/1616 --- api/application-service/protocols.yaml | 12 +++++----- api/application-service/query_room.yaml | 2 +- api/application-service/query_user.yaml | 2 +- api/application-service/transactions.yaml | 2 +- specification/application_service_api.rst | 28 +++++++++++++++++++++++ 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index e6489cc5..f1dd39b0 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -19,13 +19,13 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json paths: - "/_matrix/app/unstable/thirdparty/protocol/{protocol}": + "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- @@ -72,7 +72,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user/{protocol}": + "/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- @@ -125,7 +125,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location/{protocol}": + "/thirdparty/location/{protocol}": get: summary: Retrieve Matrix-side portal rooms leading to a third party location. description: |- @@ -176,7 +176,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location": + "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. description: |- @@ -221,7 +221,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user": + "/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. description: |- diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index b885cb86..22f15dcf 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -20,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 0431b5e4..af7dec5c 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -20,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 8735cc8f..9bedc502 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -20,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 produces: - application/json paths: diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 5b7abf4a..3aa8a8ed 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -169,6 +169,34 @@ An example registration file for an IRC-bridging application service is below: Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Legacy routes ++++++++++++++ + +Previous drafts of the application service specification had a mix of endpoints +that have been used in the wild for a significant amount of time. The application +service specification now defines a version on all endpoints to be more compatible +with the rest of the Matrix specification and the future. + +Homeservers should attempt to use the specified endpoints first when communicating +with application services. However, if the application service receives an http status +code that does not indicate success (ie: 404, 500, 501, etc) then the homeserver +should fall back to the older endpoints for the application service. + +The older endpoints have the exact same request body and response format, they +just belong at a different path. The equivalent path for each is as follows: + +* ``/_matrix/app/v1/transactions/{txnId}`` becomes ``/transactions/{txnId}`` +* ``/_matrix/app/v1/users/{userId}`` becomes ``/users/{userId}`` +* ``/_matrix/app/v1/rooms/{roomAlias}`` becomes ``/rooms/{roomAlias}`` +* ``/_matrix/app/v1/thirdparty/protocol/{protocol}`` becomes ``/_matrix/app/unstable/thirdparty/protocol/{protocol}`` +* ``/_matrix/app/v1/thirdparty/user/{user}`` becomes ``/_matrix/app/unstable/thirdparty/user/{user}`` +* ``/_matrix/app/v1/thirdparty/location/{location}`` becomes ``/_matrix/app/unstable/thirdparty/location/{location}`` +* ``/_matrix/app/v1/thirdparty/user`` becomes ``/_matrix/app/unstable/thirdparty/user`` +* ``/_matrix/app/v1/thirdparty/location`` becomes ``/_matrix/app/unstable/thirdparty/location`` + +Homeservers should periodically try again for the newer endpoints because the +application service may have been updated. + Pushing events ++++++++++++++ From 1a3f11233731d92a9dec01dd8b207c62c27c528e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 11:07:09 -0600 Subject: [PATCH 57/58] Clarify that the external_url is unsafe --- specification/application_service_api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 127b3ade..cd65fad7 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -385,3 +385,7 @@ where an HTTP URL may be available to reference. Clients should provide users with a way to access the ``external_url`` if it is present. Clients should additionally ensure the URL has a scheme of ``https`` or ``http`` before making use of it. + +The presence of an ``external_url`` on an event does not necessarily mean the +event was sent from an application service. Clients should be wary of the URL +contained within, as it may not be a legitimate reference to the event's source. From 138419cdd48780e7b9b1e0e41704563eca179805 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 11:19:55 -0600 Subject: [PATCH 58/58] fix indentation on release process doc --- meta/releasing_a_spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index 3a9da893..ac3d21fa 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -27,7 +27,7 @@ The remainder of the process is as follows: 1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). * This is done by making a PR to the `unstyled_docs/spec` folder for the version and specification you're releasing. - * Don't forget to symlink the new release as `latest`. + * Don't forget to symlink the new release as `latest`. 1. Perform a release on GitHub to tag the release. 1. Yell from the mountaintop to the world about the new release.