From aab72c3d1417ae1eb8cb3288172ec8594fa2c36f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 5 Apr 2021 10:49:45 -0600 Subject: [PATCH 01/11] Move drafts to attic to reduce confusion --- .../drafts}/ancient_federated_versioning_design_notes.rst | 0 {drafts => attic/drafts}/application_services.rst | 0 {drafts => attic/drafts}/data_flows.rst | 0 {drafts => attic/drafts}/erik-model.rst | 0 {drafts => attic/drafts}/erikj_federation.rst | 0 {drafts => attic/drafts}/flows_and_auth.rst | 0 {drafts => attic/drafts}/general_api.rst | 0 {drafts => attic/drafts}/human-id-rules.rst | 0 {drafts => attic/drafts}/model/presence.rst | 0 {drafts => attic/drafts}/model/profiles.rst | 0 {drafts => attic/drafts}/model/protocol_examples.rst | 0 {drafts => attic/drafts}/model/room-join-workflow.rst | 0 {drafts => attic/drafts}/model/rooms.rst | 0 {drafts => attic/drafts}/model/third-party-id.rst | 0 {drafts => attic/drafts}/object_model.rst | 0 {drafts => attic/drafts}/pstn_gatewaying.txt | 0 {drafts => attic/drafts}/reputation_thoughts.rst | 0 {drafts => attic/drafts}/use_cases.rst | 0 {drafts => attic/drafts}/websockets.rst | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename {drafts => attic/drafts}/ancient_federated_versioning_design_notes.rst (100%) rename {drafts => attic/drafts}/application_services.rst (100%) rename {drafts => attic/drafts}/data_flows.rst (100%) rename {drafts => attic/drafts}/erik-model.rst (100%) rename {drafts => attic/drafts}/erikj_federation.rst (100%) rename {drafts => attic/drafts}/flows_and_auth.rst (100%) rename {drafts => attic/drafts}/general_api.rst (100%) rename {drafts => attic/drafts}/human-id-rules.rst (100%) rename {drafts => attic/drafts}/model/presence.rst (100%) rename {drafts => attic/drafts}/model/profiles.rst (100%) rename {drafts => attic/drafts}/model/protocol_examples.rst (100%) rename {drafts => attic/drafts}/model/room-join-workflow.rst (100%) rename {drafts => attic/drafts}/model/rooms.rst (100%) rename {drafts => attic/drafts}/model/third-party-id.rst (100%) rename {drafts => attic/drafts}/object_model.rst (100%) rename {drafts => attic/drafts}/pstn_gatewaying.txt (100%) rename {drafts => attic/drafts}/reputation_thoughts.rst (100%) rename {drafts => attic/drafts}/use_cases.rst (100%) rename {drafts => attic/drafts}/websockets.rst (100%) diff --git a/drafts/ancient_federated_versioning_design_notes.rst b/attic/drafts/ancient_federated_versioning_design_notes.rst similarity index 100% rename from drafts/ancient_federated_versioning_design_notes.rst rename to attic/drafts/ancient_federated_versioning_design_notes.rst diff --git a/drafts/application_services.rst b/attic/drafts/application_services.rst similarity index 100% rename from drafts/application_services.rst rename to attic/drafts/application_services.rst diff --git a/drafts/data_flows.rst b/attic/drafts/data_flows.rst similarity index 100% rename from drafts/data_flows.rst rename to attic/drafts/data_flows.rst diff --git a/drafts/erik-model.rst b/attic/drafts/erik-model.rst similarity index 100% rename from drafts/erik-model.rst rename to attic/drafts/erik-model.rst diff --git a/drafts/erikj_federation.rst b/attic/drafts/erikj_federation.rst similarity index 100% rename from drafts/erikj_federation.rst rename to attic/drafts/erikj_federation.rst diff --git a/drafts/flows_and_auth.rst b/attic/drafts/flows_and_auth.rst similarity index 100% rename from drafts/flows_and_auth.rst rename to attic/drafts/flows_and_auth.rst diff --git a/drafts/general_api.rst b/attic/drafts/general_api.rst similarity index 100% rename from drafts/general_api.rst rename to attic/drafts/general_api.rst diff --git a/drafts/human-id-rules.rst b/attic/drafts/human-id-rules.rst similarity index 100% rename from drafts/human-id-rules.rst rename to attic/drafts/human-id-rules.rst diff --git a/drafts/model/presence.rst b/attic/drafts/model/presence.rst similarity index 100% rename from drafts/model/presence.rst rename to attic/drafts/model/presence.rst diff --git a/drafts/model/profiles.rst b/attic/drafts/model/profiles.rst similarity index 100% rename from drafts/model/profiles.rst rename to attic/drafts/model/profiles.rst diff --git a/drafts/model/protocol_examples.rst b/attic/drafts/model/protocol_examples.rst similarity index 100% rename from drafts/model/protocol_examples.rst rename to attic/drafts/model/protocol_examples.rst diff --git a/drafts/model/room-join-workflow.rst b/attic/drafts/model/room-join-workflow.rst similarity index 100% rename from drafts/model/room-join-workflow.rst rename to attic/drafts/model/room-join-workflow.rst diff --git a/drafts/model/rooms.rst b/attic/drafts/model/rooms.rst similarity index 100% rename from drafts/model/rooms.rst rename to attic/drafts/model/rooms.rst diff --git a/drafts/model/third-party-id.rst b/attic/drafts/model/third-party-id.rst similarity index 100% rename from drafts/model/third-party-id.rst rename to attic/drafts/model/third-party-id.rst diff --git a/drafts/object_model.rst b/attic/drafts/object_model.rst similarity index 100% rename from drafts/object_model.rst rename to attic/drafts/object_model.rst diff --git a/drafts/pstn_gatewaying.txt b/attic/drafts/pstn_gatewaying.txt similarity index 100% rename from drafts/pstn_gatewaying.txt rename to attic/drafts/pstn_gatewaying.txt diff --git a/drafts/reputation_thoughts.rst b/attic/drafts/reputation_thoughts.rst similarity index 100% rename from drafts/reputation_thoughts.rst rename to attic/drafts/reputation_thoughts.rst diff --git a/drafts/use_cases.rst b/attic/drafts/use_cases.rst similarity index 100% rename from drafts/use_cases.rst rename to attic/drafts/use_cases.rst diff --git a/drafts/websockets.rst b/attic/drafts/websockets.rst similarity index 100% rename from drafts/websockets.rst rename to attic/drafts/websockets.rst From 52cd88f070d3d17216e8f34309eb4d2590121041 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 5 Apr 2021 11:36:11 -0600 Subject: [PATCH 02/11] Rewrite readme and update contributor docs Incorporates https://github.com/matrix-org/matrix-doc/pull/3025/ --- .gitignore | 1 + CONTRIBUTING.rst | 12 +-- README.md | 97 ++++++++++++++++++++++++ README.rst | 143 ----------------------------------- meta/documentation_style.rst | 40 +++------- 5 files changed, 117 insertions(+), 176 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/.gitignore b/.gitignore index 5245c82b..cce5848f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ _rendered.rst /.vscode/ /.idea/ +/spec/ \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2d26e8a8..b9c66805 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -29,9 +29,7 @@ some time to complete. Changes to the protocol (new endpoints, ideas, etc) need to go through the `proposals process `_. Other changes, such as fixing bugs, typos, or clarifying existing behaviour do not need a proposal. -If you're not sure, visit us at `#matrix-spec:matrix.org`_ -and ask. - +If you're not sure, visit us at `#matrix-spec:matrix.org`_ and ask. Other changes ~~~~~~~~~~~~~ @@ -64,12 +62,17 @@ following: to fix. On the other hand, introducing new behaviour is best represented by a proposal. +* Design or aesthetic changes, such as improving accessibility, colour schemes, + etc. Please check in with us at `#matrix-docs:matrix.org`_ with your proposed + design change before opening a PR so we can work with you on it. + For such changes, please do just open a `pull request`_. If you're not sure if your change is covered by the above, please visit `#matrix-spec:matrix.org` and ask. .. _`pull request`: https://help.github.com/articles/about-pull-requests .. _`#matrix-spec:matrix.org`: https://matrix.to/#/#matrix-spec:matrix.org +.. _`#matrix-docs:matrix.org`: https://matrix.to/#/#matrix-docs:matrix.org Adding to the changelog @@ -100,8 +103,7 @@ the ``newsfragments`` directory. The ``type`` can be one of the following: All news fragments must have a brief summary explaining the change in the contents of the file. The summary must end in a full stop to be in line with -the style guide and and formatting must be done using `Restructured Text -`_. +the style guide and and formatting must be done using Markdown. Changes that do not change the spec, such as changes to the build script, formatting, CSS, etc should not get a news fragment. diff --git a/README.md b/README.md new file mode 100644 index 00000000..a9d8dd9b --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +# Matrix Specification + +This repository contains the Matrix Specification, rendered at [spec.matrix.org](http://spec.matrix.org/). + +Developers looking to use Matrix should join [#matrix-dev:matrix.org](http://matrix.to/#/#matrix-dev:matrix.org) +on Matrix for help. + +Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](http://matrix.to/#/#matrix-spec:matrix.org). We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details. + +## Structure + +The Matrix spec is compiled with [Hugo](https://gohugo.io/) (a static site generator) with the following structure: + +* `/assets`: assets that need postprocessing using [Hugo Pipes](https://gohugo.io/hugo-pipes/introduction/). + For example, Sass files would go here. + +* `/content`: files that will become pages in the site go here. Typically these are Markdown files with some YAML front + matter indicating, [among other things](https://gohugo.io/content-management/front-matter/), what layout should be + applied to this page. The organization of files under `/content` determines the organization of pages in the built + site. + +* `/data`: this can contain TOML, YAML, or JSON files. Files kept here are directly available to template code as + [data objects](https://gohugo.io/templates/data-templates/), so templates don't need to load them from a file and + parse them. This is also where our + +* `/layouts`: this contains [Hugo templates](https://gohugo.io/templates/). Some templates define the overall layout of + a page: for example, whether it has header, footer, sidebar, and so on. + * `/layouts/partials`: these templates can be called from other templates, so they can be used to factor out + template code that's used in more than one template. An obvious example here is something like a sidebar, where + several different page layouts might all include the sidebar. But also, partial templates can return values: this + means they can be used like functions, that can be called by multiple templates to do some common processing. + * `/layouts/shortcodes`: these templates can be called directly from files in `/content`. + +* `/static`: static files which don't need preprocessing. JS or CSS files could live here. + +* `/themes`: you can use just Hugo or use it with a theme. Themes primarily provide additional templates, which are + supplied in a `/themes/$theme_name/layouts` directory. You can use a theme but customise it by providing your own + versions of any of the them layouts in the base `/layouts` directory. That is, if a theme provides + `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of this + template will be used. + +It also has the following top-level file: + +* `config.toml`: site-wide configuration settings. Some of these are built-in and you can add your own. Config settings + defined here are available in templates. All these directories above are configurable via `config.toml` settings. + +Additionally, the following directories may be of interest: + +* `/attic`: Here contains historical sections of specification and legacy drafts for the specification. +* `/changelogs`: Various bits of changelog for the specification areas. +* `/event-schemas`: [JSON Schema](http://json-schema.org/) definitions for the spec. +* `/data-definitions`: Bits of structured data consumable by Matrix implementations. +* `/meta`: Documentation relating to the spec's processes that are otherwise untracked (release instructions, etc). +* `/scripts`: Various scripts for generating the spec. +* `/proposals`: Matrix Spec Change (MSC) proposals. See . +* `/api`: [OpenAPI](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md) / Swagger definitions for + the spec. + +## Authoring changes to the spec + +Please read [CONTRIBUTING.rst](./CONTRIBUTING.rst) before authoring a change to the spec. Note that spec authoring takes +place after an MSC has been accepted, not as part of a proposal itself. + +1. Install the extended version (often the OS default) of Hugo: +2. Run `git submodule update --init --recursive` for good measure. +3. Run `npm i` to install the dependencies. Note that this will require NodeJS to be installed. +4. Run `npm run get-proposals` to seed the proposals data. This is not required. +5. Run `hugo serve` to run a local webserver which builds whenever a file change is detected. If watching doesn't appear + to be working for you, try `hugo serve --disableFastRender` instead. +6. Edit the specification 🙂 + +We use a highly customized [Docsy](https://www.docsy.dev/) theme for our generated site, which uses Bootstrap and Font +Awesome. If you're looking at making design-related changes to the spec site, please coordinate with us in +[#matrix-docs:matrix.org](https://matrix.to/#/#matrix-docs:matrix.org) before opening a PR. + +## Building the specification + +If for some reason you're not a CI/CD system and want to render the spec yourself, follow the above steps for authoring +changes to the specification and instead of `hugo serve` run `hugo -d "spec"` - this will generate the spec to `/spec`. +If you'd like to serve the spec off a path instead of a domain root (eg: `/unstable`), add `--baseURL "/unstable"` to +the `hugo -d "spec"` command. + +For building the swagger definitions, create a python3 virtualenv and activate it. Then run `pip install -r ./scripts/requirements.txt` and finally `python ./scripts/dump-swagger.py` to generate it to `./scripts/swagger/api-docs.json`. +To make use of the generated file, there are a number of options: + +* It can be uploaded from your filesystem to an online editor/viewer such as + http://editor.swagger.io/ +* You can run a local HTTP server by running `./scripts/swagger-http-server.py`, and then view the documentation via an + online viewer; for example, at +* You can host the swagger UI yourself. See for advice on how to + do so. + +## Issue tracking + +Specification issues are tracked on github at . + +See [meta/github-labels.rst](./meta/github-labels.rst) for information on what the labels mean. diff --git a/README.rst b/README.rst deleted file mode 100644 index 61c27f15..00000000 --- a/README.rst +++ /dev/null @@ -1,143 +0,0 @@ -This repository contains the Matrix specification. - -If you want to ask more about the specification, join us on -`#matrix-dev:matrix.org `_. - -We welcome contributions to the spec! See the notes below on `Building the -specification`_, and ``_ to get started making contributions. - -Note that the Matrix Project lists, which were previously kept in this -repository, are now in https://github.com/matrix-org/matrix.org. - -Structure of this repository -============================ - -- ``api`` : `OpenAPI`_ (swagger) specifications for the the HTTP APIs. -- ``attic``: historical sections of specification for reference - purposes. -- ``changelogs``: change logs for the various parts of the - specification. -- ``drafts``: Previously, contained documents which were under discussion for - future incusion into the specification and/or supporting documentation. This - is now historical, as we use separate discussion documents (see - ``_). -- ``event-schemas``: the `JSON Schema`_ for all Matrix events - contained in the specification, along with example JSON files. -- ``meta``: documents outlining the processes involved when writing - documents, e.g. documentation style, guidelines. -- ``scripts``: scripts to generate formatted versions of the - documentation, typically HTML. -- ``specification``: the specification split up into sections. - -.. _OpenAPI: https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md -.. _JSON Schema: http://json-schema.org/ - -Building the specification -========================== - -The Matrix Spec is generated by a set of scripts, from the RST documents, API -specs and event schemas in this repository. - -Preparation ------------ - -To use the scripts, it is best to create a Python 3.4+ virtualenv as follows:: - - virtualenv -p python3 env - env/bin/pip install -r scripts/requirements.txt - -(Benjamin Saunders has contributed a script for `Nix`_ users, which can be -invoked with ``nix-shell scripts/contrib/shell.nix``.) - -.. TODO: Possibly we need some libs installed; should record what they are. - -.. _`Nix`: https://nixos.org/nix/ - -Generating the specification ----------------------------- - -To rebuild the specification, use ``scripts/gendoc.py``:: - - source env/bin/activate - ./scripts/gendoc.py - -The above will write the rendered version of the specification to -``scripts/gen``. To view it, point your browser at ``scripts/gen/index.html``. - -Windows users -~~~~~~~~~~~~~ -The ``source`` program does not exist on Windows, so instead run one of the -``activate`` files in ``.\env\Scripts\`` to activate the virtual environment. - -If you're on Windows Vista or higher, be sure that the "Symbolic Links" -option was selected when installing Git prior to cloning this repository. If -you're still seeing errors about files not being found it is likely because -the symlink at ``api/client-server/definitions/event-schemas`` looks like a -file. To correct the problem, open an Administrative/Elevated Command Prompt in your -cloned matrix-doc directory and run the following:: - - cd api\client-server\definitions - del event-schemas - mklink /D event-schemas "..\..\..\event-schemas" - -This will delete the file and replace it with a symlink. Git should not detect -this as a change, and you should be able to go back to building the project. - -Generating the OpenAPI (Swagger) specs --------------------------------------- - -`Swagger`_ is a framework for representing RESTful APIs. We use it to generate -interactive documentation for our APIs. - -Before the Swagger docs can be used in the Swagger UI (or other tool expecting -a Swagger specs, they must be combined into a single json file. This can be -done as follows:: - - source env/bin/activate - ./scripts/dump-swagger.py - -By default, ``dump-swagger`` will write to ``scripts/swagger/api-docs.json``. - -To make use of the generated file, there are a number of options: - -* It can be uploaded from your filesystem to an online editor/viewer such as - http://editor.swagger.io/ -* You can run a local HTTP server by running - ``./scripts/swagger-http-server.py``, and then view the documentation via an - online viewer; for example, at - http://petstore.swagger.io/?url=http://localhost:8000/api-docs.json -* You can host the swagger UI yourself. See - https://github.com/swagger-api/swagger-ui#how-to-run for advice on how to do - so. - -.. _`Swagger`: http://swagger.io/ - -Continuserv ------------ - -Continuserv is a script which will rebuild the specification every time a file -is changed, and will serve it to a browser over HTTP. It is intended for use by -specification authors, so that they can quickly see the effects of their -changes. - -It is written in Go, so you will need the ``go`` compiler installed on your -computer. You will also need to install fsnotify by running:: - - go get gopkg.in/fsnotify/fsnotify.v1 - -Then, create a virtualenv as described above under `Preparation`_, -and:: - - source env/bin/activate - go run ./scripts/continuserv/main.go - -You will then be able to view the generated spec by visiting -http://localhost:8000/index.html. - -Issue tracking -============== - -Issues with the Matrix specification are tracked in `GitHub -`_. - -See `meta/github-labels.rst `_ for notes on what the labels mean. diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index e3d363c4..5528f73e 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -8,52 +8,36 @@ in. Format ------ -Documentation is written either in github-flavored markdown or RST. +Documentation is written either in github-flavored markdown. Sections -------- -RST support lots of different punctuation characters for underlines on sections. -Content in the specification MUST use the same characters in order for the -complete specification to be merged correctly. These characters are: - -- ``=`` -- ``-`` -- ``~`` -- ``+`` -- ``^`` -- \ ````` -- ``@`` -- ``:`` - -If you find yourself using ``^`` or beyond, you should rethink your document -layout if possible. +Markdown supports headings through the `#` prefix on text. Please avoid heavily +nested titles (h6, or 6 `#` characters) and instead re-evaluate the document structure. Correct capitalisation for long section names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Headings should start with a capital letter, and use lower-case otherwise. - +Headings should start with a capital letter, and use lower-case otherwise. This +document is an example of what we mean. TODOs ----- -Any RST file in this repository may make it onto ``matrix.org``. We do not want -``TODO`` markers visible there. For internal comments, notes, TODOs, use standard -RST comments like so:: - - .. TODO-Bob - There is something to do here. This will not be rendered by something like - rst2html.py so it is safe to put internal comments here. - -You SHOULD put your username with the TODO so we know who to ask about it. +Any file in this repository might make it onto the matrix.org site, and as such +we do not want ``TODO`` markers visible there. For internal comments, notes, TODOs, +etc please use standard markdown comments (``). Please +include your name in the TODO comment so we know who to ask about it in the future. Line widths ----------- -We use 80 characters for line widths. This is a guideline and can be flouted IF +We use 80 characters for line widths. This is a guideline and can be ignored IF AND ONLY IF it makes reading more legible. Use common sense. +For proposals, please use 120 characters as a guide. + Stylistic notes --------------- From 726be0484183d1f84a8dff4415182f883e7e7b61 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 23 Aug 2021 18:43:17 +0100 Subject: [PATCH 03/11] remove documentation of non-existent directory --- README.md | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a9d8dd9b..009b94e6 100644 --- a/README.md +++ b/README.md @@ -11,37 +11,37 @@ Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org]( The Matrix spec is compiled with [Hugo](https://gohugo.io/) (a static site generator) with the following structure: -* `/assets`: assets that need postprocessing using [Hugo Pipes](https://gohugo.io/hugo-pipes/introduction/). +* `/assets`: assets that need postprocessing using [Hugo Pipes](https://gohugo.io/hugo-pipes/introduction/). For example, Sass files would go here. -* `/content`: files that will become pages in the site go here. Typically these are Markdown files with some YAML front - matter indicating, [among other things](https://gohugo.io/content-management/front-matter/), what layout should be - applied to this page. The organization of files under `/content` determines the organization of pages in the built +* `/content`: files that will become pages in the site go here. Typically these are Markdown files with some YAML front + matter indicating, [among other things](https://gohugo.io/content-management/front-matter/), what layout should be + applied to this page. The organization of files under `/content` determines the organization of pages in the built site. -* `/data`: this can contain TOML, YAML, or JSON files. Files kept here are directly available to template code as - [data objects](https://gohugo.io/templates/data-templates/), so templates don't need to load them from a file and - parse them. This is also where our +* `/data`: this can contain TOML, YAML, or JSON files. Files kept here are directly available to template code as + [data objects](https://gohugo.io/templates/data-templates/), so templates don't need to load them from a file and + parse them. This is also where our -* `/layouts`: this contains [Hugo templates](https://gohugo.io/templates/). Some templates define the overall layout of +* `/layouts`: this contains [Hugo templates](https://gohugo.io/templates/). Some templates define the overall layout of a page: for example, whether it has header, footer, sidebar, and so on. - * `/layouts/partials`: these templates can be called from other templates, so they can be used to factor out - template code that's used in more than one template. An obvious example here is something like a sidebar, where - several different page layouts might all include the sidebar. But also, partial templates can return values: this + * `/layouts/partials`: these templates can be called from other templates, so they can be used to factor out + template code that's used in more than one template. An obvious example here is something like a sidebar, where + several different page layouts might all include the sidebar. But also, partial templates can return values: this means they can be used like functions, that can be called by multiple templates to do some common processing. * `/layouts/shortcodes`: these templates can be called directly from files in `/content`. * `/static`: static files which don't need preprocessing. JS or CSS files could live here. -* `/themes`: you can use just Hugo or use it with a theme. Themes primarily provide additional templates, which are - supplied in a `/themes/$theme_name/layouts` directory. You can use a theme but customise it by providing your own - versions of any of the them layouts in the base `/layouts` directory. That is, if a theme provides - `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of this +* `/themes`: you can use just Hugo or use it with a theme. Themes primarily provide additional templates, which are + supplied in a `/themes/$theme_name/layouts` directory. You can use a theme but customise it by providing your own + versions of any of the them layouts in the base `/layouts` directory. That is, if a theme provides + `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of this template will be used. It also has the following top-level file: -* `config.toml`: site-wide configuration settings. Some of these are built-in and you can add your own. Config settings +* `config.toml`: site-wide configuration settings. Some of these are built-in and you can add your own. Config settings defined here are available in templates. All these directories above are configurable via `config.toml` settings. Additionally, the following directories may be of interest: @@ -53,8 +53,6 @@ Additionally, the following directories may be of interest: * `/meta`: Documentation relating to the spec's processes that are otherwise untracked (release instructions, etc). * `/scripts`: Various scripts for generating the spec. * `/proposals`: Matrix Spec Change (MSC) proposals. See . -* `/api`: [OpenAPI](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md) / Swagger definitions for - the spec. ## Authoring changes to the spec @@ -69,8 +67,8 @@ place after an MSC has been accepted, not as part of a proposal itself. to be working for you, try `hugo serve --disableFastRender` instead. 6. Edit the specification 🙂 -We use a highly customized [Docsy](https://www.docsy.dev/) theme for our generated site, which uses Bootstrap and Font -Awesome. If you're looking at making design-related changes to the spec site, please coordinate with us in +We use a highly customized [Docsy](https://www.docsy.dev/) theme for our generated site, which uses Bootstrap and Font +Awesome. If you're looking at making design-related changes to the spec site, please coordinate with us in [#matrix-docs:matrix.org](https://matrix.to/#/#matrix-docs:matrix.org) before opening a PR. ## Building the specification @@ -87,7 +85,7 @@ To make use of the generated file, there are a number of options: http://editor.swagger.io/ * You can run a local HTTP server by running `./scripts/swagger-http-server.py`, and then view the documentation via an online viewer; for example, at -* You can host the swagger UI yourself. See for advice on how to +* You can host the swagger UI yourself. See for advice on how to do so. ## Issue tracking From 8d73a17f5bf74a515b784fd2e4112fb2e1f355f1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 5 Apr 2021 12:00:19 -0600 Subject: [PATCH 04/11] Cut out legacy build scripts --- .buildkite/pipeline.yaml | 2 +- .circleci/config.yml | 26 -- scripts/continuserv/README.md | 3 - scripts/continuserv/index.html | 15 - scripts/continuserv/main.go | 274 -------------- scripts/contrib/shell.nix | 6 - scripts/gendoc.py | 561 ----------------------------- scripts/generate-matrix-org-assets | 12 +- scripts/proposals.py | 218 ----------- scripts/templating/build.py | 0 scripts/test-and-build.sh | 38 -- 11 files changed, 3 insertions(+), 1152 deletions(-) delete mode 100644 scripts/continuserv/README.md delete mode 100644 scripts/continuserv/index.html delete mode 100644 scripts/continuserv/main.go delete mode 100644 scripts/contrib/shell.nix delete mode 100755 scripts/gendoc.py delete mode 100755 scripts/proposals.py mode change 100755 => 100644 scripts/templating/build.py delete mode 100755 scripts/test-and-build.sh diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index b18d7bb1..b73542fe 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,5 +1,5 @@ steps: - - label: ":books: Build the legacy spec" + - label: ":snake: Build swagger definitions for matrix.org" command: # Install the python dependencies necessary to build the spec - python3 -m venv env && . env/bin/activate diff --git a/.circleci/config.yml b/.circleci/config.yml index 09c5a6fc..e0815d76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,3 @@ -genlegacydoc: &genlegacydoc - name: Generate the legacy docs - command: | - source /env/bin/activate - scripts/gendoc.py - gendoc: &gendoc name: Generate the docs # Note: Node dependencies are required for the hugo build. @@ -54,12 +48,6 @@ buildspeculator: &buildspeculator cd scripts/speculator go build -v -buildcontinuserv: &buildcontinuserv - name: Build Continuserv - command: | - cd scripts/continuserv - go build -v - version: 2 jobs: validate-docs: @@ -74,17 +62,6 @@ jobs: steps: - checkout - run: *checkexamples - build-legacy-docs: - docker: - - image: uhoreg/matrix-doc-build - steps: - - checkout - - run: *genlegacydoc - - store_artifacts: - path: scripts/gen - - run: - name: "Legacy doc build is available at:" - command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL build-docs: docker: - image: alpine @@ -121,8 +98,6 @@ jobs: name: Install Dependencies command: | go get -v github.com/hashicorp/golang-lru - go get -v gopkg.in/fsnotify/fsnotify.v1 - - run: *buildcontinuserv - run: *buildspeculator workflows: @@ -130,7 +105,6 @@ workflows: build-spec: jobs: - - build-legacy-docs - build-docs - build-swagger - check-docs diff --git a/scripts/continuserv/README.md b/scripts/continuserv/README.md deleted file mode 100644 index 40321bb6..00000000 --- a/scripts/continuserv/README.md +++ /dev/null @@ -1,3 +0,0 @@ -continuserv proactively re-generates the spec on filesystem changes, and serves -it over HTTP. For notes on using it, see [the main -readme](../../README.rst#continuserv). diff --git a/scripts/continuserv/index.html b/scripts/continuserv/index.html deleted file mode 100644 index 24ed7ecb..00000000 --- a/scripts/continuserv/index.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go deleted file mode 100644 index 1bd07e6e..00000000 --- a/scripts/continuserv/main.go +++ /dev/null @@ -1,274 +0,0 @@ -// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. -// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished. -// It does not currently pre-empt stale generations, but will block until they are complete. -package main - -import ( - "bytes" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - fsnotify "gopkg.in/fsnotify/fsnotify.v1" -) - -var ( - port = flag.Int("port", 8000, "Port on which to serve HTTP") - - mu sync.Mutex // Prevent multiple updates in parallel. - toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero. - - wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. - wg sync.WaitGroup // Indicates how many updates are pending. -) - -func main() { - flag.Parse() - - w, err := fsnotify.NewWatcher() - if err != nil { - log.Fatalf("Error making watcher: %v", err) - } - - dir, err := os.Getwd() - if err != nil { - log.Fatalf("Error getting wd: %v", err) - } - for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) { - if dir == "/" { - log.Fatalf("Could not find git root") - } - } - - walker := makeWalker(dir, w) - paths := []string{"api", "changelogs", "event-schemas", "scripts", - "specification", "schemas", "data-definitions"} - - for _, p := range paths { - filepath.Walk(path.Join(dir, p), walker) - } - - wg.Add(1) - populateOnce(dir) - - ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes - go doPopulate(ch, dir) - - go watchFS(ch, w) - fmt.Printf("Listening on port %d\n", *port) - http.HandleFunc("/", serve) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) - -} - -func watchFS(ch chan struct{}, w *fsnotify.Watcher) { - for { - select { - case e := <-w.Events: - if filter(e) { - fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) - ch <- struct{}{} - } - } - } -} - -func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc { - return func(path string, i os.FileInfo, err error) error { - if err != nil { - log.Fatalf("Error walking: %v", err) - } - if !i.IsDir() { - // we set watches on directories, not files - return nil - } - - rel, err := filepath.Rel(base, path) - if err != nil { - log.Fatalf("Failed to get relative path of %s: %v", path, err) - } - - // Normalize slashes - rel = filepath.ToSlash(rel) - - // skip a few things that we know don't form part of the spec - if rel == "api/node_modules" || - rel == "scripts/gen" || - rel == "scripts/tmp" { - return filepath.SkipDir - } - - // log.Printf("Adding watch on %s", path) - if err := w.Add(path); err != nil { - log.Fatalf("Failed to add watch on %s: %v", path, err) - } - return nil - } -} - -// Return true if event should trigger re-population -func filter(e fsnotify.Event) bool { - // vim is *really* noisy about how it writes files - if e.Op != fsnotify.Write { - return false - } - - _, fname := filepath.Split(e.Name) - - // Avoid some temp files that vim or emacs writes - if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(fname, ".") || - (strings.HasPrefix(fname, "#") && strings.HasSuffix(fname, "#")) { - return false - } - - // Forcefully ignore directories we don't care about (Windows, at least, tries to notify about some directories) - filePath := filepath.ToSlash(e.Name) // normalize slashes - if strings.Contains(filePath, "/scripts/tmp") || - strings.Contains(filePath, "/scripts/gen") || - strings.Contains(filePath, "/api/node_modules") { - return false - } - - return true -} - -func serve(w http.ResponseWriter, req *http.Request) { - wgMu.Lock() - wg.Wait() - wgMu.Unlock() - - m := toServe.Load().(bytesOrErr) - if m.err != nil { - w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(m.err.Error())) - return - } - - ok := true - var b []byte - - file := req.URL.Path - if file[0] == '/' { - file = file[1:] - } - b, ok = m.bytes[filepath.FromSlash(file)] // de-normalize slashes - - if ok && file == "api-docs.json" { - w.Header().Set("Access-Control-Allow-Origin", "*") - } - - if ok { - w.Header().Set("Content-Type", "text/html") - w.Write([]byte(b)) - return - } - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(404) - w.Write([]byte("Not found")) -} - -func generate(dir string) (map[string][]byte, error) { - cmd := exec.Command("python", "gendoc.py") - cmd.Dir = path.Join(dir, "scripts") - var b bytes.Buffer - cmd.Stderr = &b - err := cmd.Run() - if err != nil { - return nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) - } - - // cheekily dump the swagger docs into the gen directory so that it is - // easy to serve - cmd = exec.Command("python", "dump-swagger.py", "-o", "gen/api-docs.json") - cmd.Dir = path.Join(dir, "scripts") - cmd.Stderr = &b - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("error generating api docs: %v\nOutput from dump-swagger:\n%v", err, b.String()) - } - - files := make(map[string][]byte) - base := path.Join(dir, "scripts", "gen") - walker := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - rel, err := filepath.Rel(base, path) - if err != nil { - return fmt.Errorf("Failed to get relative path of %s: %v", path, err) - } - - bytes, err := ioutil.ReadFile(path) - if err != nil { - return err - } - files[rel] = bytes - return nil - } - - if err := filepath.Walk(base, walker); err != nil { - return nil, fmt.Errorf("error reading spec: %v", err) - } - - // load the special index - indexpath := path.Join(dir, "scripts", "continuserv", "index.html") - bytes, err := ioutil.ReadFile(indexpath) - if err != nil { - return nil, fmt.Errorf("error reading index: %v", err) - } - files[""] = bytes - - return files, nil -} - -func populateOnce(dir string) { - defer wg.Done() - mu.Lock() - defer mu.Unlock() - - files, err := generate(dir) - toServe.Store(bytesOrErr{files, err}) -} - -func doPopulate(ch chan struct{}, dir string) { - var pending int - for { - select { - case <-ch: - if pending == 0 { - wgMu.Lock() - wg.Add(1) - wgMu.Unlock() - } - pending++ - case <-time.After(10 * time.Millisecond): - if pending > 0 { - pending = 0 - populateOnce(dir) - } - } - } -} - -func exists(path string) bool { - _, err := os.Stat(path) - return !os.IsNotExist(err) -} - -type bytesOrErr struct { - bytes map[string][]byte // filename -> contents - err error -} diff --git a/scripts/contrib/shell.nix b/scripts/contrib/shell.nix deleted file mode 100644 index 7bdcdffa..00000000 --- a/scripts/contrib/shell.nix +++ /dev/null @@ -1,6 +0,0 @@ -with import {}; - -(python.buildEnv.override { - extraLibs = with pythonPackages; - [ docutils pyyaml jinja2 pygments ]; -}).env diff --git a/scripts/gendoc.py b/scripts/gendoc.py deleted file mode 100755 index 7e68ccd7..00000000 --- a/scripts/gendoc.py +++ /dev/null @@ -1,561 +0,0 @@ -#! /usr/bin/env python - -# Copyright 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from argparse import ArgumentParser -from docutils.core import publish_file -import copy -import fileinput -import glob -import os -import os.path -import re -import shutil -import subprocess -import sys -import yaml - -script_dir = os.path.dirname(os.path.abspath(__file__)) -docs_dir = os.path.dirname(script_dir) -spec_dir = os.path.join(docs_dir, "specification") -tmp_dir = os.path.join(script_dir, "tmp") -changelog_dir = os.path.join(docs_dir, "changelogs") - -VERBOSE = False - -""" -Read a RST file and replace titles with a different title level if required. -Args: - filename: The name of the file being read (for debugging) - file_stream: The open file stream to read from. - title_level: The integer which determines the offset to *start* from. - title_styles: An array of characters detailing the right title styles to use - e.g. ["=", "-", "~", "+"] -Returns: - string: The file contents with titles adjusted. -Example: - Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file - when read line-by-line encounters the titles "===", "---", "---", "===", "---". - This function will bump every title encountered down a sub-heading e.g. - "=" to "-" and "-" to "~" because title_level = 1, so the output would be - "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. -""" -def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): - rst_lines = [] - - prev_line_title_level = 0 # We expect the file to start with '=' titles - file_offset = None - prev_non_title_line = None - for i, line in enumerate(file_stream): - if (prev_non_title_line is None - or not is_title_line(prev_non_title_line, line, title_styles) - ): - rst_lines.append(line) - prev_non_title_line = line - continue - - line_title_style = line[0] - line_title_level = title_styles.index(line_title_style) - - # Not all files will start with "===" and we should be flexible enough - # to allow that. The first title we encounter sets the "file offset" - # which is added to the title_level desired. - if file_offset is None: - file_offset = line_title_level - if file_offset != 0: - logv((" WARNING: %s starts with a title style of '%s' but '%s' " + - "is preferable.") % (filename, line_title_style, title_styles[0])) - - # Sanity checks: Make sure that this file is obeying the title levels - # specified and bail if it isn't. - # The file is allowed to go 1 deeper or any number shallower - if prev_line_title_level - line_title_level < -1: - raise Exception( - ("File '%s' line '%s' has a title " + - "style '%s' which doesn't match one of the " + - "allowed title styles of %s because the " + - "title level before this line was '%s'") % - (filename, (i + 1), line_title_style, title_styles, - title_styles[prev_line_title_level]) - ) - prev_line_title_level = line_title_level - - adjusted_level = ( - title_level + line_title_level - file_offset - ) - - # Sanity check: Make sure we can bump down the title and we aren't at the - # lowest level already - if adjusted_level >= len(title_styles): - raise Exception( - ("Files '%s' line '%s' has a sub-title level too low and it " + - "cannot be adjusted to fit. You can add another level to the " + - "'title_styles' key in targets.yaml to fix this.") % - (filename, (i + 1)) - ) - - if adjusted_level == line_title_level: - # no changes required - rst_lines.append(line) - continue - - # Adjusting line levels - logv( - "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % - (filename, line_title_style, title_styles[adjusted_level], - file_offset, title_level) - ) - rst_lines.append(line.replace( - line_title_style, - title_styles[adjusted_level] - )) - - return "".join(rst_lines) - - -def is_title_line(prev_line, line, title_styles): - # The title underline must match at a minimum the length of the title - if len(prev_line) > len(line): - return False - - line = line.rstrip() - - # must be at least 3 chars long - if len(line) < 3: - return False - - # must start with a title char - title_char = line[0] - if title_char not in title_styles: - return False - - # all characters must be the same - for char in line[1:]: - if char != title_char: - return False - - # looks like a title line - return True - - -def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): - # string are file paths to RST blobs - if isinstance(file_info, str): - log("%s %s" % (">" * (1 + title_level), file_info)) - with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f: - rst = None - if adjust_titles: - rst = load_with_adjusted_titles( - file_info, f, title_level, title_styles - ) - else: - rst = f.read() - - rst += "\n\n" - return rst - # dicts look like {0: filepath, 1: filepath} where the key is the title level - elif isinstance(file_info, dict): - levels = sorted(file_info.keys()) - rst = [] - for l in levels: - rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) - return "".join(rst) - # lists are multiple file paths e.g. [filepath, filepath] - elif isinstance(file_info, list): - rst = [] - for f in file_info: - rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) - return "".join(rst) - raise Exception( - "The following 'file' entry in this target isn't a string, list or dict. " + - "It really really should be. Entry: %s" % (file_info,) - ) - - -def build_spec(target, out_filename): - log("Building templated file %s" % out_filename) - with open(out_filename, "w", encoding="utf-8") as outfile: - for file_info in target["files"]: - section = get_rst( - file_info=file_info, - title_level=0, - title_styles=target["title_styles"], - spec_dir=spec_dir, - adjust_titles=True - ) - outfile.write(section) - - -""" -Replaces relative title styles with actual title styles. - -The templating system has no idea what the right title style is when it produces -RST because it depends on the build target. As a result, it uses relative title -styles defined in targets.yaml to say "down a level, up a level, same level". - -This function replaces these relative titles with actual title styles from the -array in targets.yaml. -""" -def fix_relative_titles(target, filename, out_filename): - log("Fix relative titles, %s -> %s" % (filename, out_filename)) - title_styles = target["title_styles"] - relative_title_chars = [ - target["relative_title_styles"]["subtitle"], - target["relative_title_styles"]["sametitle"], - target["relative_title_styles"]["supertitle"] - ] - relative_title_matcher = re.compile( - "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" - ) - title_matcher = re.compile( - "^[" + re.escape("".join(title_styles)) + "]{3,}$" - ) - current_title_style = None - with open(filename, "r", encoding="utf-8") as infile: - with open(out_filename, "w", encoding="utf-8") as outfile: - for line in infile.readlines(): - if not relative_title_matcher.match(line): - if title_matcher.match(line): - current_title_style = line[0] - outfile.write(line) - continue - line_char = line[0] - replacement_char = None - current_title_level = title_styles.index(current_title_style) - if line_char == target["relative_title_styles"]["subtitle"]: - if (current_title_level + 1) == len(title_styles): - raise Exception( - "Encountered sub-title line style but we can't go " + - "any lower." - ) - replacement_char = title_styles[current_title_level + 1] - elif line_char == target["relative_title_styles"]["sametitle"]: - replacement_char = title_styles[current_title_level] - elif line_char == target["relative_title_styles"]["supertitle"]: - if (current_title_level - 1) < 0: - raise Exception( - "Encountered super-title line style but we can't go " + - "any higher." - ) - replacement_char = title_styles[current_title_level - 1] - else: - raise Exception( - "Unknown relative line char %s" % (line_char,) - ) - - outfile.write( - line.replace(line_char, replacement_char) - ) - - - -def rst2html(i, o, stylesheets): - log("rst2html %s -> %s" % (i, o)) - with open(i, "r", encoding="utf-8") as in_file: - with open(o, "w", encoding="utf-8") as out_file: - publish_file( - source=in_file, - destination=out_file, - reader_name="standalone", - parser_name="restructuredtext", - writer_name="html", - settings_overrides={ - "stylesheet_path": stylesheets, - "syntax_highlight": "short", - }, - ) - - -def addAnchors(path): - log("add anchors %s" % path) - - with open(path, "r", encoding="utf-8") as f: - lines = f.readlines() - - replacement = r'

\n\1' - with open(path, "w", encoding="utf-8") as f: - for line in lines: - line = re.sub(r'()', replacement, line.rstrip()) - line = re.sub(r'(
)', replacement, line.rstrip()) - f.write(line + "\n") - - -def run_through_template(input_files, set_verbose, substitutions): - args = [ - 'python', script_dir+'/templating/build.py', - "-o", tmp_dir, - "-i", "matrix_templates", - ] - - for k, v in substitutions.items(): - args.append("--substitution=%s=%s" % (k, v)) - - if set_verbose: - args.insert(2, "-v") - - args.extend(input_files) - - log("EXEC: %s" % " ".join(args)) - log(" ==== build.py output ==== ") - subprocess.check_call(args) - -""" -Extract and resolve groups for the given target in the given targets listing. -Args: - all_targets (dict): The parsed YAML file containing a list of targets - target_name (str): The name of the target to extract from the listings. -Returns: - dict: Containing "filees" (a list of file paths), "relative_title_styles" - (a dict of relative style keyword to title character) and "title_styles" - (a list of characters which represent the global title style to follow, - with the top section title first, the second section second, and so on.) -""" -def get_build_target(all_targets, target_name): - build_target = { - "title_styles": [], - "relative_title_styles": {}, - "files": [] - } - - build_target["title_styles"] = all_targets["title_styles"] - build_target["relative_title_styles"] = all_targets["relative_title_styles"] - target = all_targets["targets"].get(target_name) - if not target: - raise Exception( - "No target by the name '" + target_name + "' exists in '" + - targets_listing + "'." - ) - if not isinstance(target.get("files"), list): - raise Exception( - "Found target but 'files' key is not a list." - ) - - def get_group(group_id, depth): - group_name = group_id[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '%s' but it doesn't exist." % group_name - ) - if not isinstance(group, list): - raise Exception( - "Expected group '%s' to be a list but it isn't." % group_name - ) - # deep copy so changes to depths don't contaminate multiple uses of this group - group = copy.deepcopy(group) - # swap relative depths for absolute ones - for i, entry in enumerate(group): - if isinstance(entry, dict): - group[i] = { - (rel_depth + depth): v for (rel_depth, v) in entry.items() - } - return group - - resolved_files = [] - for file_entry in target["files"]: - # file_entry is a group id - if isinstance(file_entry, str) and file_entry.startswith("group:"): - group = get_group(file_entry, 0) - # The group may be resolved to a list of file entries, in which case - # we want to extend the array to insert each of them rather than - # insert the entire list as a single element (which is what append does) - if isinstance(group, list): - resolved_files.extend(group) - else: - resolved_files.append(group) - # file_entry is a dict which has more file entries as values - elif isinstance(file_entry, dict): - resolved_entry = {} - for (depth, entry) in file_entry.items(): - if not isinstance(entry, str): - raise Exception( - "Double-nested depths are not supported. Entry: %s" % (file_entry,) - ) - if entry.startswith("group:"): - resolved_entry[depth] = get_group(entry, depth) - else: - # map across without editing (e.g. normal file path) - resolved_entry[depth] = entry - resolved_files.append(resolved_entry) - continue - # file_entry is just a plain ol' file path - else: - resolved_files.append(file_entry) - build_target["files"] = resolved_files - return build_target - -def log(line): - print("gendoc: %s" % line) - -def logv(line): - if VERBOSE: - print("gendoc:V: %s" % line) - - -def cleanup_env(): - shutil.rmtree(tmp_dir) - - -def mkdirp(d) : - if not os.path.exists(d): - os.makedirs(d) - - -def main(targets, dest_dir, keep_intermediates, substitutions): - try: - mkdirp(dest_dir) - except Exception as e: - log("Error creating destination directory '%s': %s" % (dest_dir, str(e))) - return 1 - try: - mkdirp(tmp_dir) - except Exception as e: - log("Error creating temporary directory '%s': %s" % (tmp_dir, str(e))) - return 1 - - with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: - target_defs = yaml.load(targ_file.read()) - - if targets == ["all"]: - targets = target_defs["targets"].keys() - - log("Building spec [targets=%s]" % targets) - - templated_files = {} # map from target name to templated file - - for target_name in targets: - templated_file = os.path.join(tmp_dir, "templated_%s.rst" % (target_name,)) - - target = get_build_target(target_defs, target_name) - build_spec(target=target, out_filename=templated_file) - templated_files[target_name] = templated_file - - # we do all the templating at once, because it's slow - run_through_template(templated_files.values(), VERBOSE, substitutions) - - stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - - for target_name, templated_file in templated_files.items(): - target = target_defs["targets"].get(target_name) - version_label = None - if target: - version_label = target.get("version_label") - if version_label: - for old, new in substitutions.items(): - version_label = version_label.replace(old, new) - - rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) - if version_label: - d = os.path.join(dest_dir, target_name.split('@')[0]) - if not os.path.exists(d): - os.mkdir(d) - html_file = os.path.join(d, "%s.html" % version_label) - else: - html_file = os.path.join(dest_dir, "%s.html" % (target_name, )) - - fix_relative_titles( - target=target_defs, filename=templated_file, - out_filename=rst_file, - ) - rst2html(rst_file, html_file, stylesheets=stylesheets) - addAnchors(html_file) - - if not keep_intermediates: - cleanup_env() - - return 0 - - -def list_targets(): - with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: - target_defs = yaml.load(targ_file.read()) - targets = target_defs["targets"].keys() - print("\n".join(targets)) - - -def extract_major(s): - major_version = s - match = re.match("^(r\d+)(\.\d+)*$", s) - if match: - major_version = match.group(1) - return major_version - - -if __name__ == '__main__': - parser = ArgumentParser( - "gendoc.py - Generate the Matrix specification as HTML." - ) - parser.add_argument( - "--nodelete", "-n", action="store_true", - help="Do not delete intermediate files. They will be found in scripts/tmp/" - ) - parser.add_argument( - "--target", "-t", action="append", - help="Specify the build target to build from specification/targets.yaml. " + - "The value 'all' will build all of the targets therein." - ) - parser.add_argument( - "--verbose", "-v", action="store_true", - help="Turn on verbose mode." - ) - parser.add_argument( - "--client_release", "-c", action="store", default="unstable", - help="The client-server release tag to generate, e.g. r1.2" - ) - parser.add_argument( - "--server_release", "-s", action="store", default="unstable", - help="The server-server release tag to generate, e.g. r1.2" - ) - parser.add_argument( - "--appservice_release", "-a", action="store", default="unstable", - help="The appservice release tag to generate, e.g. r1.2" - ) - parser.add_argument( - "--push_gateway_release", "-p", action="store", default="unstable", - help="The push gateway release tag to generate, e.g. r1.2" - ) - parser.add_argument( - "--identity_release", "-i", action="store", default="unstable", - help="The identity service release tag to generate, e.g. r1.2" - ) - parser.add_argument( - "--list_targets", action="store_true", - help="Do not update the specification. Instead print a list of targets.", - ) - parser.add_argument( - "--dest", "-d", default=os.path.join(script_dir, "gen"), - help="Set destination directory (default: scripts/gen)", - ) - - args = parser.parse_args() - VERBOSE = args.verbose - - if args.list_targets: - list_targets() - exit(0) - - substitutions = { - "%CLIENT_RELEASE_LABEL%": args.client_release, - # we hardcode the major versions. This ends up in the example - # API URLs. When we have released a new major version, we'll - # have to bump them. - "%CLIENT_MAJOR_VERSION%": "r0", - "%SERVER_RELEASE_LABEL%": args.server_release, - "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, - "%IDENTITY_RELEASE_LABEL%": args.identity_release, - "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, - } - - exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index 3aff31f5..6b9881d1 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,16 +8,8 @@ cd `dirname $0`/.. mkdir -p assets -# generate specification/proposals.rst -./scripts/proposals.py - -# generate the legacy spec docs -./scripts/gendoc.py -d assets/spec - # and the swagger ./scripts/dump-swagger.py -o assets/spec/client_server/unstable.json -# create a tarball of the assets. Exclude the spec index for now, as -# we want to leave it pointing at the release versions of the specs. -# (XXX: how to maintain this?) -tar -czf assets.tar.gz --exclude="assets/spec/index.html" assets +# create a tarball of the assets. +tar -czf assets.tar.gz assets diff --git a/scripts/proposals.py b/scripts/proposals.py deleted file mode 100755 index faa10a83..00000000 --- a/scripts/proposals.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python -# -# proposals.py: generate an RST file (proposals.rst) from queries to github.com/matrix.org/matrix-doc/issues. - -import requests -import re -from datetime import datetime - -# a list of the labels we care about -LABELS_LIST=[ - 'proposal-in-review', - 'proposed-final-comment-period', - 'final-comment-period', - 'finished-final-comment-period', - 'spec-pr-missing', - 'spec-pr-in-review', - 'merged', - 'proposal-postponed', - 'abandoned', - 'obsolete', -] - - -authors = set() -prs = set() - -def getpage(url): - """Request the given URL, and extract the pagecount from the response headers - - Args: - url (str): URL to fetch - - Returns: - Tuple[int, list]: number of pages, and the list of items on this page - """ - resp = requests.get(url) - - pagecount = 1 - for link in resp.links.values(): - if link['rel'] == 'last': - # we extract the pagecount from the `page` param of the last url - # in the response, eg - # 'https://api.github.com/repositories/24998719/issues?state=all&labels=proposal&page=10' - pagecount = int(re.search('page=(\d+)', link['url']).group(1)) - - val = resp.json() - if not isinstance(val, list): - print(val) # Just dump the raw (likely error) response to the log - raise Exception("Error calling %s" % url) - return (pagecount, val) - -def getbylabel(label): - """Fetch all the issues with a given label - - Args: - label (str): label to fetch - - Returns: - Iterator[dict]: an iterator over the issue list. - """ - urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page=' - page = 1 - while True: - (pagecount, results) = getpage(urlbase + str(page)) - for i in results: - yield i - page += 1 - if page > pagecount: - return - -def print_issue_list(text_file, label, issues): - text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") - - if (len(issues) == 0): - text_file.write("No proposals.\n\n") - return - - text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") - text_file.write(" * - MSC\n") - text_file.write(" - Proposal Title\n") - text_file.write(" - Creation Date\n") - text_file.write(" - Update Date\n") - text_file.write(" - Documentation\n") - text_file.write(" - Author\n") - text_file.write(" - Shepherd\n") - text_file.write(" - PRs\n") - - for item in issues: - # set the created date, find local field, otherwise Github - body = str(item['body']) - created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE) - if created is not None: - created = created.group(1).strip() - try: - created = datetime.strptime(created, "%d/%m/%Y") - created = created.strftime('%Y-%m-%d') - except: - pass - try: - created = datetime.strptime(created, "%Y-%m-%d") - created = created.strftime('%Y-%m-%d') - except: - pass - else : - created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") - created = created.strftime('%Y-%m-%d') - item['created'] = created - - issues_to_print = sorted(issues, key=lambda issue_sort: issue_sort["created"]) - - for item in issues_to_print: - # MSC number - text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") - - # title from Github issue - text_file.write(" - " + item['title'] + "\n") - - # created date - text_file.write(" - " + item['created'] + "\n") - - # last updated, purely Github - updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") - text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") - - # list of document links (urls comma-separated) - maindoc = re.search('^Documentation: (.+?)$', str(item['body']), flags=re.MULTILINE) - if maindoc is not None: - maindoc = maindoc.group(1) - doc_list_formatted = ["`" + str(item['number']) + "-" + str(i) + " <" + x.strip() + ">`_" for i, x in enumerate(maindoc.split(','),1)] - text_file.write(" - " + ', '.join(doc_list_formatted)) - else: - text_file.write(" - ") - text_file.write("\n") - - # author list, if missing just use Github issue creator - author = re.search('^Author: (.+?)$', str(item['body']), flags=re.MULTILINE) - if author is not None: - author_list_formatted = set() - author_list = author.group(1) - for a in author_list.split(","): - authors.add(a.strip()) - author_list_formatted.add("`" + str(a.strip()) + "`_") - text_file.write(" - " + ', '.join(author_list_formatted)) - else: - author = "@" + item['user']['login'] - authors.add(author) - text_file.write(" - `" + str(author) + "`_") - text_file.write("\n") - - # shepherd (currently only one) - shepherd = re.search('Shepherd: (.+?)\n', str(item['body'])) - if shepherd is not None: - authors.add(shepherd.group(1).strip()) - shepherd = "`" + shepherd.group(1).strip() + "`_" - text_file.write(" - " + str(shepherd) + "\n") - - # PRs - try: - pr_list = re.search('PRs: (.+?)$', str(item['body'])) - if pr_list is not None: - pr_list_formatted = set() - pr_list = pr_list.group(1) - for p in pr_list.split(","): - if re.match(r"#\d", p.strip()): - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") - elif re.match(r"https://github.com/matrix-org/matrix-doc/pulls/\d", p.strip()): - pr = "#" + p.strip().replace('https://github.com/matrix-org/matrix-doc/pulls/', '') - prs.add(pr) - pr_list_formatted.add("`PR" + str(pr) + "`_") - else: - raise RuntimeWarning - text_file.write(" - " + ', '.join(pr_list_formatted)) - text_file.write("\n") - else: - text_file.write(" - \n") - except: - print("exception parsing PRs for MSC" + str(item['number'])) - text_file.write(" - \n") - - text_file.write("\n\n\n") - - -# first get all of the issues, filtering by label -issues = {n: [] for n in LABELS_LIST} -# use the magic 'None' key for a proposal in progress -issues[None] = [] - -for prop in getbylabel('proposal'): - print("%s: %s" % (prop['number'], [l['name'] for l in prop['labels']])) - found_label = False - for label in prop['labels']: - label_name = label['name'] - if label_name in issues: - issues[label_name].append(prop) - found_label = True - - # if it doesn't have any other label, assume it's work-in-progress - if not found_label: - issues[None].append(prop) - -text_file = open("specification/proposals.rst", "w") - -text_file.write("Tables of Tracked Proposals\n---------------------------\n\n") - -print_issue_list(text_file, "", issues[None]) -for label in LABELS_LIST: - print_issue_list(text_file, label, issues[label]) - -text_file.write("\n") - -for author in authors: - text_file.write("\n.. _" + author + ": https://github.com/" + author[1:]) - -for pr in prs: - text_file.write("\n.. _PR" + pr + ": https://github.com/matrix-org/matrix-doc/pull/" + pr.replace('#', '')) - -text_file.close() diff --git a/scripts/templating/build.py b/scripts/templating/build.py old mode 100755 new mode 100644 diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh deleted file mode 100755 index 76f49e2e..00000000 --- a/scripts/test-and-build.sh +++ /dev/null @@ -1,38 +0,0 @@ -#! /bin/bash - -set -ex - -cd `dirname $0`/.. - -virtualenv -p python3 env -. env/bin/activate - -# Print out the python versions for debugging purposes -python --version -pip --version - -# Install python dependencies -pip install -r scripts/requirements.txt - -# Install node dependencies -npm install --prefix=scripts - -# do sanity checks on the examples and swagger -scripts/check-event-schema-examples.py -scripts/check-swagger-sources.py -node scripts/validator.js --schema "data/api/client-server" - -: ${GOPATH:=${WORKSPACE}/.gopath} -mkdir -p "${GOPATH}" -export GOPATH -go get github.com/hashicorp/golang-lru -go get gopkg.in/fsnotify/fsnotify.v1 - -# make sure that the scripts build -(cd scripts/continuserv && go build) -(cd scripts/speculator && go build) - -# build the spec for matrix.org. -# (we don't actually use it on travis, but it's still useful to check we -# can build it. On Buildkite, this is then used to deploy to matrix.org). -./scripts/generate-matrix-org-assets From becd2257c0a4fd1328ebba2c7f986020fcf9bc6a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 5 Apr 2021 12:10:01 -0600 Subject: [PATCH 05/11] Remove legacy spec docs --- specification/appendices.rst | 19 - specification/appendices/base64.rst | 57 - .../appendices/identifier_grammar.rst | 408 ---- specification/appendices/signing_json.rst | 327 --- specification/appendices/test_vectors.rst | 182 -- specification/appendices/threat_model.rst | 140 -- specification/appendices/threepids.rst | 48 - specification/application_service_api.rst | 447 ---- specification/client_server_api.rst | 2083 ----------------- specification/feature_profiles.rst | 149 -- specification/identity_service_api.rst | 499 ---- specification/index.rst | 575 ----- specification/modules.rst | 27 - specification/modules/_template.rst | 70 - specification/modules/account_data.rst | 48 - specification/modules/admin.rst | 26 - specification/modules/anonymous_access.rst | 64 - specification/modules/content_repo.rst | 136 -- specification/modules/device_management.rst | 41 - specification/modules/dm.rst | 58 - .../modules/end_to_end_encryption.rst | 1537 ------------ specification/modules/event_context.rst | 33 - specification/modules/guest_access.rst | 103 - specification/modules/history_visibility.rst | 101 - specification/modules/ignore_users.rst | 62 - specification/modules/instant_messaging.rst | 515 ---- specification/modules/mentions.rst | 74 - specification/modules/moderation_policies.rst | 128 - specification/modules/openid.rst | 24 - specification/modules/presence.rst | 88 - specification/modules/push.rst | 768 ------ specification/modules/read_markers.rst | 67 - specification/modules/receipts.rst | 98 - specification/modules/report_content.rst | 35 - specification/modules/room_previews.rst | 58 - specification/modules/room_upgrades.rst | 78 - specification/modules/search.rst | 109 - specification/modules/secrets.rst | 361 --- specification/modules/send_to_device.rst | 150 -- specification/modules/server_acls.rst | 70 - specification/modules/server_notices.rst | 78 - specification/modules/sso_login.rst | 347 --- specification/modules/stickers.rst | 53 - specification/modules/tags.rst | 70 - specification/modules/third_party_invites.rst | 258 -- .../modules/third_party_networks.rst | 20 - .../modules/typing_notifications.rst | 56 - specification/modules/voip_events.rst | 116 - specification/proposals.rst | 6 - specification/proposals_intro.rst | 503 ---- specification/push_gateway.rst | 95 - specification/rooms/v1.rst | 368 --- specification/rooms/v2.rst | 204 -- specification/rooms/v3.rst | 124 - specification/rooms/v4.rst | 76 - specification/rooms/v5.rst | 59 - specification/rooms/v6.rst | 100 - specification/server_server_api.rst | 1269 ---------- specification/targets.yaml | 113 - 59 files changed, 13778 deletions(-) delete mode 100644 specification/appendices.rst delete mode 100644 specification/appendices/base64.rst delete mode 100644 specification/appendices/identifier_grammar.rst delete mode 100644 specification/appendices/signing_json.rst delete mode 100644 specification/appendices/test_vectors.rst delete mode 100644 specification/appendices/threat_model.rst delete mode 100644 specification/appendices/threepids.rst delete mode 100644 specification/application_service_api.rst delete mode 100644 specification/client_server_api.rst delete mode 100644 specification/feature_profiles.rst delete mode 100644 specification/identity_service_api.rst delete mode 100644 specification/index.rst delete mode 100644 specification/modules.rst delete mode 100644 specification/modules/_template.rst delete mode 100644 specification/modules/account_data.rst delete mode 100644 specification/modules/admin.rst delete mode 100644 specification/modules/anonymous_access.rst delete mode 100644 specification/modules/content_repo.rst delete mode 100644 specification/modules/device_management.rst delete mode 100644 specification/modules/dm.rst delete mode 100644 specification/modules/end_to_end_encryption.rst delete mode 100644 specification/modules/event_context.rst delete mode 100644 specification/modules/guest_access.rst delete mode 100644 specification/modules/history_visibility.rst delete mode 100644 specification/modules/ignore_users.rst delete mode 100644 specification/modules/instant_messaging.rst delete mode 100644 specification/modules/mentions.rst delete mode 100644 specification/modules/moderation_policies.rst delete mode 100644 specification/modules/openid.rst delete mode 100644 specification/modules/presence.rst delete mode 100644 specification/modules/push.rst delete mode 100644 specification/modules/read_markers.rst delete mode 100644 specification/modules/receipts.rst delete mode 100644 specification/modules/report_content.rst delete mode 100644 specification/modules/room_previews.rst delete mode 100644 specification/modules/room_upgrades.rst delete mode 100644 specification/modules/search.rst delete mode 100644 specification/modules/secrets.rst delete mode 100644 specification/modules/send_to_device.rst delete mode 100644 specification/modules/server_acls.rst delete mode 100644 specification/modules/server_notices.rst delete mode 100644 specification/modules/sso_login.rst delete mode 100644 specification/modules/stickers.rst delete mode 100644 specification/modules/tags.rst delete mode 100644 specification/modules/third_party_invites.rst delete mode 100644 specification/modules/third_party_networks.rst delete mode 100644 specification/modules/typing_notifications.rst delete mode 100644 specification/modules/voip_events.rst delete mode 100644 specification/proposals.rst delete mode 100644 specification/proposals_intro.rst delete mode 100644 specification/push_gateway.rst delete mode 100644 specification/rooms/v1.rst delete mode 100644 specification/rooms/v2.rst delete mode 100644 specification/rooms/v3.rst delete mode 100644 specification/rooms/v4.rst delete mode 100644 specification/rooms/v5.rst delete mode 100644 specification/rooms/v6.rst delete mode 100644 specification/server_server_api.rst delete mode 100644 specification/targets.yaml diff --git a/specification/appendices.rst b/specification/appendices.rst deleted file mode 100644 index 4a106a3a..00000000 --- a/specification/appendices.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. Copyright 2015 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Appendices -========== - -.. contents:: Table of Contents -.. sectnum:: diff --git a/specification/appendices/base64.rst b/specification/appendices/base64.rst deleted file mode 100644 index a918c2f9..00000000 --- a/specification/appendices/base64.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. Copyright 2017 Vector Creations Limited -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Unpadded Base64 ---------------- - -*Unpadded* Base64 refers to 'standard' Base64 encoding as defined in `RFC -4648`_, without "=" padding. Specifically, where RFC 4648 requires that encoded -data be padded to a multiple of four characters using ``=`` characters, -unpadded Base64 omits this padding. - -For reference, RFC 4648 uses the following alphabet for Base 64:: - - Value Encoding Value Encoding Value Encoding Value Encoding - 0 A 17 R 34 i 51 z - 1 B 18 S 35 j 52 0 - 2 C 19 T 36 k 53 1 - 3 D 20 U 37 l 54 2 - 4 E 21 V 38 m 55 3 - 5 F 22 W 39 n 56 4 - 6 G 23 X 40 o 57 5 - 7 H 24 Y 41 p 58 6 - 8 I 25 Z 42 q 59 7 - 9 J 26 a 43 r 60 8 - 10 K 27 b 44 s 61 9 - 11 L 28 c 45 t 62 + - 12 M 29 d 46 u 63 / - 13 N 30 e 47 v - 14 O 31 f 48 w - 15 P 32 g 49 x - 16 Q 33 h 50 y - -Examples of strings encoded using unpadded Base64:: - - UNPADDED_BASE64("") = "" - UNPADDED_BASE64("f") = "Zg" - UNPADDED_BASE64("fo") = "Zm8" - UNPADDED_BASE64("foo") = "Zm9v" - UNPADDED_BASE64("foob") = "Zm9vYg" - UNPADDED_BASE64("fooba") = "Zm9vYmE" - UNPADDED_BASE64("foobar") = "Zm9vYmFy" - -When decoding Base64, implementations SHOULD accept input with or without -padding characters wherever possible, to ensure maximum interoperability. - -.. _`RFC 4648`: https://tools.ietf.org/html/rfc4648 diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst deleted file mode 100644 index f8b2aef5..00000000 --- a/specification/appendices/identifier_grammar.rst +++ /dev/null @@ -1,408 +0,0 @@ -.. Copyright 2016 Openmarket Ltd. -.. Copyright 2017, 2018 New Vector Ltd. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Identifier Grammar ------------------- - -Some identifiers are specific to given room versions, please refer to the -`room versions specification`_ for more information. - -.. _`room versions specification`: index.html#room-versions - - -Server Name -~~~~~~~~~~~ - -A homeserver is uniquely identified by its server name. This value is used in a -number of identifiers, as described below. - -The server name represents the address at which the homeserver in question can -be reached by other homeservers. All valid server names are included by the -following grammar:: - - server_name = hostname [ ":" port ] - - port = 1*5DIGIT - - hostname = IPv4address / "[" IPv6address "]" / dns-name - - IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT - - IPv6address = 2*45IPv6char - - IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." - ; 0-9, A-F, a-f, :, . - - dns-name = 1*255dns-char - - dns-char = DIGIT / ALPHA / "-" / "." - -— in other words, the server name is the hostname, followed by an optional -numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, -an IPv6 address literal surrounded with square brackets, or a DNS name. - -IPv4 literals must be a sequence of four decimal numbers in the -range 0 to 255, separated by ``.``. IPv6 literals must be as specified by -`RFC3513, section 2.2 `_. - -DNS names for use with Matrix should follow the conventional restrictions for -internet hostnames: they should consist of a series of labels separated by -``.``, where each label consists of the alphanumeric characters or hyphens. - -Examples of valid server names are: - -* ``matrix.org`` -* ``matrix.org:8888`` -* ``1.2.3.4`` (IPv4 literal) -* ``1.2.3.4:1234`` (IPv4 literal with explicit port) -* ``[1234:5678::abcd]`` (IPv6 literal) -* ``[1234:5678::abcd]:5678`` (IPv6 literal with explicit port) - -.. Note:: - - This grammar is based on the standard for internet host names, as specified - by `RFC1123, section 2.1 `_, - with an extension for IPv6 literals. - -Server names must be treated case-sensitively: in other words, -``@user:matrix.org`` is a different person from ``@user:MATRIX.ORG``. - -Some recommendations for a choice of server name follow: - -* The length of the complete server name should not exceed 230 characters. -* Server names should not use upper-case characters. - -Common Identifier Format -~~~~~~~~~~~~~~~~~~~~~~~~ - -The Matrix protocol uses a common format to assign unique identifiers to a -number of entities, including users, events and rooms. Each identifier takes -the form:: - - &string - -where ``&`` represents a 'sigil' character; ``string`` is the string which makes -up the identifier. - -The sigil characters are as follows: - -* ``@``: User ID -* ``!``: Room ID -* ``$``: Event ID -* ``+``: Group ID -* ``#``: Room alias - -User IDs, group IDs, room IDs, room aliases, and sometimes event IDs take the form:: - - &localpart:domain - -where ``domain`` is the `server name`_ of the homeserver which allocated the -identifier, and ``localpart`` is an identifier allocated by that homeserver. - -The precise grammar defining the allowable format of an identifier depends on -the type of identifier. For example, event IDs can sometimes be represented with -a ``domain`` component under some conditions - see the `Event IDs <#room-ids-and-event-ids>`_ -section below for more information. - -User Identifiers -++++++++++++++++ - -Users within Matrix are uniquely identified by their Matrix user ID. The user -ID is namespaced to the homeserver which allocated the account and has the -form:: - - @localpart:domain - -The ``localpart`` of a user ID is an opaque identifier for that user. It MUST -NOT be empty, and MUST contain only the characters ``a-z``, ``0-9``, ``.``, -``_``, ``=``, ``-``, and ``/``. - -The ``domain`` of a user ID is the `server name`_ of the homeserver which -allocated the account. - -The length of a user ID, including the ``@`` sigil and the domain, MUST NOT -exceed 255 characters. - -The complete grammar for a legal user ID is:: - - user_id = "@" user_id_localpart ":" server_name - user_id_localpart = 1*user_id_char - user_id_char = DIGIT - / %x61-7A ; a-z - / "-" / "." / "=" / "_" / "/" - -.. admonition:: Rationale - - A number of factors were considered when defining the allowable characters - for a user ID. - - Firstly, we chose to exclude characters outside the basic US-ASCII character - set. User IDs are primarily intended for use as an identifier at the protocol - level, and their use as a human-readable handle is of secondary - benefit. Furthermore, they are useful as a last-resort differentiator between - users with similar display names. Allowing the full Unicode character set - would make very difficult for a human to distinguish two similar user IDs. The - limited character set used has the advantage that even a user unfamiliar with - the Latin alphabet should be able to distinguish similar user IDs manually, if - somewhat laboriously. - - We chose to disallow upper-case characters because we do not consider it - valid to have two user IDs which differ only in case: indeed it should be - possible to reach ``@user:matrix.org`` as ``@USER:matrix.org``. However, - user IDs are necessarily used in a number of situations which are inherently - case-sensitive (notably in the ``state_key`` of ``m.room.member`` - events). Forbidding upper-case characters (and requiring homeservers to - downcase usernames when creating user IDs for new users) is a relatively simple - way to ensure that ``@USER:matrix.org`` cannot refer to a different user to - ``@user:matrix.org``. - - Finally, we decided to restrict the allowable punctuation to a very basic set - to reduce the possibility of conflicts with special characters in various - situations. For example, "*" is used as a wildcard in some APIs (notably the - filter API), so it cannot be a legal user ID character. - - The length restriction is derived from the limit on the length of the - ``sender`` key on events; since the user ID appears in every event sent by the - user, it is limited to ensure that the user ID does not dominate over the actual - content of the events. - -Matrix user IDs are sometimes informally referred to as MXIDs. - -Historical User IDs -<<<<<<<<<<<<<<<<<<< - -Older versions of this specification were more tolerant of the characters -permitted in user ID localparts. There are currently active users whose user -IDs do not conform to the permitted character set, and a number of rooms whose -history includes events with a ``sender`` which does not conform. In order to -handle these rooms successfully, clients and servers MUST accept user IDs with -localparts from the expanded character set:: - - extended_user_id_char = %x21-39 / %x3B-7E ; all ASCII printing chars except : - -Mapping from other character sets -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -In certain circumstances it will be desirable to map from a wider character set -onto the limited character set allowed in a user ID localpart. Examples include -a homeserver creating a user ID for a new user based on the username passed to -``/register``, or a bridge mapping user ids from another protocol. - -.. TODO-spec - - We need to better define the mechanism by which homeservers can allow users - to have non-Latin login credentials. The general idea is for clients to pass - the non-Latin in the ``username`` field to ``/register`` and ``/login``, and - the HS then maps it onto the MXID space when turning it into the - fully-qualified ``user_id`` which is returned to the client and used in - events. - -Implementations are free to do this mapping however they choose. Since the user -ID is opaque except to the implementation which created it, the only -requirement is that the implementation can perform the mapping -consistently. However, we suggest the following algorithm: - -1. Encode character strings as UTF-8. - -2. Convert the bytes ``A-Z`` to lower-case. - - * In the case where a bridge must be able to distinguish two different users - with ids which differ only by case, escape upper-case characters by - prefixing with ``_`` before downcasing. For example, ``A`` becomes - ``_a``. Escape a real ``_`` with a second ``_``. - -3. Encode any remaining bytes outside the allowed character set, as well as - ``=``, as their hexadecimal value, prefixed with ``=``. For example, ``#`` - becomes ``=23``; ``á`` becomes ``=c3=a1``. - -.. admonition:: Rationale - - The suggested mapping is an attempt to preserve human-readability of simple - ASCII identifiers (unlike, for example, base-32), whilst still allowing - representation of *any* character (unlike punycode, which provides no way to - encode ASCII punctuation). - - -Room IDs and Event IDs -++++++++++++++++++++++ - -A room has exactly one room ID. A room ID has the format:: - - !opaque_id:domain - -An event has exactly one event ID. The format of an event ID depends upon the -`room version specification `_. - -The ``domain`` of a room ID is the `server name`_ of the homeserver which -created the room/event. The domain is used only for namespacing to avoid the -risk of clashes of identifiers between different homeservers. There is no -implication that the room or event in question is still available at the -corresponding homeserver. - -Event IDs and Room IDs are case-sensitive. They are not meant to be human-readable. -They are intended to be treated as fully opaque strings by clients. - -.. TODO-spec - What is the grammar for the opaque part? https://matrix.org/jira/browse/SPEC-389 - - -Group Identifiers -+++++++++++++++++ - -Groups within Matrix are uniquely identified by their group ID. The group -ID is namespaced to the group server which hosts this group and has the -form:: - - +localpart:domain - -The ``localpart`` of a group ID is an opaque identifier for that group. It MUST -NOT be empty, and MUST contain only the characters ``a-z``, ``0-9``, ``.``, -``_``, ``=``, ``-``, and ``/``. - -The ``domain`` of a group ID is the `server name`_ of the group server which -hosts this group. - -The length of a group ID, including the ``+`` sigil and the domain, MUST NOT -exceed 255 characters. - -The complete grammar for a legal group ID is:: - - group_id = "+" group_id_localpart ":" server_name - group_id_localpart = 1*group_id_char - group_id_char = DIGIT - / %x61-7A ; a-z - / "-" / "." / "=" / "_" / "/" - - -Room Aliases -++++++++++++ - -A room may have zero or more aliases. A room alias has the format:: - - #room_alias:domain - -The ``domain`` of a room alias is the `server name`_ of the homeserver which -created the alias. Other servers may contact this homeserver to look up the -alias. - -Room aliases MUST NOT exceed 255 bytes (including the ``#`` sigil and the -domain). - -.. TODO-spec - - Need to specify precise grammar for Room Aliases. https://matrix.org/jira/browse/SPEC-391 - -matrix.to navigation -++++++++++++++++++++ - -.. NOTE:: - This namespacing is in place pending a ``matrix://`` (or similar) URI scheme. - This is **not** meant to be interpreted as an available web service - see - below for more details. - -Rooms, users, aliases, and groups may be represented as a "matrix.to" URI. -This URI can be used to reference particular objects in a given context, such -as mentioning a user in a message or linking someone to a particular point -in the room's history (a permalink). - -A matrix.to URI has the following format, based upon the specification defined -in RFC 3986: - - https://matrix.to/#//? - -The identifier may be a room ID, room alias, user ID, or group ID. The extra -parameter is only used in the case of permalinks where an event ID is referenced. -The matrix.to URI, when referenced, must always start with ``https://matrix.to/#/`` -followed by the identifier. - -The ```` and the preceding question mark are optional and -only apply in certain circumstances, documented below. - -Clients should not rely on matrix.to URIs falling back to a web server if accessed -and instead should perform some sort of action within the client. For example, if -the user were to click on a matrix.to URI for a room alias, the client may open -a view for the user to participate in the room. - -The components of the matrix.to URI (```` and ````) -are to be percent-encoded as per RFC 3986. - -Examples of matrix.to URIs are: - -* Room alias: ``https://matrix.to/#/%23somewhere%3Aexample.org`` -* Room: ``https://matrix.to/#/!somewhere%3Aexample.org`` -* Permalink by room: ``https://matrix.to/#/!somewhere%3Aexample.org/%24event%3Aexample.org`` -* Permalink by room alias: ``https://matrix.to/#/%23somewhere:example.org/%24event%3Aexample.org`` -* User: ``https://matrix.to/#/%40alice%3Aexample.org`` -* Group: ``https://matrix.to/#/%2Bexample%3Aexample.org`` - -.. Note:: - Historically, clients have not produced URIs which are fully encoded. Clients should - try to interpret these cases to the best of their ability. For example, an unencoded - room alias should still work within the client if possible. - -.. Note:: - Clients should be aware that decoding a matrix.to URI may result in extra slashes - appearing due to some `room versions `_. These slashes - should normally be encoded when producing matrix.to URIs, however. - -Routing -<<<<<<< - -Room IDs are not routable on their own as there is no reliable domain to send requests -to. This is partially mitigated with the addition of a ``via`` argument on a matrix.to -URI, however the problem of routability is still present. Clients should do their best -to route Room IDs to where they need to go, however they should also be aware of -`issue #1579 `_. - -A room (or room permalink) which isn't using a room alias should supply at least one -server using ``via`` in the ````, like so: -``https://matrix.to/!somewhere%3Aexample.org?via=example.org&via=alt.example.org``. The -parameter can be supplied multiple times to specify multiple servers to try. - -The values of ``via`` are intended to be passed along as the ``server_name`` parameters -on the Client Server ``/join`` API. - -When generating room links and permalinks, the application should pick servers which -have a high probability of being in the room in the distant future. How these servers -are picked is left as an implementation detail, however the current recommendation is -to pick 3 unique servers based on the following criteria: - -* The first server should be the server of the highest power level user in the room, - provided they are at least power level 50. If no user meets this criterion, pick the - most popular server in the room (most joined users). The rationale for not picking - users with power levels under 50 is that they are unlikely to be around into the - distant future while higher ranking users (and therefore servers) are less likely - to give up their power and move somewhere else. Most rooms in the public federation - have a power level 100 user and have not deviated from the default structure where - power level 50 users have moderator-style privileges. - -* The second server should be the next highest server by population, or the first - highest by population if the first server was based on a user's power level. The - rationale for picking popular servers is that the server is unlikely to be removed - as the room naturally grows in membership due to that server joining users. The - server could be refused participation in the future due to server ACLs or similar, - however the chance of that happening to a server which is organically joining the - room is unlikely. - -* The third server should be the next highest server by population. - -* Servers which are blocked due to server ACLs should never be chosen. - -* Servers which are IP addresses should never be chosen. Servers which use a domain - name are less likely to be unroutable in the future whereas IP addresses cannot be - pointed to a different location and therefore higher risk options. - -* All 3 servers should be unique from each other. If the room does not have enough users - to supply 3 servers, the application should only specify the servers it can. For example, - a room with only 2 users in it would result in maximum 2 ``via`` parameters. diff --git a/specification/appendices/signing_json.rst b/specification/appendices/signing_json.rst deleted file mode 100644 index 1bc99180..00000000 --- a/specification/appendices/signing_json.rst +++ /dev/null @@ -1,327 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Signing JSON ------------- - -Various points in the Matrix specification require JSON objects to be -cryptographically signed. This requires us to encode the JSON as a binary -string. Unfortunately the same JSON can be encoded in different ways by -changing how much white space is used or by changing the order of keys within -objects. - -Signing an object therefore requires it to be encoded as a sequence of bytes -using `Canonical JSON`_, computing the signature for that sequence and then -adding the signature to the original JSON object. - -Canonical JSON -~~~~~~~~~~~~~~ - -We define the canonical JSON encoding for a value to be the shortest UTF-8 JSON -encoding with dictionary keys lexicographically sorted by Unicode codepoint. -Numbers in the JSON must be integers in the range ``[-(2**53)+1, (2**53)-1]``. - -We pick UTF-8 as the encoding as it should be available to all platforms and -JSON received from the network is likely to be already encoded using UTF-8. -We sort the keys to give a consistent ordering. We force integers to be in the -range where they can be accurately represented using IEEE double precision -floating point numbers since a number of JSON libraries represent all numbers -using this representation. - -.. WARNING:: - Events in room versions 1, 2, 3, 4, and 5 might not be fully compliant with - these restrictions. Servers SHOULD be capable of handling JSON which is considered - invalid by these restrictions where possible. - - The most notable consideration is that integers might not be in the range - specified above. - -.. Note:: - Float values are not permitted by this encoding. - -.. code:: python - - import json - - def canonical_json(value): - return json.dumps( - value, - # Encode code-points outside of ASCII as UTF-8 rather than \u escapes - ensure_ascii=False, - # Remove unnecessary white space. - separators=(',',':'), - # Sort the keys of dictionaries. - sort_keys=True, - # Encode the resulting Unicode as UTF-8 bytes. - ).encode("UTF-8") - -Grammar -+++++++ - -Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing -insignificant whitespace, fractions, exponents and redundant character escapes. - -.. code:: - - value = false / null / true / object / array / number / string - false = %x66.61.6c.73.65 - null = %x6e.75.6c.6c - true = %x74.72.75.65 - object = %x7B [ member *( %x2C member ) ] %7D - member = string %x3A value - array = %x5B [ value *( %x2C value ) ] %5B - number = [ %x2D ] int - int = %x30 / ( %x31-39 *digit ) - digit = %x30-39 - string = %x22 *char %x22 - char = unescaped / %x5C escaped - unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - escaped = %x22 ; " quotation mark U+0022 - / %x5C ; \ reverse solidus U+005C - / %x62 ; b backspace U+0008 - / %x66 ; f form feed U+000C - / %x6E ; n line feed U+000A - / %x72 ; r carriage return U+000D - / %x74 ; t tab U+0009 - / %x75.30.30.30 (%x30-37 / %x62 / %x65-66) ; u000X - / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X - -Examples -++++++++ - -To assist in the development of compatible implementations, the following test -values may be useful for verifying the canonical transformation code. - -Given the following JSON object: - -.. code:: json - - {} - -The following canonical JSON should be produced: - -.. code:: json - - {} - -Given the following JSON object: - -.. code:: json - - { - "one": 1, - "two": "Two" - } - -The following canonical JSON should be produced: - -.. code:: json - - {"one":1,"two":"Two"} - -Given the following JSON object: - -.. code:: json - - { - "b": "2", - "a": "1" - } - -The following canonical JSON should be produced: - -.. code:: json - - {"a":"1","b":"2"} - -Given the following JSON object: - -.. code:: json - - {"b":"2","a":"1"} - -The following canonical JSON should be produced: - -.. code:: json - - {"a":"1","b":"2"} - -Given the following JSON object: - -.. code:: json - - { - "auth": { - "success": true, - "mxid": "@john.doe:example.com", - "profile": { - "display_name": "John Doe", - "three_pids": [ - { - "medium": "email", - "address": "john.doe@example.org" - }, - { - "medium": "msisdn", - "address": "123456789" - } - ] - } - } - } - - -The following canonical JSON should be produced: - -.. code:: json - - {"auth":{"mxid":"@john.doe:example.com","profile":{"display_name":"John Doe","three_pids":[{"address":"john.doe@example.org","medium":"email"},{"address":"123456789","medium":"msisdn"}]},"success":true}} - - -Given the following JSON object: - -.. code:: json - - { - "a": "日本語" - } - -The following canonical JSON should be produced: - -.. code:: json - - {"a":"日本語"} - -Given the following JSON object: - -.. code:: json - - { - "本": 2, - "日": 1 - } - -The following canonical JSON should be produced: - -.. code:: json - - {"日":1,"本":2} - -Given the following JSON object: - -.. code:: json - - { - "a": "\u65E5" - } - -The following canonical JSON should be produced: - -.. code:: json - - {"a":"日"} - -Given the following JSON object: - -.. code:: json - - { - "a": null - } - -The following canonical JSON should be produced: - -.. code:: json - - {"a":null} - -Signing Details -~~~~~~~~~~~~~~~ - -JSON is signed by encoding the JSON object without ``signatures`` or keys grouped -as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the -signature algorithm and the signature is encoded using `unpadded Base64`_. -The resulting base64 signature is added to an object under the -*signing key identifier* which is added to the ``signatures`` object under the -name of the entity signing it which is added back to the original JSON object -along with the ``unsigned`` object. - -The *signing key identifier* is the concatenation of the *signing algorithm* -and a *key identifier*. The *signing algorithm* identifies the algorithm used -to sign the JSON. The currently supported value for *signing algorithm* is -``ed25519`` as implemented by NACL (http://nacl.cr.yp.to/). The *key identifier* -is used to distinguish between different signing keys used by the same entity. - -The ``unsigned`` object and the ``signatures`` object are not covered by the -signature. Therefore intermediate entities can add unsigned data such as -timestamps and additional signatures. - - -.. code:: json - - { - "name": "example.org", - "signing_keys": { - "ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ" - }, - "unsigned": { - "age_ts": 922834800000 - }, - "signatures": { - "example.org": { - "ed25519:1": "s76RUgajp8w172am0zQb/iPTHsRnb4SkrzGoeCOSFfcBY2V/1c8QfrmdXHpvnc2jK5BD1WiJIxiMW95fMjK7Bw" - } - } - } - -.. code:: python - - def sign_json(json_object, signing_key, signing_name): - signatures = json_object.pop("signatures", {}) - unsigned = json_object.pop("unsigned", None) - - signed = signing_key.sign(encode_canonical_json(json_object)) - signature_base64 = encode_base64(signed.signature) - - key_id = "%s:%s" % (signing_key.alg, signing_key.version) - signatures.setdefault(signing_name, {})[key_id] = signature_base64 - - json_object["signatures"] = signatures - if unsigned is not None: - json_object["unsigned"] = unsigned - - return json_object - -Checking for a Signature -~~~~~~~~~~~~~~~~~~~~~~~~ - -To check if an entity has signed a JSON object an implementation does the -following: - -1. Checks if the ``signatures`` member of the object contains an entry with - the name of the entity. If the entry is missing then the check fails. -2. Removes any *signing key identifiers* from the entry with algorithms it - doesn't understand. If there are no *signing key identifiers* left then the - check fails. -3. Looks up *verification keys* for the remaining *signing key identifiers* - either from a local cache or by consulting a trusted key server. If it - cannot find a *verification key* then the check fails. -4. Decodes the base64 encoded signature bytes. If base64 decoding fails then - the check fails. -5. Removes the ``signatures`` and ``unsigned`` members of the object. -6. Encodes the remainder of the JSON object using the `Canonical JSON`_ - encoding. -7. Checks the signature bytes against the encoded object using the - *verification key*. If this fails then the check fails. Otherwise the check - succeeds. diff --git a/specification/appendices/test_vectors.rst b/specification/appendices/test_vectors.rst deleted file mode 100644 index 05b115db..00000000 --- a/specification/appendices/test_vectors.rst +++ /dev/null @@ -1,182 +0,0 @@ -.. Copyright 2015 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - - -Cryptographic Test Vectors --------------------------- - -To assist in the development of compatible implementations, the following test -values may be useful for verifying the cryptographic event signing code. - -Signing Key -~~~~~~~~~~~ - -The following test vectors all use the 32-byte value given by the following -Base64-encoded string as the seed for generating the ``ed25519`` signing key: - -.. code:: - - SIGNING_KEY_SEED = decode_base64( - "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" - ) - -In each case, the server name and key ID are as follows: - -.. code:: - - SERVER_NAME = "domain" - - KEY_ID = "ed25519:1" - -JSON Signing -~~~~~~~~~~~~ - -Given an empty JSON object: - -.. code:: json - - {} - -The JSON signing algorithm should emit the following signed data: - -.. code:: json - - { - "signatures": { - "domain": { - "ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ" - } - } - } - -Given the following JSON object with data values in it: - -.. code:: json - - { - "one": 1, - "two": "Two" - } - -The JSON signing algorithm should emit the following signed JSON: - -.. code:: json - - { - "one": 1, - "signatures": { - "domain": { - "ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw" - } - }, - "two": "Two" - } - -Event Signing -~~~~~~~~~~~~~ - -Given the following minimally-sized event: - -.. code:: json - - { - "room_id": "!x:domain", - "sender": "@a:domain", - "origin": "domain", - "origin_server_ts": 1000000, - "signatures": {}, - "hashes": {}, - "type": "X", - "content": {}, - "prev_events": [], - "auth_events": [], - "depth": 3, - "unsigned": { - "age_ts": 1000000 - } - } - -The event signing algorithm should emit the following signed event: - -.. code:: json - - { - "auth_events": [], - "content": {}, - "depth": 3, - "hashes": { - "sha256": "5jM4wQpv6lnBo7CLIghJuHdW+s2CMBJPUOGOC89ncos" - }, - "origin": "domain", - "origin_server_ts": 1000000, - "prev_events": [], - "room_id": "!x:domain", - "sender": "@a:domain", - "signatures": { - "domain": { - "ed25519:1": "KxwGjPSDEtvnFgU00fwFz+l6d2pJM6XBIaMEn81SXPTRl16AqLAYqfIReFGZlHi5KLjAWbOoMszkwsQma+lYAg" - } - }, - "type": "X", - "unsigned": { - "age_ts": 1000000 - } - } - -Given the following event containing redactable content: - -.. code:: json - - { - "content": { - "body": "Here is the message content" - }, - "event_id": "$0:domain", - "origin": "domain", - "origin_server_ts": 1000000, - "type": "m.room.message", - "room_id": "!r:domain", - "sender": "@u:domain", - "signatures": {}, - "unsigned": { - "age_ts": 1000000 - } - } - -The event signing algorithm should emit the following signed event: - -.. code:: json - - { - "content": { - "body": "Here is the message content" - }, - "event_id": "$0:domain", - "hashes": { - "sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" - }, - "origin": "domain", - "origin_server_ts": 1000000, - "type": "m.room.message", - "room_id": "!r:domain", - "sender": "@u:domain", - "signatures": { - "domain": { - "ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" - } - }, - "unsigned": { - "age_ts": 1000000 - } - } diff --git a/specification/appendices/threat_model.rst b/specification/appendices/threat_model.rst deleted file mode 100644 index 354e9240..00000000 --- a/specification/appendices/threat_model.rst +++ /dev/null @@ -1,140 +0,0 @@ -.. Copyright 2015 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Security Threat Model ----------------------- - -Denial of Service -~~~~~~~~~~~~~~~~~ - -The attacker could attempt to prevent delivery of messages to or from the -victim in order to: - -* Disrupt service or marketing campaign of a commercial competitor. -* Censor a discussion or censor a participant in a discussion. -* Perform general vandalism. - -Threat: Resource Exhaustion -+++++++++++++++++++++++++++ - -An attacker could cause the victim's server to exhaust a particular resource -(e.g. open TCP connections, CPU, memory, disk storage) - -Threat: Unrecoverable Consistency Violations -++++++++++++++++++++++++++++++++++++++++++++ - -An attacker could send messages which created an unrecoverable "split-brain" -state in the cluster such that the victim's servers could no longer derive a -consistent view of the chatroom state. - -Threat: Bad History -+++++++++++++++++++ - -An attacker could convince the victim to accept invalid messages which the -victim would then include in their view of the chatroom history. Other servers -in the chatroom would reject the invalid messages and potentially reject the -victims messages as well since they depended on the invalid messages. - -.. TODO-spec - Track trustworthiness of HS or users based on if they try to pretend they - haven't seen recent events, and fake a splitbrain... --M - -Threat: Block Network Traffic -+++++++++++++++++++++++++++++ - -An attacker could try to firewall traffic between the victim's server and some -or all of the other servers in the chatroom. - -Threat: High Volume of Messages -+++++++++++++++++++++++++++++++ - -An attacker could send large volumes of messages to a chatroom with the victim -making the chatroom unusable. - -Threat: Banning users without necessary authorisation -+++++++++++++++++++++++++++++++++++++++++++++++++++++ - -An attacker could attempt to ban a user from a chatroom without the necessary -authorisation. - -Spoofing -~~~~~~~~ - -An attacker could try to send a message claiming to be from the victim without -the victim having sent the message in order to: - -* Impersonate the victim while performing illicit activity. -* Obtain privileges of the victim. - -Threat: Altering Message Contents -+++++++++++++++++++++++++++++++++ - -An attacker could try to alter the contents of an existing message from the -victim. - -Threat: Fake Message "origin" Field -+++++++++++++++++++++++++++++++++++ - -An attacker could try to send a new message purporting to be from the victim -with a phony "origin" field. - -Spamming -~~~~~~~~ - -The attacker could try to send a high volume of solicited or unsolicited -messages to the victim in order to: - -* Find victims for scams. -* Market unwanted products. - -Threat: Unsolicited Messages -++++++++++++++++++++++++++++ - -An attacker could try to send messages to victims who do not wish to receive -them. - -Threat: Abusive Messages -++++++++++++++++++++++++ - -An attacker could send abusive or threatening messages to the victim - -Spying -~~~~~~ - -The attacker could try to access message contents or metadata for messages sent -by the victim or to the victim that were not intended to reach the attacker in -order to: - -* Gain sensitive personal or commercial information. -* Impersonate the victim using credentials contained in the messages. - (e.g. password reset messages) -* Discover who the victim was talking to and when. - -Threat: Disclosure during Transmission -++++++++++++++++++++++++++++++++++++++ - -An attacker could try to expose the message contents or metadata during -transmission between the servers. - -Threat: Disclosure to Servers Outside Chatroom -++++++++++++++++++++++++++++++++++++++++++++++ - -An attacker could try to convince servers within a chatroom to send messages to -a server it controls that was not authorised to be within the chatroom. - -Threat: Disclosure to Servers Within Chatroom -+++++++++++++++++++++++++++++++++++++++++++++ - -An attacker could take control of a server within a chatroom to expose message -contents or metadata for messages in that room. diff --git a/specification/appendices/threepids.rst b/specification/appendices/threepids.rst deleted file mode 100644 index 84860740..00000000 --- a/specification/appendices/threepids.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. Copyright 2017 Kamax.io -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -3PID Types ----------- -Third Party Identifiers (3PIDs) represent identifiers on other namespaces that -might be associated with a particular person. They comprise a tuple of ``medium`` -which is a string that identifies the namespace in which the identifier exists, -and an ``address``: a string representing the identifier in that namespace. This -must be a canonical form of the identifier, *i.e.* if multiple strings could -represent the same identifier, only one of these strings must be used in a 3PID -address, in a well-defined manner. - -For example, for e-mail, the ``medium`` is 'email' and the ``address`` would be the -email address, *e.g.* the string ``bob@example.com``. Since domain resolution is -case-insensitive, the email address ``bob@Example.com`` is also has the 3PID address -of ``bob@example.com`` (without the capital 'e') rather than ``bob@Example.com``. - -The namespaces defined by this specification are listed below. More namespaces -may be defined in future versions of this specification. - -E-Mail -~~~~~~ -Medium: ``email`` - -Represents E-Mail addresses. The ``address`` is the raw email address in -``user@domain`` form with the domain in lowercase. It must not contain other text -such as real name, angle brackets or a mailto: prefix. - -PSTN Phone numbers -~~~~~~~~~~~~~~~~~~ -Medium: ``msisdn`` - -Represents telephone numbers on the public switched telephone network. The -``address`` is the telephone number represented as a MSISDN (Mobile Station -International Subscriber Directory Number) as defined by the E.164 numbering -plan. Note that MSISDNs do not include a leading '+'. diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst deleted file mode 100644 index 4f386171..00000000 --- a/specification/application_service_api.rst +++ /dev/null @@ -1,447 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Application Service API -======================= - -{{unstable_warning_block_APPSERVICE_RELEASE_LABEL}} - -The Matrix client-server API and server-server APIs provide the means to -implement a consistent self-contained federated messaging fabric. However, they -provide limited means of implementing custom server-side behaviour in Matrix -(e.g. gateways, filters, extensible hooks etc). The Application Service API (AS API) -defines a standard API to allow such extensible functionality to be implemented -irrespective of the underlying homeserver implementation. - -.. TODO-spec - Add in Client-Server services? Overview of bots? Seems weird to be in the spec - given it is VERY implementation specific. - -.. contents:: Table of Contents -.. sectnum:: - -Changelog ---------- - - -.. topic:: Version: %APPSERVICE_RELEASE_LABEL% -{{application_service_changelog}} - -This version of the specification is generated from -`matrix-doc `_ as of Git commit -`{{git_version}} `_. - -For the full historical changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst - -Other versions of this specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following other versions are also available, in reverse chronological order: - -- `HEAD `_: Includes all changes since the latest versioned release. -- `r0.1.1 `_ -- `r0.1.0 `_ - - -Application Services --------------------- -Application services are passive and can only observe events from homeserver. -They can inject events into rooms they are participating in. -They cannot prevent events from being sent, nor can they modify the content of -the event being sent. In order to observe events from a homeserver, the -homeserver needs to be configured to pass certain types of traffic to the -application service. This is achieved by manually configuring the homeserver -with information about the application service. - -Registration -~~~~~~~~~~~~ - -.. NOTE:: - Previously, application services could register with a homeserver via HTTP - APIs. This was removed as it was seen as a security risk. A compromised - application service could re-register for a global ``*`` regex and sniff - *all* traffic on the homeserver. To protect against this, application - services now have to register via configuration files which are linked to - the homeserver configuration file. The addition of configuration files - allows homeserver admins to sanity check the registration for suspicious - regex strings. - -.. TODO - Removing the API entirely is probably a mistake - having a standard cross-HS - way of doing this stops ASes being coupled to particular HS implementations. - A better solution would be to somehow mandate that the API done to avoid - abuse. - -Application services register "namespaces" of user IDs, room aliases and room IDs. -These namespaces are represented as regular expressions. An application service -is said to be "interested" in a given event if one of the IDs in the event match -the regular expression provided by the application service, such as the room having -an alias or ID in the relevant namespaces. Similarly, the application service is -said to be interested in a given event if one of the application service's namespaced -users is the target of the event, or is a joined member of the room where the event -occurred. - -An application service can also state whether they should be the only ones who -can manage a specified namespace. This is referred to as an "exclusive" -namespace. An exclusive namespace prevents humans and other application -services from creating/deleting entities in that namespace. Typically, -exclusive namespaces are used when the rooms represent real rooms on -another service (e.g. IRC). Non-exclusive namespaces are used when the -application service is merely augmenting the room itself (e.g. providing -logging or searching facilities). Namespaces are represented by POSIX extended -regular expressions and look like: - -.. code-block:: yaml - - users: - - exclusive: true - regex: "@_irc_bridge_.*" - -Application services may define the following namespaces (with none being explicitly required): - -+------------------+-----------------------------------------------------------+ -| Name | Description | -+==================+===========================================================+ -| users | Events which are sent from certain users. | -+------------------+-----------------------------------------------------------+ -| aliases | Events which are sent in rooms with certain room aliases. | -+------------------+-----------------------------------------------------------+ -| rooms | Events which are sent in rooms with certain room IDs. | -+------------------+-----------------------------------------------------------+ - -Each individual namespace MUST declare the following fields: - -+------------------+-----------------------------------------------------------------------------------------------------------------------------------+ -| Name | Description | -+==================+===================================================================================================================================+ -| exclusive | **Required** A true or false value stating whether this application service has exclusive access to events within this namespace. | -+------------------+-----------------------------------------------------------------------------------------------------------------------------------+ -| regex | **Required** A regular expression defining which values this namespace includes. | -+------------------+-----------------------------------------------------------------------------------------------------------------------------------+ - -Exclusive user and alias namespaces should begin with an underscore after the -sigil to avoid collisions with other users on the homeserver. Application -services should additionally attempt to identify the service they represent -in the reserved namespace. For example, ``@_irc_.*`` would be a good namespace -to register for an application service which deals with IRC. - -The registration is represented by a series of key-value pairs, which this -specification will present as YAML. See below for the possible options along -with their explanation: - -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Name | Description | -+==================+====================================================================================================================================================+ -| id | **Required.** A unique, user-defined ID of the application service which will never change. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| url | **Required.** The URL for the application service. May include a path after the domain name. Optionally set to ``null`` if no traffic is required. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| as_token | **Required.** A unique token for application services to use to authenticate requests to Homeservers. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| hs_token | **Required.** A unique token for Homeservers to use to authenticate requests to application services. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| sender_localpart | **Required.** The localpart of the user associated with the application service. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| namespaces | **Required.** A list of ``users``, ``aliases`` and ``rooms`` namespaces that the application service controls. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| rate_limited | Whether requests from masqueraded users are rate-limited. The sender is excluded. | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ -| protocols | The external protocols which the application service provides (e.g. IRC). | -+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ - -An example registration file for an IRC-bridging application service is below: - -.. code-block:: yaml - - id: "IRC Bridge" - url: "http://127.0.0.1:1234" - as_token: "30c05ae90a248a4188e620216fa72e349803310ec83e2a77b34fe90be6081f46" - hs_token: "312df522183efd404ec1cd22d2ffa4bbc76a8c1ccf541dd692eef281356bb74e" - sender_localpart: "_irc_bot" # Will result in @_irc_bot:example.org - namespaces: - users: - - exclusive: true - regex: "@_irc_bridge_.*" - aliases: - - exclusive: false - regex: "#_irc_bridge_.*" - rooms: [] - -.. WARNING:: - If the homeserver in question has multiple application services, each - ``as_token`` and ``id`` MUST be unique per application service as these are - used to identify the application service. The homeserver MUST enforce this. - -Homeserver -> Application Service API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Authorization -+++++++++++++ - -Homeservers MUST include a query parameter named ``access_token`` containing the -``hs_token`` from the application service's registration when making requests to -the application service. Application services MUST verify the provided ``access_token`` -matches their known ``hs_token``, failing the request with an ``M_FORBIDDEN`` error -if it does not match. - -Legacy routes -+++++++++++++ - -Previous drafts of the application service specification had a mix of endpoints -that have been used in the wild for a significant amount of time. The application -service specification now defines a version on all endpoints to be more compatible -with the rest of the Matrix specification and the future. - -Homeservers should attempt to use the specified endpoints first when communicating -with application services. However, if the application service receives an HTTP status -code that does not indicate success (i.e.: 404, 500, 501, etc) then the homeserver -should fall back to the older endpoints for the application service. - -The older endpoints have the exact same request body and response format, they -just belong at a different path. The equivalent path for each is as follows: - -* ``/_matrix/app/v1/transactions/{txnId}`` should fall back to ``/transactions/{txnId}`` -* ``/_matrix/app/v1/users/{userId}`` should fall back to ``/users/{userId}`` -* ``/_matrix/app/v1/rooms/{roomAlias}`` should fall back to ``/rooms/{roomAlias}`` -* ``/_matrix/app/v1/thirdparty/protocol/{protocol}`` should fall back to ``/_matrix/app/unstable/thirdparty/protocol/{protocol}`` -* ``/_matrix/app/v1/thirdparty/user/{user}`` should fall back to ``/_matrix/app/unstable/thirdparty/user/{user}`` -* ``/_matrix/app/v1/thirdparty/location/{location}`` should fall back to ``/_matrix/app/unstable/thirdparty/location/{location}`` -* ``/_matrix/app/v1/thirdparty/user`` should fall back to ``/_matrix/app/unstable/thirdparty/user`` -* ``/_matrix/app/v1/thirdparty/location`` should fall back to ``/_matrix/app/unstable/thirdparty/location`` - -Homeservers should periodically try again for the newer endpoints because the -application service may have been updated. - -Pushing events -++++++++++++++ - -The application service API provides a transaction API for sending a list of -events. Each list of events includes a transaction ID, which works as follows: - -:: - - Typical - HS ---> AS : Homeserver sends events with transaction ID T. - <--- : Application Service sends back 200 OK. - - AS ACK Lost - HS ---> AS : Homeserver sends events with transaction ID T. - <-/- : AS 200 OK is lost. - HS ---> AS : Homeserver retries with the same transaction ID of T. - <--- : Application Service sends back 200 OK. If the AS had processed these - events already, it can NO-OP this request (and it knows if it is the - same events based on the transaction ID). - -The events sent to the application service should be linearised, as if they were -from the event stream. The homeserver MUST maintain a queue of transactions to -send to the application service. If the application service cannot be reached, the -homeserver SHOULD backoff exponentially until the application service is reachable again. -As application services cannot *modify* the events in any way, these requests can -be made without blocking other aspects of the homeserver. Homeservers MUST NOT -alter (e.g. add more) events they were going to send within that transaction ID -on retries, as the application service may have already processed the events. - -{{transactions_as_http_api}} - -Querying -++++++++ - -The application service API includes two querying APIs: for room aliases and for -user IDs. The application service SHOULD create the queried entity if it desires. -During this process, the application service is blocking the homeserver until the -entity is created and configured. If the homeserver does not receive a response -to this request, the homeserver should retry several times before timing out. This -should result in an HTTP status 408 "Request Timeout" on the client which initiated -this request (e.g. to join a room alias). - -.. admonition:: Rationale - - Blocking the homeserver and expecting the application service to create the entity - using the client-server API is simpler and more flexible than alternative methods - such as returning an initial sync style JSON blob and get the HS to provision - the room/user. This also meant that there didn't need to be a "backchannel" to inform - the application service about information about the entity such as room ID to - room alias mappings. - -{{query_user_as_http_api}} - -{{query_room_as_http_api}} - - -Third party networks -++++++++++++++++++++ - -Application services may declare which protocols they support via their registration -configuration for the homeserver. These networks are generally for third party services -such as IRC that the application service is managing. Application services may populate -a Matrix room directory for their registered protocols, as defined in the Client-Server -API Extensions. - -Each protocol may have several "locations" (also known as "third party locations" or "3PLs"). -A location within a protocol is a place in the third party network, such as an IRC channel. -Users of the third party network may also be represented by the application service. - -Locations and users can be searched by fields defined by the application service, such -as by display name or other attribute. When clients request the homeserver to search -in a particular "network" (protocol), the search fields will be passed along to the -application service for filtering. - -{{protocols_as_http_api}} - - -.. _create the user: `sect:asapi-permissions`_ - -Client-Server API Extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Application services can use a more powerful version of the -client-server API by identifying itself as an application service to the -homeserver. - -Endpoints defined in this section MUST be supported by homeservers in the -client-server API as accessible only by application services. - -Identity assertion -++++++++++++++++++ -The client-server API infers the user ID from the ``access_token`` provided in -every request. To avoid the application service from having to keep track of each -user's access token, the application service should identify itself to the Client-Server -API by providing its ``as_token`` for the ``access_token`` alongside the user the -application service would like to masquerade as. - -Inputs: - - Application service token (``as_token``) - - User ID in the AS namespace to act as. - -Notes: - - This applies to all aspects of the Client-Server API, except for Account Management. - - The ``as_token`` is inserted into ``access_token`` which is usually where the - client token is, such as via the query string or ``Authorization`` header. This - is done on purpose to allow application services to reuse client SDKs. - - The ``access_token`` should be supplied through the ``Authorization`` header where - possible to prevent the token appearing in HTTP request logs by accident. - -The application service may specify the virtual user to act as through use of a -``user_id`` query string parameter on the request. The user specified in the query -string must be covered by one of the application service's ``user`` namespaces. If -the parameter is missing, the homeserver is to assume the application service intends -to act as the user implied by the ``sender_localpart`` property of the registration. - -An example request would be:: - - GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org - Authorization: Bearer YourApplicationServiceTokenHere - -.. TODO-TravisR: Temporarily take out timestamp massaging while we're releasing r0. - See https://github.com/matrix-org/matrix-doc/issues/1585 -.. Timestamp massaging - +++++++++++++++++++ - The application service may want to inject events at a certain time (reflecting - the time on the network they are tracking e.g. irc, xmpp). Application services - need to be able to adjust the ``origin_server_ts`` value to do this. - - Inputs: - - Application service token (``as_token``) - - Desired timestamp (in milliseconds since the unix epoch) - - Notes: - - This will only apply when sending events. - - :: - - PUT /_matrix/client/r0/rooms/!somewhere:example.org/send/m.room.message/txnId?ts=1534535223283 - Authorization: Bearer YourApplicationServiceTokenHere - - Content: The event to send, as per the Client-Server API. - -Timestamp massaging -+++++++++++++++++++ - -Previous drafts of the Application Service API permitted application services -to alter the timestamp of their sent events by providing a ``ts`` query parameter -when sending an event. This API has been excluded from the first release due to -design concerns, however some servers may still support the feature. Please visit -`issue #1585 `_ for more -information. - -Server admin style permissions -++++++++++++++++++++++++++++++ - -.. _sect:asapi-permissions: - -The homeserver needs to give the application service *full control* over its -namespace, both for users and for room aliases. This means that the AS should -be able to create/edit/delete any room alias in its namespace, as well as -create/delete any user in its namespace. No additional API changes need to be -made in order for control of room aliases to be granted to the AS. Creation of -users needs API changes in order to: - -- Work around captchas. -- Have a 'passwordless' user. - -This involves bypassing the registration flows entirely. This is achieved by -including the ``as_token`` on a ``/register`` request, along with a login type of -``m.login.application_service`` to set the desired user ID without a password. - -:: - - POST /_matrix/client/%CLIENT_MAJOR_VERSION%/register - Authorization: Bearer YourApplicationServiceTokenHere - - Content: - { - type: "m.login.application_service", - username: "_irc_example" - } - -Application services which attempt to create users or aliases *outside* of -their defined namespaces will receive an error code ``M_EXCLUSIVE``. Similarly, -normal users who attempt to create users or aliases *inside* an application -service-defined namespace will receive the same ``M_EXCLUSIVE`` error code, -but only if the application service has defined the namespace as ``exclusive``. - -Using ``/sync`` and ``/events`` -+++++++++++++++++++++++++++++++ - -Application services wishing to use ``/sync`` or ``/events`` from the Client-Server -API MUST do so with a virtual user (provide a ``user_id`` via the query string). It -is expected that the application service use the transactions pushed to it to -handle events rather than syncing with the user implied by ``sender_localpart``. - -Application service room directories -++++++++++++++++++++++++++++++++++++ - -Application services can maintain their own room directories for their defined -third party protocols. These room directories may be accessed by clients through -additional parameters on the ``/publicRooms`` client-server endpoint. - -{{appservice_room_directory_cs_http_api}} - -Referencing messages from a third party network -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Application services should include an ``external_url`` in the ``content`` of -events it emits to indicate where the message came from. This typically applies -to application services that bridge other networks into Matrix, such as IRC, -where an HTTP URL may be available to reference. - -Clients should provide users with a way to access the ``external_url`` if it -is present. Clients should additionally ensure the URL has a scheme of ``https`` -or ``http`` before making use of it. - -The presence of an ``external_url`` on an event does not necessarily mean the -event was sent from an application service. Clients should be wary of the URL -contained within, as it may not be a legitimate reference to the event's source. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst deleted file mode 100644 index 8afd6d31..00000000 --- a/specification/client_server_api.rst +++ /dev/null @@ -1,2083 +0,0 @@ -.. Copyright 2016-2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Client-Server API -================= - -{{unstable_warning_block_CLIENT_RELEASE_LABEL}} - -The client-server API provides a simple lightweight API to let clients send -messages, control rooms and synchronise conversation history. It is designed to -support both lightweight clients which store no state and lazy-load data from -the server as required - as well as heavyweight clients which maintain a full -local persistent copy of server state. - -.. contents:: Table of Contents -.. sectnum:: - -Changelog ---------- - -.. topic:: Version: %CLIENT_RELEASE_LABEL% -{{client_server_changelog}} - -This version of the specification is generated from -`matrix-doc `_ as of Git commit -`{{git_version}} `_. - -For the full historical changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/changelogs/client_server.rst - -Other versions of this specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following other versions are also available, in reverse chronological order: - -- `HEAD `_: Includes all changes since the latest versioned release. -- `r0.6.1 `_ -- `r0.6.0 `_ -- `r0.5.0 `_ -- `r0.4.0 `_ -- `r0.3.0 `_ -- `r0.2.0 `_ -- `r0.1.0 `_ -- `r0.0.1 `_ -- `r0.0.0 `_ -- `Legacy `_: The last draft before the spec was formally released in version r0.0.0. - - -API Standards -------------- - -.. TODO: Move a lot of this to a common area for all specs. - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks - We need to specify capability negotiation for extensible transports - -The mandatory baseline for client-server communication in Matrix is exchanging -JSON objects over HTTP APIs. HTTPS is recommended for communication, although -HTTP may be supported as a fallback to support basic -HTTP clients. More efficient optional transports -will in future be supported as optional extensions - e.g. a -packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all -API calls use a Content-Type of ``application/json``. In addition, all strings -MUST be encoded as UTF-8. Clients are authenticated using opaque -``access_token`` strings (see `Client Authentication`_ for details), passed as a -query string parameter on all requests. - -The names of the API endpoints for the HTTP transport follow a convention of -using underscores to separate words (for example ``/delete_devices``). The key -names in JSON objects passed over the API also follow this convention. - -.. NOTE:: - There are a few historical exceptions to this rule, such as - ``/createRoom``. A future version of this specification will address the - inconsistency. - -Any errors which occur at the Matrix API level MUST return a "standard error -response". This is a JSON object which looks like: - -.. code:: json - - { - "errcode": "", - "error": "" - } - -The ``error`` string will be a human-readable error message, usually a sentence -explaining what went wrong. The ``errcode`` string will be a unique string -which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error -codes should have their namespace first in ALL CAPS, followed by a single _ to -ease separating the namespace from the error code. For example, if there was a -custom namespace ``com.mydomain.here``, and a -``FORBIDDEN`` code, the error code should look like -``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the -error, but the keys ``error`` and ``errcode`` MUST always be present. - -Errors are generally best expressed by their error code rather than the HTTP -status code returned. When encountering the error code ``M_UNKNOWN``, clients -should prefer the HTTP status code as a more reliable reference for what the -issue was. For example, if the client receives an error code of ``M_NOT_FOUND`` -but the request gave a 400 Bad Request status code, the client should treat -the error as if the resource was not found. However, if the client were to -receive an error code of ``M_UNKNOWN`` with a 400 Bad Request, the client -should assume that the request being made was invalid. - -The common error codes are: - -:``M_FORBIDDEN``: - Forbidden access, e.g. joining a room without permission, failed login. - -:``M_UNKNOWN_TOKEN``: - The access token specified was not recognised. - - An additional response parameter, ``soft_logout``, might be present on the response - for 401 HTTP status codes. See `the soft logout section <#soft-logout>`_ for more - information. - -:``M_MISSING_TOKEN``: - No access token was specified for the request. - -:``M_BAD_JSON``: - Request contained valid JSON, but it was malformed in some way, e.g. missing - required keys, invalid values for keys. - -:``M_NOT_JSON``: - Request did not contain valid JSON. - -:``M_NOT_FOUND``: - No resource was found for this request. - -:``M_LIMIT_EXCEEDED``: - Too many requests have been sent in a short period of time. Wait a while then - try again. - -:``M_UNKNOWN``: - An unknown error has occurred. - -Other error codes the client might encounter are: - -:``M_UNRECOGNIZED``: - The server did not understand the request. - -:``M_UNAUTHORIZED``: - The request was not correctly authorized. Usually due to login failures. - -:``M_USER_DEACTIVATED``: - The user ID associated with the request has been deactivated. Typically for - endpoints that prove authentication, such as ``/login``. - -:``M_USER_IN_USE``: - Encountered when trying to register a user ID which has been taken. - -:``M_INVALID_USERNAME``: - Encountered when trying to register a user ID which is not valid. - -:``M_ROOM_IN_USE``: - Sent when the room alias given to the ``createRoom`` API is already in use. - -:``M_INVALID_ROOM_STATE``: - Sent when the initial state given to the ``createRoom`` API is invalid. - -:``M_THREEPID_IN_USE``: - Sent when a threepid given to an API cannot be used because the same threepid is already in use. - -:``M_THREEPID_NOT_FOUND``: - Sent when a threepid given to an API cannot be used because no record matching the threepid was found. - -:``M_THREEPID_AUTH_FAILED``: - Authentication could not be performed on the third party identifier. - -:``M_THREEPID_DENIED``: - The server does not permit this third party identifier. This may happen if the server only - permits, for example, email addresses from a particular domain. - -:``M_SERVER_NOT_TRUSTED``: - The client's request used a third party server, e.g. identity server, that this server does not trust. - -:``M_UNSUPPORTED_ROOM_VERSION``: - The client's request to create a room used a room version that the server does not support. - -:``M_INCOMPATIBLE_ROOM_VERSION``: - The client attempted to join a room that has a version the server does not support. Inspect the - ``room_version`` property of the error response for the room's version. - -:``M_BAD_STATE``: - The state change requested cannot be performed, such as attempting to unban - a user who is not banned. - -:``M_GUEST_ACCESS_FORBIDDEN``: - The room or resource does not permit guests to access it. - -:``M_CAPTCHA_NEEDED``: - A Captcha is required to complete the request. - -:``M_CAPTCHA_INVALID``: - The Captcha provided did not match what was expected. - -:``M_MISSING_PARAM``: - A required parameter was missing from the request. - -:``M_INVALID_PARAM``: - A parameter that was specified has the wrong value. For example, the server - expected an integer and instead received a string. - -:``M_TOO_LARGE``: - The request or entity was too large. - -:``M_EXCLUSIVE``: - The resource being requested is reserved by an application service, or the - application service making the request has not created the resource. - -:``M_RESOURCE_LIMIT_EXCEEDED``: - The request cannot be completed because the homeserver has reached a resource - limit imposed on it. For example, a homeserver held in a shared hosting environment - may reach a resource limit if it starts using too much memory or disk space. The - error MUST have an ``admin_contact`` field to provide the user receiving the error - a place to reach out to. Typically, this error will appear on routes which attempt - to modify state (e.g.: sending messages, account data, etc) and not routes which only - read state (e.g.: ``/sync``, get account data, etc). - -:``M_CANNOT_LEAVE_SERVER_NOTICE_ROOM``: - The user is unable to reject an invite to join the server notices room. See the - `Server Notices <#server-notices>`_ module for more information. - -.. TODO: More error codes (covered by other issues) -.. * M_CONSENT_NOT_GIVEN - GDPR: https://github.com/matrix-org/matrix-doc/issues/1512 - -.. _sect:txn_ids: - -The client-server API typically uses ``HTTP PUT`` to submit requests with a -client-generated transaction identifier. This means that these requests are -idempotent. The scope of a transaction identifier is a particular access token. -It **only** serves to identify new -requests from retransmits. After the request has finished, the ``{txnId}`` -value should be changed (how is not specified; a monotonically increasing -integer is recommended). - -Some API endpoints may allow or require the use of ``POST`` requests without a -transaction ID. Where this is optional, the use of a ``PUT`` request is strongly -recommended. - -{{versions_cs_http_api}} - - -.. _`CORS`: - -Web Browser Clients -------------------- - -It is realistic to expect that some clients will be written to be run within a -web browser or similar environment. In these cases, the homeserver should respond -to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers on -all requests. - -Servers MUST expect that clients will approach them with ``OPTIONS`` requests, -allowing clients to discover the CORS headers. All endpoints in this specification -support the ``OPTIONS`` method, however the server MUST NOT perform any logic defined -for the endpoints when approached with an ``OPTIONS`` request. - -When a client approaches the server with a request, the server should respond with -the CORS headers for that route. The recommended CORS headers to be returned by -servers on all requests are: - -.. code:: - - Access-Control-Allow-Origin: * - Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS - Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization - -Server Discovery ----------------- - -In order to allow users to connect to a Matrix server without needing to -explicitly specify the homeserver's URL or other parameters, clients SHOULD use -an auto-discovery mechanism to determine the server's URL based on a user's -Matrix ID. Auto-discovery should only be done at login time. - -In this section, the following terms are used with specific meanings: - -``PROMPT`` - Retrieve the specific piece of information from the user in a way which - fits within the existing client user experience, if the client is inclined to - do so. Failure can take place instead if no good user experience for this is - possible at this point. - -``IGNORE`` - Stop the current auto-discovery mechanism. If no more auto-discovery - mechanisms are available, then the client may use other methods of - determining the required parameters, such as prompting the user, or using - default values. - -``FAIL_PROMPT`` - Inform the user that auto-discovery failed due to invalid/empty data and - ``PROMPT`` for the parameter. - -``FAIL_ERROR`` - Inform the user that auto-discovery did not return any usable URLs. Do not - continue further with the current login process. At this point, valid data - was obtained, but no server is available to serve the client. No further - guess should be attempted and the user should make a conscientious decision - what to do next. - -Well-known URI -~~~~~~~~~~~~~~ - -.. Note:: - Servers hosting the ``.well-known`` JSON file SHOULD offer CORS headers, as - per the `CORS`_ section in this specification. - -The ``.well-known`` method uses a JSON file at a predetermined location to -specify parameter values. The flow for this method is as follows: - -1. Extract the server name from the user's Matrix ID by splitting the Matrix ID - at the first colon. -2. Extract the hostname from the server name. -3. Make a GET request to ``https://hostname/.well-known/matrix/client``. - - a. If the returned status code is 404, then ``IGNORE``. - b. If the returned status code is not 200, or the response body is empty, - then ``FAIL_PROMPT``. - c. Parse the response body as a JSON object - - i. If the content cannot be parsed, then ``FAIL_PROMPT``. - - d. Extract the ``base_url`` value from the ``m.homeserver`` property. This - value is to be used as the base URL of the homeserver. - - i. If this value is not provided, then ``FAIL_PROMPT``. - - e. Validate the homeserver base URL: - - i. Parse it as a URL. If it is not a URL, then ``FAIL_ERROR``. - ii. Clients SHOULD validate that the URL points to a valid homeserver - before accepting it by connecting to the |/_matrix/client/versions|_ - endpoint, ensuring that it does not return an error, and parsing and - validating that the data conforms with the expected response - format. If any step in the validation fails, then - ``FAIL_ERROR``. Validation is done as a simple check against - configuration errors, in order to ensure that the discovered address - points to a valid homeserver. - - f. If the ``m.identity_server`` property is present, extract the - ``base_url`` value for use as the base URL of the identity server. - Validation for this URL is done as in the step above, but using - ``/_matrix/identity/api/v1`` as the endpoint to connect to. If the - ``m.identity_server`` property is present, but does not have a - ``base_url`` value, then ``FAIL_ERROR``. - -{{wellknown_cs_http_api}} - -Client Authentication ---------------------- - -Most API endpoints require the user to identify themselves by presenting -previously obtained credentials in the form of an ``access_token`` query -parameter or through an Authorization Header of ``Bearer $access_token``. -An access token is typically obtained via the `Login`_ or `Registration`_ processes. - -.. NOTE:: - - This specification does not mandate a particular format for the access - token. Clients should treat it as an opaque byte sequence. Servers are free - to choose an appropriate format. Server implementors may like to investigate - `macaroons `_. - -Using access tokens -~~~~~~~~~~~~~~~~~~~ - -Access tokens may be provided in two ways, both of which the homeserver MUST -support: - -1. Via a query string parameter, ``access_token=TheTokenHere``. -#. Via a request header, ``Authorization: Bearer TheTokenHere``. - -Clients are encouraged to use the ``Authorization`` header where possible -to prevent the access token being leaked in access/HTTP logs. The query -string should only be used in cases where the ``Authorization`` header is -inaccessible for the client. - -When credentials are required but missing or invalid, the HTTP call will -return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or -``M_UNKNOWN_TOKEN`` respectively. - -Relationship between access tokens and devices -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Client `devices`_ are closely related to access tokens. Matrix servers should -record which device each access token is assigned to, so that subsequent -requests can be handled correctly. - -By default, the `Login`_ and `Registration`_ processes auto-generate a new -``device_id``. A client is also free to generate its own ``device_id`` or, -provided the user remains the same, reuse a device: in either case the client -should pass the ``device_id`` in the request body. If the client sets the -``device_id``, the server will invalidate any access token previously assigned -to that device. There is therefore at most one active access token assigned to -each device at any one time. - -Soft logout -~~~~~~~~~~~ - -When a request fails due to a 401 status code per above, the server can -include an extra response parameter, ``soft_logout``, to indicate if the client's -persisted information can be retained. This defaults to ``false``, indicating -that the server has destroyed the session. Any persisted state held by the client, -such as encryption keys and device information, must not be reused and must be discarded. - -When ``soft_logout`` is true, the client can acquire a new access token by -specifying the device ID it is already using to the login API. In most cases -a ``soft_logout: true`` response indicates that the user's session has expired -on the server-side and the user simply needs to provide their credentials again. - -In either case, the client's previously known access token will no longer function. - -.. _`user-interactive authentication`: - -User-Interactive Authentication API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Overview -<<<<<<<< - -Some API endpoints require authentication that interacts with the user. The -homeserver may provide many different ways of authenticating, such as -user/password auth, login via a single-sign-on server (SSO), etc. This -specification does not define how homeservers should authorise their users but -instead defines the standard interface which implementations should follow so -that ANY client can log in to ANY homeserver. - -The process takes the form of one or more 'stages'. At each stage the client -submits a set of data for a given authentication type and awaits a response -from the server, which will either be a final success or a request to perform -an additional stage. This exchange continues until the final success. - -For each endpoint, a server offers one or more 'flows' that the client can use -to authenticate itself. Each flow comprises a series of stages, as described -above. The client is free to choose which flow it follows, however the flow's -stages must be completed in order. Failing to follow the flows in order must -result in an HTTP 401 response, as defined below. When all stages in a flow -are complete, authentication is complete and the API call succeeds. - -User-interactive API in the REST API -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -In the REST API described in this specification, authentication works by the -client and server exchanging JSON dictionaries. The server indicates what -authentication data it requires via the body of an HTTP 401 response, and the -client submits that authentication data via the ``auth`` request parameter. - -A client should first make a request with no ``auth`` parameter [#]_. The -homeserver returns an HTTP 401 response, with a JSON body, as follows: - -.. code:: - - HTTP/1.1 401 Unauthorized - Content-Type: application/json - - { - "flows": [ - { - "stages": [ "example.type.foo", "example.type.bar" ] - }, - { - "stages": [ "example.type.foo", "example.type.baz" ] - } - ], - "params": { - "example.type.baz": { - "example_key": "foobar" - } - }, - "session": "xxxxxx" - } - -In addition to the ``flows``, this object contains some extra -information: - -params - This section contains any information that the client will need to know in - order to use a given type of authentication. For each authentication type - presented, that type may be present as a key in this dictionary. For example, - the public part of an OAuth client ID could be given here. -session - This is a session identifier that the client must pass back to the homeserver, - if one is provided, in subsequent attempts to authenticate in the same API call. - -The client then chooses a flow and attempts to complete the first stage. It -does this by resubmitting the same request with the addition of an ``auth`` -key in the object that it submits. This dictionary contains a ``type`` key whose -value is the name of the authentication type that the client is attempting to complete. -It must also contain a ``session`` key with the value of the session key given -by the homeserver, if one was given. It also contains other keys dependent on -the auth type being attempted. For example, if the client is attempting to -complete auth type ``example.type.foo``, it might submit something like this: - -.. code:: - - POST /_matrix/client/r0/endpoint HTTP/1.1 - Content-Type: application/json - - { - "a_request_parameter": "something", - "another_request_parameter": "something else", - "auth": { - "type": "example.type.foo", - "session": "xxxxxx", - "example_credential": "verypoorsharedsecret" - } - } - -If the homeserver deems the authentication attempt to be successful but still -requires more stages to be completed, it returns HTTP status 401 along with the -same object as when no authentication was attempted, with the addition of the -``completed`` key which is an array of auth types the client has completed -successfully: - -.. code:: - - HTTP/1.1 401 Unauthorized - Content-Type: application/json - - { - "completed": [ "example.type.foo" ], - "flows": [ - { - "stages": [ "example.type.foo", "example.type.bar" ] - }, - { - "stages": [ "example.type.foo", "example.type.baz" ] - } - ], - "params": { - "example.type.baz": { - "example_key": "foobar" - } - }, - "session": "xxxxxx" - } - -Individual stages may require more than one request to complete, in which case -the response will be as if the request was unauthenticated with the addition of -any other keys as defined by the auth type. - -If the homeserver decides that an attempt on a stage was unsuccessful, but the -client may make a second attempt, it returns the same HTTP status 401 response -as above, with the addition of the standard ``errcode`` and ``error`` fields -describing the error. For example: - -.. code:: - - HTTP/1.1 401 Unauthorized - Content-Type: application/json - - { - "errcode": "M_FORBIDDEN", - "error": "Invalid password", - "completed": [ "example.type.foo" ], - "flows": [ - { - "stages": [ "example.type.foo", "example.type.bar" ] - }, - { - "stages": [ "example.type.foo", "example.type.baz" ] - } - ], - "params": { - "example.type.baz": { - "example_key": "foobar" - } - }, - "session": "xxxxxx" - } - -If the request fails for a reason other than authentication, the server returns an error -message in the standard format. For example: - -.. code:: - - HTTP/1.1 400 Bad request - Content-Type: application/json - - { - "errcode": "M_EXAMPLE_ERROR", - "error": "Something was wrong" - } - -If the client has completed all stages of a flow, the homeserver performs the -API call and returns the result as normal. Completed stages cannot be retried -by clients, therefore servers must return either a 401 response with the completed -stages, or the result of the API call if all stages were completed when a client -retries a stage. - -Some authentication types may be completed by means other than through the -Matrix client, for example, an email confirmation may be completed when the user -clicks on the link in the email. In this case, the client retries the request -with an auth dict containing only the session key. The response to this will be -the same as if the client were attempting to complete an auth state normally, -i.e. the request will either complete or request auth, with the presence or -absence of that auth type in the 'completed' array indicating whether -that stage is complete. - -.. [#] A request to an endpoint that uses User-Interactive Authentication never - succeeds without auth. Homeservers may allow requests that don't require - auth by offering a stage with only the ``m.login.dummy`` auth type, but - they must still give a 401 response to requests with no auth data. - -Example -+++++++ -At a high level, the requests made for an API call completing an auth flow with -three stages will resemble the following diagram:: - - _______________________ - | Stage 0 | - | No auth | - | ___________________ | - | |_Request_1_________| | <-- Returns "session" key which is used throughout. - |_______________________| - | - | - _________V_____________ - | Stage 1 | - | type: "" | - | ___________________ | - | |_Request_1_________| | - |_______________________| - | - | - _________V_____________ - | Stage 2 | - | type: "" | - | ___________________ | - | |_Request_1_________| | - | ___________________ | - | |_Request_2_________| | - | ___________________ | - | |_Request_3_________| | - |_______________________| - | - | - _________V_____________ - | Stage 3 | - | type: "" | - | ___________________ | - | |_Request_1_________| | <-- Returns API response - |_______________________| - - -Authentication types -++++++++++++++++++++ - -This specification defines the following auth types: - - ``m.login.password`` - - ``m.login.recaptcha`` - - ``m.login.sso`` - - ``m.login.email.identity`` - - ``m.login.msisdn`` - - ``m.login.dummy`` - -Password-based -<<<<<<<<<<<<<< -:Type: - ``m.login.password`` -:Description: - The client submits an identifier and secret password, both sent in plain-text. - -To use this authentication type, clients should submit an auth dict as follows: - -.. code:: json - - { - "type": "m.login.password", - "identifier": { - ... - }, - "password": "", - "session": "" - } - -where the ``identifier`` property is a user identifier object, as described in -`Identifier types`_. - -For example, to authenticate using the user's Matrix ID, clients would submit: - -.. code:: json - - { - "type": "m.login.password", - "identifier": { - "type": "m.id.user", - "user": "" - }, - "password": "", - "session": "" - } - -Alternatively reply using a 3PID bound to the user's account on the homeserver -using the |/account/3pid|_ API rather than giving the ``user`` explicitly as -follows: - -.. code:: json - - { - "type": "m.login.password", - "identifier": { - "type": "m.id.thirdparty", - "medium": "", - "address": "" - }, - "password": "", - "session": "" - } - -In the case that the homeserver does not know about the supplied 3PID, the -homeserver must respond with 403 Forbidden. - -Google ReCaptcha -<<<<<<<<<<<<<<<< -:Type: - ``m.login.recaptcha`` -:Description: - The user completes a Google ReCaptcha 2.0 challenge - -To use this authentication type, clients should submit an auth dict as follows: - -.. code:: json - - { - "type": "m.login.recaptcha", - "response": "", - "session": "" - } - -Single Sign-On -<<<<<<<<<<<<<< -:Type: - ``m.login.sso`` -:Description: - Authentication is supported by authorising with an external single sign-on - provider. - -A client wanting to complete authentication using SSO should use the -`Fallback`_ mechanism. See `SSO during User-Interactive Authentication`_ for -more information. - -Email-based (identity / homeserver) -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -:Type: - ``m.login.email.identity`` -:Description: - Authentication is supported by authorising an email address with an identity - server, or homeserver if supported. - -Prior to submitting this, the client should authenticate with an identity -server (or homeserver). After authenticating, the session information should -be submitted to the homeserver. - -To use this authentication type, clients should submit an auth dict as follows: - -.. code:: json - - { - "type": "m.login.email.identity", - "threepidCreds": [ - { - "sid": "", - "client_secret": "", - "id_server": "", - "id_access_token": "" - } - ], - "session": "" - } - -Note that ``id_server`` (and therefore ``id_access_token``) is optional if the -``/requestToken`` request did not include them. - -Phone number/MSISDN-based (identity / homeserver) -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -:Type: - ``m.login.msisdn`` -:Description: - Authentication is supported by authorising a phone number with an identity - server, or homeserver if supported. - -Prior to submitting this, the client should authenticate with an identity -server (or homeserver). After authenticating, the session information should -be submitted to the homeserver. - -To use this authentication type, clients should submit an auth dict as follows: - -.. code:: json - - { - "type": "m.login.msisdn", - "threepidCreds": [ - { - "sid": "", - "client_secret": "", - "id_server": "", - "id_access_token": "" - } - ], - "session": "" - } - -Note that ``id_server`` (and therefore ``id_access_token``) is optional if the -``/requestToken`` request did not include them. - -Dummy Auth -<<<<<<<<<< -:Type: - ``m.login.dummy`` -:Description: - Dummy authentication always succeeds and requires no extra parameters. Its - purpose is to allow servers to not require any form of User-Interactive - Authentication to perform a request. It can also be used to differentiate - flows where otherwise one flow would be a subset of another flow. e.g. if - a server offers flows ``m.login.recaptcha`` and ``m.login.recaptcha, - m.login.email.identity`` and the client completes the recaptcha stage first, - the auth would succeed with the former flow, even if the client was intending - to then complete the email auth stage. A server can instead send flows - ``m.login.recaptcha, m.login.dummy`` and ``m.login.recaptcha, - m.login.email.identity`` to fix the ambiguity. - -To use this authentication type, clients should submit an auth dict with just -the type and session, if provided: - -.. code:: json - - { - "type": "m.login.dummy", - "session": "" - } - - -Fallback -++++++++ -Clients cannot be expected to be able to know how to process every single login -type. If a client does not know how to handle a given login type, it can direct -the user to a web browser with the URL of a fallback page which will allow the -user to complete that login step out-of-band in their web browser. The URL it -should open is:: - - /_matrix/client/%CLIENT_MAJOR_VERSION%/auth//fallback/web?session= - -Where ``auth type`` is the type name of the stage it is attempting and -``session ID`` is the ID of the session given by the homeserver. - -.. _`user-interactive authentication fallback completion`: - -This MUST return an HTML page which can perform this authentication stage. This -page must use the following JavaScript when the authentication has been -completed: - -.. code:: javascript - - if (window.onAuthDone) { - window.onAuthDone(); - } else if (window.opener && window.opener.postMessage) { - window.opener.postMessage("authDone", "*"); - } - -This allows the client to either arrange for the global function ``onAuthDone`` -to be defined in an embedded browser, or to use the HTML5 `cross-document -messaging `_ API, to receive -a notification that the authentication stage has been completed. - -Once a client receives the notification that the authentication stage has been -completed, it should resubmit the request with an auth dict with just the -session ID: - -.. code:: json - - { - "session": "" - } - - -Example -<<<<<<< -A client webapp might use the following JavaScript to open a popup window which will -handle unknown login types: - -.. code:: javascript - - /** - * Arguments: - * homeserverUrl: the base url of the homeserver (e.g. "https://matrix.org") - * - * apiEndpoint: the API endpoint being used (e.g. - * "/_matrix/client/%CLIENT_MAJOR_VERSION%/account/password") - * - * loginType: the loginType being attempted (e.g. "m.login.recaptcha") - * - * sessionID: the session ID given by the homeserver in earlier requests - * - * onComplete: a callback which will be called with the results of the request - */ - function unknownLoginType(homeserverUrl, apiEndpoint, loginType, sessionID, onComplete) { - var popupWindow; - - var eventListener = function(ev) { - // check it's the right message from the right place. - if (ev.data !== "authDone" || ev.origin !== homeserverUrl) { - return; - } - - // close the popup - popupWindow.close(); - window.removeEventListener("message", eventListener); - - // repeat the request - var requestBody = { - auth: { - session: sessionID, - }, - }; - - request({ - method:'POST', url:apiEndpoint, json:requestBody, - }, onComplete); - }; - - window.addEventListener("message", eventListener); - - var url = homeserverUrl + - "/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/" + - encodeURIComponent(loginType) + - "/fallback/web?session=" + - encodeURIComponent(sessionID); - - - popupWindow = window.open(url); - } - - -Identifier types -++++++++++++++++ - -Some authentication mechanisms use a user identifier object to identify a -user. The user identifier object has a ``type`` field to indicate the type of -identifier being used, and depending on the type, has other fields giving the -information required to identify the user as described below. - -This specification defines the following identifier types: - - ``m.id.user`` - - ``m.id.thirdparty`` - - ``m.id.phone`` - -Matrix User ID -<<<<<<<<<<<<<< -:Type: - ``m.id.user`` -:Description: - The user is identified by their Matrix ID. - -A client can identify a user using their Matrix ID. This can either be the -fully qualified Matrix user ID, or just the localpart of the user ID. - -.. code:: json - - "identifier": { - "type": "m.id.user", - "user": "" - } - -Third-party ID -<<<<<<<<<<<<<< -:Type: - ``m.id.thirdparty`` -:Description: - The user is identified by a third-party identifier in canonicalised form. - -A client can identify a user using a 3PID associated with the user's account on -the homeserver, where the 3PID was previously associated using the -|/account/3pid|_ API. See the `3PID Types`_ Appendix for a list of Third-party -ID media. - -.. code:: json - - "identifier": { - "type": "m.id.thirdparty", - "medium": "", - "address": "" - } - -Phone number -<<<<<<<<<<<< -:Type: - ``m.id.phone`` -:Description: - The user is identified by a phone number. - -A client can identify a user using a phone number associated with the user's -account, where the phone number was previously associated using the -|/account/3pid|_ API. The phone number can be passed in as entered by the -user; the homeserver will be responsible for canonicalising it. If the client -wishes to canonicalise the phone number, then it can use the -``m.id.thirdparty`` identifier type with a ``medium`` of ``msisdn`` instead. - -.. code:: json - - "identifier": { - "type": "m.id.phone", - "country": "", - "phone": "" - } - -The ``country`` is the two-letter uppercase ISO-3166-1 alpha-2 country code -that the number in ``phone`` should be parsed as if it were dialled from. - -Login -~~~~~ - -A client can obtain access tokens using the ``/login`` API. - -Note that this endpoint does `not` currently use the `User-Interactive Authentication API`_. - -For a simple username/password login, clients should submit a ``/login`` -request as follows: - -.. code:: json - - { - "type": "m.login.password", - "identifier": { - "type": "m.id.user", - "user": "" - }, - "password": "" - } - -Alternatively, a client can use a 3PID bound to the user's account on the -homeserver using the |/account/3pid|_ API rather than giving the ``user`` -explicitly, as follows: - -.. code:: json - - { - "type": "m.login.password", - "identifier": { - "medium": "", - "address": "" - }, - "password": "" - } - -In the case that the homeserver does not know about the supplied 3PID, the -homeserver must respond with ``403 Forbidden``. - -To log in using a login token, clients should submit a ``/login`` request as -follows: - -.. code:: json - - { - "type": "m.login.token", - "token": "" - } - -As with `token-based`_ interactive login, the ``token`` must encode the -user ID. In the case that the token is not valid, the homeserver must respond -with ``403 Forbidden`` and an error code of ``M_FORBIDDEN``. - -If the homeserver advertises ``m.login.sso`` as a viable flow, and the client -supports it, the client should redirect the user to the ``/redirect`` endpoint -for `client login via SSO`_. After authentication is complete, the -client will need to submit a ``/login`` request matching ``m.login.token``. - -{{login_cs_http_api}} - -{{logout_cs_http_api}} - -Login Fallback -<<<<<<<<<<<<<< - -If a client does not recognize any or all login flows it can use the fallback -login API:: - - GET /_matrix/static/client/login/ - -This returns an HTML and JavaScript page which can perform the entire login -process. The page will attempt to call the JavaScript function -``window.onLogin`` when login has been successfully completed. - -Non-credential parameters valid for the ``/login`` endpoint can be provided as query -string parameters here. These are to be forwarded to the login endpoint during the login -process. For example:: - - GET /_matrix/static/client/login/?device_id=GHTYAJCE - -.. _Registration: - -Account registration and management -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -{{registration_cs_http_api}} - -Notes on password management -++++++++++++++++++++++++++++ - -.. WARNING:: - Clients SHOULD enforce that the password provided is suitably complex. The - password SHOULD include a lower-case letter, an upper-case letter, a number - and a symbol and be at a minimum 8 characters in length. Servers MAY reject - weak passwords with an error code ``M_WEAK_PASSWORD``. - - -Adding Account Administrative Contact Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A homeserver may keep some contact information for administrative use. -This is independent of any information kept by any identity servers, though -can be proxied (bound) to the identity server in many cases. - -.. Note:: - This section deals with two terms: "add" and "bind". Where "add" (or "remove") - is used, it is speaking about an identifier that was not bound to an identity - server. As a result, "bind" (or "unbind") references an identifier that is found - in an identity server. Note that an identifier can be added and bound at the same - time, depending on context. - -{{administrative_contact_cs_http_api}} - -Current account information -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -{{whoami_cs_http_api}} - -Notes on identity servers -+++++++++++++++++++++++++ - -Identity servers in Matrix store bindings (relationships) between a user's third -party identifier, typically email or phone number, and their user ID. Once a user -has chosen an identity server, that identity server should be used by all clients. - -Clients can see which identity server the user has chosen through the ``m.identity_server`` -account data event, as described below. Clients SHOULD refrain from making requests -to any identity server until the presence of ``m.identity_server`` is confirmed as -(not) present. If present, the client SHOULD check for the presence of the ``base_url`` -property in the event's content. If the ``base_url`` is present, the client SHOULD -use the identity server in that property as the identity server for the user. If the -``base_url`` is missing, or the account data event is not present, the client SHOULD -use whichever default value it normally would for an identity server, if applicable. -Clients SHOULD NOT update the account data with the default identity server when the -user is missing an identity server in their account data. - -Clients SHOULD listen for changes to the ``m.identity_server`` account data event -and update the identity server they are contacting as a result. - -If the client offers a way to set the identity server to use, it MUST update the -value of ``m.identity_server`` accordingly. A ``base_url`` of ``null`` MUST be -treated as though the user does not want to use an identity server, disabling all -related functionality as a result. - -Clients SHOULD refrain from populating the account data as a migration step for users -who are lacking the account data, unless the user sets the identity server within -the client to a value. For example, a user which has no ``m.identity_server`` account -data event should not end up with the client's default identity server in their -account data, unless the user first visits their account settings to set the identity -server. - -{{m_identity_server_event}} - -Capabilities negotiation ------------------------- - -A homeserver may not support certain operations and clients must be able to -query for what the homeserver can and can't offer. For example, a homeserver -may not support users changing their password as it is configured to perform -authentication against an external system. - -The capabilities advertised through this system are intended to advertise -functionality which is optional in the API, or which depend in some way on -the state of the user or server. This system should not be used to advertise -unstable or experimental features - this is better done by the ``/versions`` -endpoint. - -Some examples of what a reasonable capability could be are: - -* Whether the server supports user presence. - -* Whether the server supports optional features, such as the user or room - directories. - -* The rate limits or file type restrictions imposed on clients by the server. - -Some examples of what should **not** be a capability are: - -* Whether the server supports a feature in the ``unstable`` specification. - -* Media size limits - these are handled by the ``/media/%CLIENT_MAJOR_VERSION%/config`` - API. - -* Optional encodings or alternative transports for communicating with the - server. - -Capabilities prefixed with ``m.`` are reserved for definition in the Matrix -specification while other values may be used by servers using the Java package -naming convention. The capabilities supported by the Matrix specification are -defined later in this section. - -{{capabilities_cs_http_api}} - - -``m.change_password`` capability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This capability has a single flag, ``enabled``, which indicates whether or not -the user can use the ``/account/password`` API to change their password. If not -present, the client should assume that password changes are possible via the -API. When present, clients SHOULD respect the capability's ``enabled`` flag -and indicate to the user if they are unable to change their password. - -An example of the capability API's response for this capability is:: - - { - "capabilities": { - "m.change_password": { - "enabled": false - } - } - } - - -``m.room_versions`` capability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This capability describes the default and available room versions a server -supports, and at what level of stability. Clients should make use of this -capability to determine if users need to be encouraged to upgrade their rooms. - -An example of the capability API's response for this capability is:: - - { - "capabilities": { - "m.room_versions": { - "default": "1", - "available": { - "1": "stable", - "2": "stable", - "3": "unstable", - "custom-version": "unstable" - } - } - } - } - -This capability mirrors the same restrictions of `room versions`_ to describe -which versions are stable and unstable. Clients should assume that the ``default`` -version is ``stable``. Any version not explicitly labelled as ``stable`` in the -``available`` versions is to be treated as ``unstable``. For example, a version -listed as ``future-stable`` should be treated as ``unstable``. - -The ``default`` version is the version the server is using to create new rooms. -Clients should encourage users with sufficient permissions in a room to upgrade -their room to the ``default`` version when the room is using an ``unstable`` -version. - -When this capability is not listed, clients should use ``"1"`` as the default -and only stable ``available`` room version. - -.. _`room versions`: ../index.html#room-versions - - -Pagination ----------- - -.. NOTE:: - The paths referred to in this section are not actual endpoints. They only - serve as examples to explain how pagination functions. - -Pagination is the process of dividing a dataset into multiple discrete pages. -Matrix makes use of pagination to allow clients to view extremely large datasets. -These datasets are not limited to events in a room (for example clients may want -to paginate a list of rooms in addition to events within those rooms). Regardless -of what is being paginated, there is a common approach which is used to give -clients an easy way of selecting subsets of a potentially changing dataset. Each -endpoint that uses pagination may use different parameters. However the theme -among them is that they take a ``from`` and ``to`` token, and occasionally -a ``limit`` and ``dir``. Together, these parameters describe the position in a -data set, where ``from`` and ``to`` are known as "stream tokens" matching the -regular expression ``[a-zA-Z0-9.=_-]+``. If supported, the ``dir`` defines the -direction of events to return: either forwards (``f``) or backwards (``b``). -The response may contain tokens that can be used for retrieving results before -or after the returned set. These tokens may be called `start` or `prev_batch` -for retrieving the previous result set, or `end`, `next_batch` or `next_token` -for retrieving the next result set. - -In the following examples, 'START' and 'END' are placeholders to signify the -start and end of the data sets respectively. - -For example, if an endpoint had events E1 -> E15. The client wants the last 5 -events and doesn't know any previous events:: - - S E - |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| - | | | - | _____| <--backwards-- | - |__________________ | | ________| - | | | | - GET /somepath?to=START&limit=5&dir=b&from=END - Returns: - E15,E14,E13,E12,E11 - - -Another example: a public room list has rooms R1 -> R17. The client is showing 5 -rooms at a time on screen, and is on page 2. They want to now show page 3 (rooms -R11 -> 15):: - - S E - | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token - |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room - |____________| |________________| - | | - Currently | - viewing | - | - GET /roomslist?from=9&to=END&limit=5 - Returns: R11,R12,R13,R14,R15 - -Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the initial request was '9' which corresponded to R10. When the 2nd -request was made, R10 did not appear again, even though from=9 was specified. If -you know the token, you already have the data. - -Responses for pagination-capable endpoints SHOULD have a ``chunk`` array alongside -the applicable stream tokens to represent the result set. - -In general, when the end of a result set is reached the applicable stream token -will be excluded from the response. For example, if a user was backwards-paginating -events in a room they'd eventually reach the first event in the room. In this scenario, -the ``prev_batch`` token would be excluded from the response. Some paginated -endpoints are open-ended in one direction, such as endpoints which expose an event -stream for an active room. In this case, it is not possible for the client to reach -the true "end" of the data set and therefore should always be presented with a token -to keep moving forwards. - -.. _`filter`: - -Filtering ---------- - -Filters can be created on the server and can be passed as a parameter to APIs -which return events. These filters alter the data returned from those APIs. -Not all APIs accept filters. - -Lazy-loading room members -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Membership events often take significant resources for clients to track. In an -effort to reduce the number of resources used, clients can enable "lazy-loading" -for room members. By doing this, servers will attempt to only send membership events -which are relevant to the client. - -It is important to understand that lazy-loading is not intended to be a -perfect optimisation, and that it may not be practical for the server to -calculate precisely which membership events are relevant to the client. As a -result, it is valid for the server to send redundant membership events to the -client to ease implementation, although such redundancy should be minimised -where possible to conserve bandwidth. - -In terms of filters, lazy-loading is enabled by enabling ``lazy_load_members`` -on a ``RoomEventFilter`` (or a ``StateFilter`` in the case of ``/sync`` only). -When enabled, lazy-loading aware endpoints (see below) will only include -membership events for the ``sender`` of events being included in the response. -For example, if a client makes a ``/sync`` request with lazy-loading enabled, -the server will only return membership events for the ``sender`` of events in -the timeline, not all members of a room. - -When processing a sequence of events (e.g. by looping on ``/sync`` or -paginating ``/messages``), it is common for blocks of events in the sequence -to share a similar set of senders. Rather than responses in the sequence -sending duplicate membership events for these senders to the client, the -server MAY assume that clients will remember membership events they have -already been sent, and choose to skip sending membership events for members -whose membership has not changed. These are called 'redundant membership -events'. Clients may request that redundant membership events are always -included in responses by setting ``include_redundant_members`` to true in the -filter. - -The expected pattern for using lazy-loading is currently: - -* Client performs an initial /sync with lazy-loading enabled, and receives - only the membership events which relate to the senders of the events it - receives. -* Clients which support display-name tab-completion or other operations which - require rapid access to all members in a room should call /members for the - currently selected room, with an ``?at`` parameter set to the /sync - response's from token. The member list for the room is then maintained by - the state in subsequent incremental /sync responses. -* Clients which do not support tab-completion may instead pull in profiles for - arbitrary users (e.g. read receipts, typing notifications) on demand by - querying the room state or ``/profile``. - -.. TODO-spec - This implies that GET /state should also take an ``?at`` param - -The current endpoints which support lazy-loading room members are: - -* |/sync|_ -* |/rooms//messages|_ -* |/rooms/{roomId}/context/{eventId}|_ - -API endpoints -~~~~~~~~~~~~~ - -{{filter_cs_http_api}} - -Events ------- - -.. _sect:events: - -The model of conversation history exposed by the client-server API can be -considered as a list of events. The server 'linearises' the -eventually-consistent event graph of events into an 'event stream' at any given -point in time:: - - [E0]->[E1]->[E2]->[E3]->[E4]->[E5] - -.. WARNING:: - - The format of events can change depending on room version. Check the - `room version specification`_ for specific details on what to expect for - event formats. Examples contained within the client-server specification - are expected to be compatible with all specified room versions, however - some differences may still apply. - - For this version of the specification, clients only need to worry about - the event ID format being different depending on room version. Clients - should not be parsing the event ID, and instead be treating it as an - opaque string. No changes should be required to support the currently - available room versions. - -.. _`room version specification`: ../index.html#room-versions - -Types of room events -~~~~~~~~~~~~~~~~~~~~ - -Room events are split into two categories: - -:State Events: - These are events which update the metadata state of the room (e.g. room topic, - room membership etc). State is keyed by a tuple of event ``type`` and a - ``state_key``. State in the room with the same key-tuple will be overwritten. - -:Message events: - These are events which describe transient "once-off" activity in a room: - typically communication such as sending an instant message or setting up a - VoIP call. - -This specification outlines several events, all with the event type prefix -``m.``. (See `Room Events`_ for the m. event specification.) However, -applications may wish to add their own type of event, and this can be achieved -using the REST API detailed in the following sections. If new events are added, -the event ``type`` key SHOULD follow the Java package naming convention, -e.g. ``com.example.myapp.event``. This ensures event types are suitably -namespaced for each application and reduces the risk of clashes. - -.. Note:: - Events are not limited to the types defined in this specification. New or custom - event types can be created on a whim using the Java package naming convention. - For example, a ``com.example.game.score`` event can be sent by clients and other - clients would receive it through Matrix, assuming the client has access to the - ``com.example`` namespace. - -Note that the structure of these events may be different than those in the -server-server API. - -{{common_event_fields}} - -{{common_room_event_fields}} - -.. This is normally where we'd put the common_state_event_fields variable for the -.. magic table of what makes up a state event, however the table is verbose in our -.. custom rendering of swagger. To combat this, we just hardcode this particular -.. table. - -State Event Fields -++++++++++++++++++ - -In addition to the fields of a Room Event, State Events have the following fields. - -+--------------+--------------+-------------------------------------------------------------+ -| Key | Type | Description | -+==============+==============+=============================================================+ -| state_key | string | **Required.** A unique key which defines the overwriting | -| | | semantics for this piece of room state. This value is often | -| | | a zero-length string. The presence of this key makes this | -| | | event a State Event. State keys starting with an ``@`` are | -| | | reserved for referencing user IDs, such as room members. | -| | | With the exception of a few events, state events set with | -| | | a given user's ID as the state key MUST only be set by that | -| | | user. | -+--------------+--------------+-------------------------------------------------------------+ -| prev_content | EventContent | Optional. The previous ``content`` for this event. If there | -| | | is no previous content, this key will be missing. | -+--------------+--------------+-------------------------------------------------------------+ - - -Size limits -~~~~~~~~~~~ - -The complete event MUST NOT be larger than 65535 bytes, when formatted as a -`PDU for the Server-Server protocol <../server_server/%SERVER_RELEASE_LABEL%#pdus>`_, -including any signatures, and encoded as `Canonical JSON`_. - -There are additional restrictions on sizes per key: - -- ``sender`` MUST NOT exceed 255 bytes (including domain). -- ``room_id`` MUST NOT exceed 255 bytes. -- ``state_key`` MUST NOT exceed 255 bytes. -- ``type`` MUST NOT exceed 255 bytes. -- ``event_id`` MUST NOT exceed 255 bytes. - -Some event types have additional size restrictions which are specified in -the description of the event. Additional keys have no limit other than that -implied by the total 65 KB limit on events. - -.. _`Canonical JSON`: ../appendices.html#canonical-json - -Room Events -~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - -This specification outlines several standard event types, all of which are -prefixed with ``m.`` - -{{m_room_canonical_alias_event}} - -{{m_room_create_event}} - -{{m_room_join_rules_event}} - -{{m_room_member_event}} - -{{m_room_power_levels_event}} - -Historical events -+++++++++++++++++ - -Some events within the ``m.`` namespace might appear in rooms, however they -serve no significant meaning in this version of the specification. They are: - -* ``m.room.aliases`` - -Previous versions of the specification have more information on these events. - -Syncing -~~~~~~~ - -To read events, the intended flow of operation is for clients to first call the -|/sync|_ API without a ``since`` parameter. This returns the most recent -message events for each room, as well as the state of the room at the start of -the returned timeline. The response also includes a ``next_batch`` field, which -should be used as the value of the ``since`` parameter in the next call to -``/sync``. Finally, the response includes, for each room, a ``prev_batch`` -field, which can be passed as a ``start`` parameter to the -|/rooms//messages|_ API to retrieve earlier messages. - -You can visualise the range of events being returned as:: - - [E0]->[E1]->[E2]->[E3]->[E4]->[E5] - ^ ^ - | | - prev_batch: '1-2-3' next_batch: 'a-b-c' - - -Clients then receive new events by "long-polling" the homeserver via the -``/sync`` API, passing the value of the ``next_batch`` field from the response -to the previous call as the ``since`` parameter. The client should also pass a -``timeout`` parameter. The server will then hold open the HTTP connection for a -short period of time waiting for new events, returning early if an event -occurs. Only the ``/sync`` API (and the deprecated ``/events`` API) support -long-polling in this way. - -The response for such an incremental sync can be visualised as:: - - [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6] - ^ ^ - | | - | next_batch: 'x-y-z' - prev_batch: 'a-b-c' - - -Normally, all new events which are visible to the client will appear in the -response to the ``/sync`` API. However, if a large number of events arrive -between calls to ``/sync``, a "limited" timeline is returned, containing only -the most recent message events. A state "delta" is also returned, summarising -any state changes in the omitted part of the timeline. The client may therefore -end up with "gaps" in its knowledge of the message timeline. The client can -fill these gaps using the |/rooms//messages|_ API. This situation -looks like this:: - - | gap | - | <-> | - [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6]->[E7]->[E8]->[E9]->[E10] - ^ ^ - | | - prev_batch: 'd-e-f' next_batch: 'u-v-w' - - -.. Warning:: - Events are ordered in this API according to the arrival time of the event on - the homeserver. This can conflict with other APIs which order events based on - their partial ordering in the event graph. This can result in duplicate events - being received (once per distinct API called). Clients SHOULD de-duplicate - events based on the event ID when this happens. - -.. NOTE:: - - The ``/sync`` API returns a ``state`` list which is separate from the - ``timeline``. This ``state`` list allows clients to keep their model of the - room state in sync with that on the server. In the case of an initial - (``since``-less) sync, the ``state`` list represents the complete state of - the room at the **start** of the returned timeline (so in the case of a - recently-created room whose state fits entirely in the ``timeline``, the - ``state`` list will be empty). - - In the case of an incremental sync, the ``state`` list gives a delta - between the state of the room at the ``since`` parameter and that at the - start of the returned ``timeline``. (It will therefore be empty - unless the timeline was ``limited``.) - - In both cases, it should be noted that the events returned in the ``state`` - list did **not** necessarily take place just before the returned - ``timeline``, so clients should not display them to the user in the timeline. - -.. admonition:: Rationale - - An early design of this specification made the ``state`` list represent the - room state at the end of the returned timeline, instead of the start. This - was unsatisfactory because it led to duplication of events between the - ``state`` list and the ``timeline``, but more importantly, it made it - difficult for clients to show the timeline correctly. - - In particular, consider a returned timeline [M0, S1, M2], where M0 and M2 are - both messages sent by the same user, and S1 is a state event where that user - changes their displayname. If the ``state`` list represents the room state at - the end of the timeline, the client must take a copy of the state dictionary, - and *rewind* S1, in order to correctly calculate the display name for M0. - -.. TODO-spec - Do we ever support streaming requests? Why not websockets? - -{{sync_cs_http_api}} - -{{old_sync_cs_http_api}} - - -Getting events for a room -~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are several APIs provided to ``GET`` events for a room: - -{{rooms_cs_http_api}} - -{{message_pagination_cs_http_api}} - -{{room_initial_sync_cs_http_api}} - - -Sending events to a room -~~~~~~~~~~~~~~~~~~~~~~~~ - -{{room_state_cs_http_api}} - - -**Examples** - -Valid requests look like:: - - PUT /rooms/!roomid:domain/state/m.example.event - { "key" : "without a state key" } - - PUT /rooms/!roomid:domain/state/m.another.example.event/foo - { "key" : "with 'foo' as the state key" } - -In contrast, these requests are invalid:: - - POST /rooms/!roomid:domain/state/m.example.event/ - { "key" : "cannot use POST here" } - - PUT /rooms/!roomid:domain/state/m.another.example.event/foo/11 - { "key" : "txnIds are not supported" } - -Care should be taken to avoid setting the wrong ``state key``:: - - PUT /rooms/!roomid:domain/state/m.another.example.event/11 - { "key" : "with '11' as the state key, but was probably intended to be a txnId" } - -The ``state_key`` is often used to store state about individual users, by using -the user ID as the ``state_key`` value. For example:: - - PUT /rooms/!roomid:domain/state/m.favorite.animal.event/%40my_user%3Aexample.org - { "animal" : "cat", "reason": "fluffy" } - -In some cases, there may be no need for a ``state_key``, so it can be omitted:: - - PUT /rooms/!roomid:domain/state/m.room.bgd.color - { "color": "red", "hex": "#ff0000" } - -{{room_send_cs_http_api}} - - -Redactions -~~~~~~~~~~ -Since events are extensible it is possible for malicious users and/or servers -to add keys that are, for example offensive or illegal. Since some events -cannot be simply deleted, e.g. membership events, we instead 'redact' events. -This involves removing all keys from an event that are not required by the -protocol. This stripped down event is thereafter returned anytime a client or -remote server requests it. Redacting an event cannot be undone, allowing server -owners to delete the offending content from the databases. Events that have been -redacted include a ``redacted_because`` key whose value is the event that caused -it to be redacted, which may include a reason. - - -The exact algorithm to apply against an event is defined in the `room version specification`_. - -The server should add the event causing the redaction to the ``unsigned`` -property of the redacted event, under the ``redacted_because`` key. When a -client receives a redaction event it should change the redacted event in the -same way a server does. - -.. NOTE:: - - Redacted events can still affect the state of the room. When redacted, - state events behave as though their properties were simply not specified, - except those protected by the redaction algorithm. For example, - a redacted ``join`` event will still result in the user being considered joined. - Similarly, a redacted topic does not necessarily cause the topic to revert to - what it was prior to the event - it causes the topic to be removed from the room. - - -Events -++++++ - -{{m_room_redaction_event}} - -Client behaviour -++++++++++++++++ - -{{redaction_cs_http_api}} - -Rooms ------ - -Creation -~~~~~~~~ -The homeserver will create an ``m.room.create`` event when a room is created, -which serves as the root of the event graph for this room. This event also has a -``creator`` key which contains the user ID of the room creator. It will also -generate several other events in order to manage permissions in this room. This -includes: - -- ``m.room.power_levels`` : Sets the power levels of users and required power - levels for various actions within the room such as sending events. -- ``m.room.join_rules`` : Whether the room is "invite-only" or not. - -See `Room Events`_ for more information on these events. To create a room, a -client has to use the following API. - -{{create_room_cs_http_api}} - -Room aliases -~~~~~~~~~~~~ - -Servers may host aliases for rooms with human-friendly names. Aliases take the -form ``#friendlyname:server.name``. - -As room aliases are scoped to a particular homeserver domain name, it is -likely that a homeserver will reject attempts to maintain aliases on other -domain names. This specification does not provide a way for homeservers to -send update requests to other servers. However, homeservers MUST handle -``GET`` requests to resolve aliases on other servers; they should do this using -the federation API if necessary. - -Rooms do not store a list of all aliases present on a room, though members -of the room with relevant permissions may publish preferred aliases through -the ``m.room.canonical_alias`` state event. The aliases in the state event -should point to the room ID they are published within, however room aliases -can and do drift to other room IDs over time. Clients SHOULD NOT treat the -aliases as accurate. They SHOULD be checked before they are used or shared -with another user. If a room appears to have a room alias of ``#alias:example.com``, -this SHOULD be checked to make sure that the room's ID matches the ``room_id`` -returned from the request. - -{{directory_cs_http_api}} - - -Permissions -~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - -Permissions for rooms are done via the concept of power levels - to do any -action in a room a user must have a suitable power level. Power levels are -stored as state events in a given room. The power levels required for operations -and the power levels for users are defined in ``m.room.power_levels``, where -both a default and specific users' power levels can be set. -By default all users have a power level of 0, other than the room creator whose -power level defaults to 100. Users can grant other users increased power levels -up to their own power level. For example, user A with a power level of 50 could -increase the power level of user B to a maximum of level 50. Power levels for -users are tracked per-room even if the user is not present in the room. -The keys contained in ``m.room.power_levels`` determine the levels required for -certain operations such as kicking, banning and sending state events. See -`m.room.power_levels`_ for more information. - -Clients may wish to assign names to particular power levels. A suggested mapping is as follows: -- 0 User -- 50 Moderator -- 100 Admin - - -Room membership -~~~~~~~~~~~~~~~ -Users need to be a member of a room in order to send and receive events in that -room. There are several states in which a user may be, in relation to a room: - -- Unrelated (the user cannot send or receive events in the room) -- Invited (the user has been invited to participate in the room, but is not - yet participating) -- Joined (the user can send and receive events in the room) -- Banned (the user is not allowed to join the room) - -There is an exception to the requirement that a user join a room before sending -events to it: users may send an ``m.room.member`` event to a room with -``content.membership`` set to ``leave`` to reject an invitation if they have -currently been invited to a room but have not joined it. - -Some rooms require that users be invited to it before they can join; others -allow anyone to join. Whether a given room is an "invite-only" room is -determined by the room config key ``m.room.join_rules``. It can have one of the -following values: - -``public`` - This room is free for anyone to join without an invite. - -``invite`` - This room can only be joined if you were invited. - -The allowable state transitions of membership are:: - - /ban - +------------------------------------------------------+ - | | - | +----------------+ +----------------+ | - | | /leave | | | | - | | v v | | - /invite +--------+ +-------+ | | - ------------>| invite |<----------| leave |----+ | | - +--------+ /invite +-------+ | | | - | | ^ | | | - | | | | | | - /join | +---------------+ | | | | - | | /join if | | | | - | | join_rules | | /ban | /unban | - | | public /leave | | | | - v v or | | | | - +------+ /kick | | | | - ------------>| join |-------------------+ | | | - /join +------+ v | | - if | +-----+ | | - join_rules +-------------------------->| ban |-----+ | - public /ban +-----+ | - ^ ^ | - | | | - ----------------------------------------------+ +----------------------+ - /ban - -{{list_joined_rooms_cs_http_api}} - -Joining rooms -+++++++++++++ - -{{inviting_cs_http_api}} - -{{joining_cs_http_api}} - -Leaving rooms -+++++++++++++ -A user can leave a room to stop receiving events for that room. A user must -have been invited to or have joined the room before they are eligible to leave -the room. Leaving a room to which the user has been invited rejects the invite. -Once a user leaves a room, it will no longer appear in the response to the -|/sync|_ API unless it is explicitly requested via a filter with the -``include_leave`` field set to ``true``. - -Whether or not they actually joined the room, if the room is an "invite-only" -room the user will need to be re-invited before they can re-join the room. - -A user can also forget a room which they have left. Rooms which have been -forgotten will never appear the response to the |/sync|_ API, until the user -re-joins or is re-invited. - -A user may wish to force another user to leave a room. This can be done by -'kicking' the other user. To do so, the user performing the kick MUST have the -required power level. Once a user has been kicked, the behaviour is the same as -if they had left of their own accord. In particular, the user is free to -re-join if the room is not "invite-only". - -{{leaving_cs_http_api}} - -{{kicking_cs_http_api}} - -Banning users in a room -+++++++++++++++++++++++ -A user may decide to ban another user in a room. 'Banning' forces the target -user to leave the room and prevents them from re-joining the room. A banned -user will not be treated as a joined user, and so will not be able to send or -receive events in the room. In order to ban someone, the user performing the -ban MUST have the required power level. To ban a user, a request should be made -to |/rooms//ban|_ with:: - - { - "user_id": "" - "reason": "string: " - } - -Banning a user adjusts the banned member's membership state to ``ban``. -Like with other membership changes, a user can directly adjust the target -member's state, by making a request to -``/rooms//state/m.room.member/``:: - - { - "membership": "ban" - } - -A user must be explicitly unbanned with a request to |/rooms//unban|_ -before they can re-join the room or be re-invited. - -{{banning_cs_http_api}} - - - -Listing rooms -~~~~~~~~~~~~~ - -{{list_public_rooms_cs_http_api}} - - -User Data ---------- - -User Directory -~~~~~~~~~~~~~~ - -{{users_cs_http_api}} - - -Profiles -~~~~~~~~ - -{{profile_cs_http_api}} - -Events on Change of Profile Information -+++++++++++++++++++++++++++++++++++++++ -Because the profile display name and avatar information are likely to be used in -many places of a client's display, changes to these fields cause an automatic -propagation event to occur, informing likely-interested parties of the new -values. This change is conveyed using two separate mechanisms: - -- an ``m.room.member`` event (with a ``join`` membership) is sent to every room - the user is a member of, to update the ``displayname`` and ``avatar_url``. -- an ``m.presence`` presence status update is sent, again containing the new - values of the ``displayname`` and ``avatar_url`` keys, in addition to the - required ``presence`` key containing the current presence state of the user. - -Both of these should be done automatically by the homeserver when a user -successfully changes their display name or avatar URL fields. - -Additionally, when homeservers emit room membership events for their own -users, they should include the display name and avatar URL fields in these -events so that clients already have these details to hand, and do not have to -perform extra round trips to query it. - -Security --------- - -Rate limiting -~~~~~~~~~~~~~ -Homeservers SHOULD implement rate limiting to reduce the risk of being -overloaded. If a request is refused due to rate limiting, it should return a -standard error response of the form:: - - { - "errcode": "M_LIMIT_EXCEEDED", - "error": "string", - "retry_after_ms": integer (optional) - } - -The ``retry_after_ms`` key SHOULD be included to tell the client how long they -have to wait in milliseconds before they can try again. - -.. TODO-spec - - Surely we should recommend an algorithm for the rate limiting, rather than letting every - homeserver come up with their own idea, causing totally unpredictable performance over - federated rooms? - -.. References - -.. _`macaroon`: http://research.google.com/pubs/pub41892.html -.. _`devices`: ../index.html#devices - -.. Links through the external API docs are below -.. ============================================= - -.. |/initialSync| replace:: ``/initialSync`` -.. _/initialSync: #get-matrix-client-%CLIENT_MAJOR_VERSION%-initialsync - -.. |/sync| replace:: ``/sync`` -.. _/sync: #get-matrix-client-%CLIENT_MAJOR_VERSION%-sync - -.. |/events| replace:: ``/events`` -.. _/events: #get-matrix-client-%CLIENT_MAJOR_VERSION%-events - -.. |/createRoom| replace:: ``/createRoom`` -.. _/createRoom: #post-matrix-client-%CLIENT_MAJOR_VERSION%-createroom - -.. |/rooms//initialSync| replace:: ``/rooms//initialSync`` -.. _/rooms//initialSync: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-initialsync - -.. |/rooms//messages| replace:: ``/rooms//messages`` -.. _/rooms//messages: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-messages - -.. |/rooms//members| replace:: ``/rooms//members`` -.. _/rooms//members: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-members - -.. |/rooms//state| replace:: ``/rooms//state`` -.. _/rooms//state: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-state - -.. |/rooms//send| replace:: ``/rooms//send`` -.. _/rooms//send: #put-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-send-eventtype-txnid - -.. |/rooms//invite| replace:: ``/rooms//invite`` -.. _/rooms//invite: #post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-invite - -.. |/rooms//join| replace:: ``/rooms//join`` -.. _/rooms//join: #post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-join - -.. |/rooms//leave| replace:: ``/rooms//leave`` -.. _/rooms//leave: #post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-leave - -.. |/rooms//ban| replace:: ``/rooms//ban`` -.. _/rooms//ban: #post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-ban - -.. |/rooms//unban| replace:: ``/rooms//unban`` -.. _/rooms//unban: #post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-unban - -.. |/rooms/{roomId}/context/{eventId}| replace:: ``/rooms/{roomId}/context/{eventId}`` -.. _/rooms/{roomId}/context/{eventId}: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-context-eventid - -.. |/rooms/{roomId}/event/{eventId}| replace:: ``/rooms/{roomId}/event/{eventId}`` -.. _/rooms/{roomId}/event/{eventId}: #get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-event-eventid - -.. |/account/3pid| replace:: ``/account/3pid`` -.. _/account/3pid: #post-matrix-client-%CLIENT_MAJOR_VERSION%-account-3pid - -.. |/user//account_data/| replace:: ``/user//account_data/`` -.. _/user//account_data/: #put-matrix-client-%CLIENT_MAJOR_VERSION%-user-userid-account-data-type - -.. |/_matrix/client/versions| replace:: ``/_matrix/client/versions`` -.. _/_matrix/client/versions: #get-matrix-client-versions - -.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 -.. _`3PID Types`: ../appendices.html#pid-types diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst deleted file mode 100644 index 8a3caf87..00000000 --- a/specification/feature_profiles.rst +++ /dev/null @@ -1,149 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Feature Profiles -================ - -.. _sect:feature-profiles: - -Matrix supports many different kinds of clients: from embedded IoT devices to -desktop clients. Not all clients can provide the same feature sets as other -clients e.g. due to lack of physical hardware such as not having a screen. -Clients can fall into one of several profiles and each profile contains a set -of features that the client MUST support. This section details a set of -"feature profiles". Clients are expected to implement a profile in its entirety -in order for it to be classified as that profile. - -Summary -------- - -===================================== ========== ========== ========== ========== ========== - Module / Profile Web Mobile Desktop CLI Embedded -===================================== ========== ========== ========== ========== ========== - `Instant Messaging`_ Required Required Required Required Optional - `Direct Messaging`_ Required Required Required Required Optional - `Mentions`_ Required Required Required Optional Optional - `Presence`_ Required Required Required Required Optional - `Push Notifications`_ Optional Required Optional Optional Optional - `Receipts`_ Required Required Required Required Optional - `Fully read markers`_ Optional Optional Optional Optional Optional - `Typing Notifications`_ Required Required Required Required Optional - `VoIP`_ Required Required Required Optional Optional - `Ignoring Users`_ Required Required Required Optional Optional - `Reporting Content`_ Optional Optional Optional Optional Optional - `Content Repository`_ Required Required Required Optional Optional - `Managing History Visibility`_ Required Required Required Required Optional - `Server Side Search`_ Optional Optional Optional Optional Optional - `Room Upgrades`_ Required Required Required Required Optional - `Server Administration`_ Optional Optional Optional Optional Optional - `Event Context`_ Optional Optional Optional Optional Optional - `Third Party Networks`_ Optional Optional Optional Optional Optional - `Send-to-Device Messaging`_ Optional Optional Optional Optional Optional - `Device Management`_ Optional Optional Optional Optional Optional - `End-to-End Encryption`_ Optional Optional Optional Optional Optional - `Guest Accounts`_ Optional Optional Optional Optional Optional - `Room Previews`_ Optional Optional Optional Optional Optional - `Client Config`_ Optional Optional Optional Optional Optional - `SSO Login`_ Optional Optional Optional Optional Optional - `OpenID`_ Optional Optional Optional Optional Optional - `Stickers`_ Optional Optional Optional Optional Optional - `Server ACLs`_ Optional Optional Optional Optional Optional - `Server Notices`_ Optional Optional Optional Optional Optional - `Moderation policies`_ Optional Optional Optional Optional Optional -===================================== ========== ========== ========== ========== ========== - -*Please see each module for more details on what clients need to implement.* - -.. _Instant Messaging: `module:im`_ -.. _Direct Messaging: `module:dm`_ -.. _Mentions: `module:mentions`_ -.. _Presence: `module:presence`_ -.. _Push Notifications: `module:push`_ -.. _Receipts: `module:receipts`_ -.. _Fully read markers: `module:read-markers`_ -.. _Typing Notifications: `module:typing`_ -.. _VoIP: `module:voip`_ -.. _Ignoring Users: `module:ignore_users`_ -.. _Reporting Content: `module:report_content`_ -.. _Content Repository: `module:content`_ -.. _Managing History Visibility: `module:history-visibility`_ -.. _Server Side Search: `module:search`_ -.. _Room Upgrades: `module:room-upgrades`_ -.. _Server Administration: `module:admin`_ -.. _Event Context: `module:event-context`_ -.. _Third Party Networks: `module:third-party-networks`_ -.. _Send-to-Device Messaging: `module:to_device`_ -.. _Device Management: `module:device-management`_ -.. _End-to-End Encryption: `module:e2e`_ -.. _Guest Accounts: `module:guest-access`_ -.. _Room Previews: `module:room-previews`_ -.. _Client Config: `module:account_data`_ -.. _SSO Login: `module:sso_login`_ -.. _OpenID: `module:openid`_ -.. _Stickers: `module:stickers`_ -.. _Server ACLs: `module:server-acls`_ -.. Server Notices already has a link elsewhere. -.. _Moderation Policies: `module:moderation-policies`_ - -Clients -------- - -Stand-alone web (``Web``) -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is a web page which heavily uses Matrix for communication. Single-page web -apps would be classified as a stand-alone web client, as would multi-page web -apps which use Matrix on nearly every page. - -Mobile (``Mobile``) -~~~~~~~~~~~~~~~~~~~ - -This is a Matrix client specifically designed for consumption on mobile devices. -This is typically a mobile app but need not be so provided the feature set can -be reached (e.g. if a mobile site could display push notifications it could be -classified as a mobile client). - -Desktop (``Desktop``) -~~~~~~~~~~~~~~~~~~~~~ - -This is a native GUI application which can run in its own environment outside a -browser. - -Command Line Interface (``CLI``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is a client which is used via a text-based terminal. - -Embedded (``Embedded``) -~~~~~~~~~~~~~~~~~~~~~~~ - -This is a client which is embedded into another application or an embedded -device. - -Application -+++++++++++ - -This is a Matrix client which is embedded in another website, e.g. using -iframes. These embedded clients are typically for a single purpose -related to the website in question, and are not intended to be fully-fledged -communication apps. - -Device -++++++ - -This is a client which is typically running on an embedded device such as a -kettle, fridge or car. These clients tend to perform a few operations and run -in a resource constrained environment. Like embedded applications, they are -not intended to be fully-fledged communication systems. diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst deleted file mode 100644 index b2446381..00000000 --- a/specification/identity_service_api.rst +++ /dev/null @@ -1,499 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2017 Kamax.io -.. Copyright 2017 New Vector Ltd -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Identity Service API -==================== - -{{unstable_warning_block_IDENTITY_RELEASE_LABEL}} - -The Matrix client-server and server-server APIs are largely expressed in Matrix -user identifiers. From time to time, it is useful to refer to users by other -("third-party") identifiers, or "3PID"s, e.g. their email address or phone -number. This Identity Service Specification describes how mappings between -third-party identifiers and Matrix user identifiers can be established, -validated, and used. This description technically may apply to any 3PID, but in -practice has only been applied specifically to email addresses and phone numbers. - -.. contents:: Table of Contents -.. sectnum:: - -Changelog ---------- - -.. topic:: Version: %IDENTITY_RELEASE_LABEL% -{{identity_service_changelog}} - -This version of the specification is generated from -`matrix-doc `_ as of Git commit -`{{git_version}} `_. - -For the full historical changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/changelogs/identity_service.rst - - -Other versions of this specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following other versions are also available, in reverse chronological order: - -- `HEAD `_: Includes all changes since the latest versioned release. -- `r0.3.0 `_ -- `r0.2.1 `_ -- `r0.2.0 `_ -- `r0.1.0 `_ - -General principles ------------------- - -The purpose of an identity server is to validate, store, and answer questions -about the identities of users. In particular, it stores associations of the form -"identifier X represents the same user as identifier Y", where identities may -exist on different systems (such as email addresses, phone numbers, -Matrix user IDs, etc). - -The identity server has some private-public keypairs. When asked about an -association, it will sign details of the association with its private key. -Clients may validate the assertions about associations by verifying the signature -with the public key of the identity server. - -In general, identity servers are treated as reliable oracles. They do not -necessarily provide evidence that they have validated associations, but claim to -have done so. Establishing the trustworthiness of an individual identity server -is left as an exercise for the client. - -3PID types are described in `3PID Types`_ Appendix. - -API standards -------------- - -The mandatory baseline for identity server communication in Matrix is exchanging -JSON objects over HTTP APIs. HTTPS is required for communication, and all API calls -use a Content-Type of ``application/json``. In addition, strings MUST be encoded as -UTF-8. - -Any errors which occur at the Matrix API level MUST return a "standard error response". -This is a JSON object which looks like: - -.. code:: json - - { - "errcode": "", - "error": "" - } - -The ``error`` string will be a human-readable error message, usually a sentence -explaining what went wrong. The ``errcode`` string will be a unique string -which can be used to handle an error message e.g. ``M_FORBIDDEN``. There may be -additional keys depending on the error, but the keys ``error`` and ``errcode`` -MUST always be present. - -Some standard error codes are below: - -:``M_NOT_FOUND``: - The resource requested could not be located. - -:``M_MISSING_PARAMS``: - The request was missing one or more parameters. - -:``M_INVALID_PARAM``: - The request contained one or more invalid parameters. - -:``M_SESSION_NOT_VALIDATED``: - The session has not been validated. - -:``M_NO_VALID_SESSION``: - A session could not be located for the given parameters. - -:``M_SESSION_EXPIRED``: - The session has expired and must be renewed. - -:``M_INVALID_EMAIL``: - The email address provided was not valid. - -:``M_EMAIL_SEND_ERROR``: - There was an error sending an email. Typically seen when attempting to verify - ownership of a given email address. - -:``M_INVALID_ADDRESS``: - The provided third party address was not valid. - -:``M_SEND_ERROR``: - There was an error sending a notification. Typically seen when attempting to - verify ownership of a given third party address. - -:``M_UNRECOGNIZED``: - The request contained an unrecognised value, such as an unknown token or medium. - -:``M_THREEPID_IN_USE``: - The third party identifier is already in use by another user. Typically this - error will have an additional ``mxid`` property to indicate who owns the - third party identifier. - -:``M_UNKNOWN``: - An unknown error has occurred. - -Privacy -------- - -Identity is a privacy-sensitive issue. While the identity server exists to -provide identity information, access should be restricted to avoid leaking -potentially sensitive data. In particular, being able to construct large-scale -connections between identities should be avoided. To this end, in general APIs -should allow a 3PID to be mapped to a Matrix user identity, but not in the other -direction (i.e. one should not be able to get all 3PIDs associated with a Matrix -user ID, or get all 3PIDs associated with a 3PID). - -Version 1 API deprecation -------------------------- - -.. TODO: Remove this section when the v1 API is removed. - -As described on each of the version 1 endpoints, the v1 API is deprecated in -favour of the v2 API described here. The major difference, with the exception -of a few isolated cases, is that the v2 API requires authentication to ensure -the user has given permission for the identity server to operate on their data. - -The v1 API is planned to be removed from the specification in a future version. - -Clients SHOULD attempt the v2 endpoints first, and if they receive a ``404``, -``400``, or similar error they should try the v1 endpoint or fail the operation. -Clients are strongly encouraged to warn the user of the risks in using the v1 API, -if they are planning on using it. - -Web browser clients -------------------- - -It is realistic to expect that some clients will be written to be run within a web -browser or similar environment. In these cases, the identity server should respond to -pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers on all -requests. - -When a client approaches the server with a pre-flight (OPTIONS) request, the server -should respond with the CORS headers for that route. The recommended CORS headers -to be returned by servers on all requests are:: - - Access-Control-Allow-Origin: * - Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS - Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization - -Authentication --------------- - -Most ``v2`` endpoints in the Identity Service API require authentication in order -to ensure that the requesting user has accepted all relevant policies and is otherwise -permitted to make the request. The ``v1`` API (currently deprecated) does not require -this authentication, however using ``v1`` is strongly discouraged as it will be removed -in a future release. - -Identity Servers use a scheme similar to the Client-Server API's concept of access -tokens to authenticate users. The access tokens provided by an Identity Server cannot -be used to authenticate Client-Server API requests. - -An access token is provided to an endpoint in one of two ways: - -1. Via a query string parameter, ``access_token=TheTokenHere``. -2. Via a request header, ``Authorization: Bearer TheTokenHere``. - -Clients are encouraged to the use the ``Authorization`` header where possible to prevent -the access token being leaked in access/HTTP logs. The query string should only be used -in cases where the ``Authorization`` header is inaccessible for the client. - -When credentials are required but missing or invalid, the HTTP call will return with a -status of 401 and the error code ``M_UNAUTHORIZED``. - -{{v2_auth_is_http_api}} - - -.. _`agree to more terms`: - -Terms of service ----------------- - -Identity Servers are encouraged to have terms of service (or similar policies) to -ensure that users have agreed to their data being processed by the server. To facilitate -this, an identity server can respond to almost any authenticated API endpoint with an -HTTP 403 and the error code ``M_TERMS_NOT_SIGNED``. The error code is used to indicate -that the user must accept new terms of service before being able to continue. - -All endpoints which support authentication can return the ``M_TERMS_NOT_SIGNED`` error. -When clients receive the error, they are expected to make a call to ``GET /terms`` to -find out what terms the server offers. The client compares this to the ``m.accepted_terms`` -account data for the user (described later) and presents the user with option to accept -the still-missing terms of service. After the user has made their selection, if applicable, -the client sends a request to ``POST /terms`` to indicate the user's acceptance. The -server cannot expect that the client will send acceptance for all pending terms, and the -client should not expect that the server will not respond with another ``M_TERMS_NOT_SIGNED`` -on their next request. The terms the user has just accepted are appended to ``m.accepted_terms``. - -{{m_accepted_terms_event}} - -{{v2_terms_is_http_api}} - - -Status check ------------- - -{{ping_is_http_api}} - -{{v2_ping_is_http_api}} - -Key management --------------- - -An identity server has some long-term public-private keypairs. These are named -in a scheme ``algorithm:identifier``, e.g. ``ed25519:0``. When signing an -association, the standard `Signing JSON`_ algorithm applies. - -.. TODO: Actually allow identity servers to revoke all keys - See: https://github.com/matrix-org/matrix-doc/issues/1633 -.. In the event of key compromise, the identity server may revoke any of its keys. - An HTTP API is offered to get public keys, and check whether a particular key is - valid. - -The identity server may also keep track of some short-term public-private -keypairs, which may have different usage and lifetime characteristics than the -service's long-term keys. - -{{pubkey_is_http_api}} - -{{v2_pubkey_is_http_api}} - -Association lookup ------------------- - -{{lookup_is_http_api}} - -{{v2_lookup_is_http_api}} - -Client behaviour -~~~~~~~~~~~~~~~~ - -.. TODO: Remove this note when v1 is removed completely -.. Note:: - This section only covers the v2 lookup endpoint. The v1 endpoint is described - in isolation above. - -Prior to performing a lookup clients SHOULD make a request to the ``/hash_details`` -endpoint to determine what algorithms the server supports (described in more detail -below). The client then uses this information to form a ``/lookup`` request and -receive known bindings from the server. - -Clients MUST support at least the ``sha256`` algorithm. - -Server behaviour -~~~~~~~~~~~~~~~~ - -.. TODO: Remove this note when v1 is removed completely -.. Note:: - This section only covers the v2 lookup endpoint. The v1 endpoint is described - in isolation above. - -Servers, upon receipt of a ``/lookup`` request, will compare the query against -known bindings it has, hashing the identifiers it knows about as needed to -verify exact matches to the request. - -Servers MUST support at least the ``sha256`` algorithm. - -Algorithms -~~~~~~~~~~ - -Some algorithms are defined as part of the specification, however other formats -can be negotiated between the client and server using ``/hash_details``. - -``sha256`` -++++++++++ - -This algorithm MUST be supported by clients and servers at a minimum. It is -additionally the preferred algorithm for lookups. - -When using this algorithm, the client converts the query first into strings -separated by spaces in the format ``
``. The ```` -is retrieved from ``/hash_details``, the ```` is typically ``email`` or -``msisdn`` (both lowercase), and the ``
`` is the 3PID to search for. -For example, if the client wanted to know about ``alice@example.org``'s bindings, -it would first format the query as ``alice@example.org email ThePepperGoesHere``. - -.. admonition:: Rationale - - Mediums and peppers are appended to the address to prevent a common prefix - for each 3PID, helping prevent attackers from pre-computing the internal state - of the hash function. - -After formatting each query, the string is run through SHA-256 as defined by -`RFC 4634 `_. The resulting bytes are then -encoded using URL-Safe `Unpadded Base64`_ (similar to `room version 4's -event ID format <../rooms/v4.html#event-ids>`_). - -An example set of queries when using the pepper ``matrixrocks`` would be:: - - "alice@example.com email matrixrocks" -> "4kenr7N9drpCJ4AfalmlGQVsOn3o2RHjkADUpXJWZUc" - "bob@example.com email matrixrocks" -> "LJwSazmv46n0hlMlsb_iYxI0_HXEqy_yj6Jm636cdT8" - "18005552067 msisdn matrixrocks" -> "nlo35_T5fzSGZzJApqu8lgIudJvmOQtDaHtr-I4rU7I" - - -The set of hashes is then given as the ``addresses`` array in ``/lookup``. Note -that the pepper used MUST be supplied as ``pepper`` in the ``/lookup`` request. - -``none`` -++++++++ - -This algorithm performs plaintext lookups on the identity server. Typically this -algorithm should not be used due to the security concerns of unhashed identifiers, -however some scenarios (such as LDAP-backed identity servers) prevent the use of -hashed identifiers. Identity servers (and optionally clients) can use this algorithm -to perform those kinds of lookups. - -Similar to the ``sha256`` algorithm, the client converts the queries into strings -separated by spaces in the format ``
`` - note the lack of ````. -For example, if the client wanted to know about ``alice@example.org``'s bindings, -it would format the query as ``alice@example.org email``. - -The formatted strings are then given as the ``addresses`` in ``/lookup``. Note that -the ``pepper`` is still required, and must be provided to ensure the client has made -an appropriate request to ``/hash_details`` first. - -Security considerations -~~~~~~~~~~~~~~~~~~~~~~~ - -.. Note:: - `MSC2134 `_ has much more - information about the security considerations made for this section of the - specification. This section covers the high-level details for why the specification - is the way it is. - -Typically the lookup endpoint is used when a client has an unknown 3PID it wants to -find a Matrix User ID for. Clients normally do this kind of lookup when inviting new -users to a room or searching a user's address book to find any Matrix users they may -not have discovered yet. Rogue or malicious identity servers could harvest this -unknown information and do nefarious things with it if it were sent in plain text. -In order to protect the privacy of users who might not have a Matrix identifier bound -to their 3PID addresses, the specification attempts to make it difficult to harvest -3PIDs. - -.. admonition:: Rationale - - Hashing identifiers, while not perfect, helps make the effort required to harvest - identifiers significantly higher. Phone numbers in particular are still difficult - to protect with hashing, however hashing is objectively better than not. - - An alternative to hashing would be using bcrypt or similar with many rounds, however - by nature of needing to serve mobile clients and clients on limited hardware the - solution needs be kept relatively lightweight. - -Clients should be cautious of servers not rotating their pepper very often, and -potentially of servers which use a weak pepper - these servers may be attempting to -brute force the identifiers or use rainbow tables to mine the addresses. Similarly, -clients which support the ``none`` algorithm should consider at least warning the user -of the risks in sending identifiers in plain text to the identity server. - -Addresses are still potentially reversable using a calculated rainbow table given -some identifiers, such as phone numbers, common email address domains, and leaked -addresses are easily calculated. For example, phone numbers can have roughly 12 -digits to them, making them an easier target for attack than email addresses. - - -Establishing associations -------------------------- - -The flow for creating an association is session-based. - -Within a session, one may prove that one has ownership of a 3PID. -Once this has been established, the user can form an association between that -3PID and a Matrix user ID. Note that this association is only proved one way; -a user can associate *any* Matrix user ID with a validated 3PID, -i.e. I can claim that any email address I own is associated with -@billg:microsoft.com. - -Sessions are time-limited; a session is considered to have been modified when -it was created, and then when a validation is performed within it. A session can -only be checked for validation, and validation can only be performed within a -session, within a 24-hour period since its most recent modification. Any -attempts to perform these actions after the expiry will be rejected, and a new -session should be created and used instead. - -To start a session, the client makes a request to the appropriate -``/requestToken`` endpoint. The identity server then sends a validation token -to the user, and the user provides the token to the client. The client then -provides the token to the appropriate ``/submitToken`` endpoint, completing the -session. At this point, the client should ``/bind`` the third party identifier -or leave it for another entity to bind. - -Format of a validation token -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The format of the validation token is left up to the identity server: it -should choose one appropriate to the 3PID type. (For example, it would be -inappropriate to expect a user to copy a long passphrase including punctuation -from an SMS message into a client.) - -Whatever format the identity server uses, the validation token must consist of -at most 255 Unicode codepoints. Clients must pass the token through without -modification. - -Email associations -~~~~~~~~~~~~~~~~~~ - -{{email_associations_is_http_api}} - -{{v2_email_associations_is_http_api}} - -Phone number associations -~~~~~~~~~~~~~~~~~~~~~~~~~ - -{{phone_associations_is_http_api}} - -{{v2_phone_associations_is_http_api}} - -General -~~~~~~~ - -{{associations_is_http_api}} - -{{v2_associations_is_http_api}} - -Invitation storage ------------------- - -An identity server can store pending invitations to a user's 3PID, which will -be retrieved and can be either notified on or look up when the 3PID is -associated with a Matrix user ID. - -At a later point, if the owner of that particular 3PID binds it with a Matrix user -ID, the identity server will attempt to make an HTTP POST to the Matrix user's -homeserver via the `/3pid/onbind`_ endpoint. The request MUST be signed with a -long-term private key for the identity server. - -{{store_invite_is_http_api}} - -{{v2_store_invite_is_http_api}} - -Ephemeral invitation signing ----------------------------- - -To aid clients who may not be able to perform crypto themselves, the identity -server offers some crypto functionality to help in accepting invitations. -This is less secure than the client doing it itself, but may be useful where -this isn't possible. - -{{invitation_signing_is_http_api}} - -{{v2_invitation_signing_is_http_api}} - -.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 -.. _`3PID Types`: ../appendices.html#pid-types -.. _`Signing JSON`: ../appendices.html#signing-json -.. _`/3pid/onbind`: ../server_server/%SERVER_RELEASE_LABEL%.html#put-matrix-federation-v1-3pid-onbind diff --git a/specification/index.rst b/specification/index.rst deleted file mode 100644 index 276ee8b2..00000000 --- a/specification/index.rst +++ /dev/null @@ -1,575 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Matrix Specification -==================== - -.. Note that this file is specifically unversioned because we don't want to -.. have to add Yet Another version number, and the commentary on what specs we -.. have should hopefully not get complex enough that we need to worry about -.. versioning it. - -Matrix defines a set of open APIs for decentralised communication, suitable for -securely publishing, persisting and subscribing to data over a global open -federation of servers with no single point of control. Uses include Instant Messaging (IM), -Voice over IP (VoIP) signalling, Internet of Things (IoT) communication, and bridging -together existing communication silos - providing the basis of a new open real-time -communication ecosystem. - -To propose a change to the Matrix Spec, see the explanations at -`Proposals for Spec Changes to Matrix `_. - -.. contents:: Table of Contents -.. sectnum:: - -Matrix APIs ------------ - -The specification consists of the following parts: - -{{apis}} - -Additionally, this introduction page contains the key baseline information required to -understand the specific APIs, including the sections on `room versions`_ -and `overall architecture <#architecture>`_. - -The `Appendices `_ contain supplemental information not specific to -one of the above APIs. - -The `Matrix Client-Server API Swagger Viewer `_ -is useful for browsing the Client-Server API. - - -Matrix versions -~~~~~~~~~~~~~~~ - -.. Note:: - As of June 10th 2019, the Matrix specification is considered out of beta - - indicating that all currently released APIs are considered stable and secure - to the best of our knowledge, and the spec should contain the complete - information necessary to develop production-grade implementations of Matrix - without the need for external reference. - -Matrix 1.0 (released June 10th, 2019) consists of the following minimum API -versions: - -======================= ======= -API/Specification Version -======================= ======= -Client-Server API r0.5.0 -Server-Server API r0.1.2 -Application Service API r0.1.1 -Identity Service API r0.1.1 -Push Gateway API r0.1.0 -Room Version v5 -======================= ======= - - -Introduction to the Matrix APIs -------------------------------- - -Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice -over IP (VoIP) and Internet of Things (IoT) communication, designed to create -and support a new global real-time communication ecosystem. The intention is to -provide an open decentralised pubsub layer for the internet for securely -persisting and publishing/subscribing JSON objects. This specification is the -ongoing result of standardising the APIs used by the various components of the -Matrix ecosystem to communicate with one another. - -The principles that Matrix attempts to follow are: - -- Pragmatic Web-friendly APIs (i.e. JSON over REST) -- Keep It Simple & Stupid - - + provide a simple architecture with minimal third-party dependencies. - -- Fully open: - - + Fully open federation - anyone should be able to participate in the global - Matrix network - + Fully open standard - publicly documented standard with no IP or patent - licensing encumbrances - + Fully open source reference implementation - liberally-licensed example - implementations with no IP or patent licensing encumbrances - -- Empowering the end-user - - + The user should be able to choose the server and clients they use - + The user should be able to control how private their communication is - + The user should know precisely where their data is stored - -- Fully decentralised - no single points of control over conversations or the - network as a whole -- Learning from history to avoid repeating it - - + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP - whilst trying to avoid their failings - - -The functionality that Matrix provides includes: - -- Creation and management of fully distributed chat rooms with no - single points of control or failure -- Eventually-consistent cryptographically secure synchronisation of room - state across a global open network of federated servers and services -- Sending and receiving extensible messages in a room with (optional) - end-to-end encryption -- Extensible user management (inviting, joining, leaving, kicking, banning) - mediated by a power-level based user privilege system. -- Extensible room state management (room naming, aliasing, topics, bans) -- Extensible user profile management (avatars, display names, etc) -- Managing user accounts (registration, login, logout) -- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, - Facebook accounts to authenticate, identify and discover users on Matrix. -- Trusted federation of identity servers for: - - + Publishing user public keys for PKI - + Mapping of 3PIDs to Matrix IDs - - -The end goal of Matrix is to be a ubiquitous messaging layer for synchronising -arbitrary data between sets of people, devices and services - be that for -instant messages, VoIP call setups, or any other objects that need to be -reliably and persistently pushed from A to B in an interoperable and federated -manner. - - -Spec Change Proposals -~~~~~~~~~~~~~~~~~~~~~ -To propose a change to the Matrix Spec, see the explanations at `Proposals -for Spec Changes to Matrix `_. - - -.. _`architecture`: - -Architecture ------------- - -Matrix defines APIs for synchronising extensible JSON objects known as -"events" between compatible clients, servers and services. Clients are -typically messaging/VoIP applications or IoT devices/hubs and communicate by -synchronising communication history with their "homeserver" using the -"Client-Server API". Each homeserver stores the communication history and -account information for all of its clients, and shares data with the wider -Matrix ecosystem by synchronising communication history with other homeservers -and their clients. - -Clients typically communicate with each other by emitting events in the -context of a virtual "room". Room data is replicated across *all of the -homeservers* whose users are participating in a given room. As such, *no -single homeserver has control or ownership over a given room*. Homeservers -model communication history as a partially ordered graph of events known as -the room's "event graph", which is synchronised with eventual consistency -between the participating servers using the "Server-Server API". This process -of synchronising shared conversation history between homeservers run by -different parties is called "Federation". Matrix optimises for the -Availability and Partitioned properties of CAP theorem at -the expense of Consistency. - -For example, for client A to send a message to client B, client A performs an -HTTP PUT of the required JSON event on its homeserver (HS) using the -client-server API. A's HS appends this event to its copy of the room's event -graph, signing the message in the context of the graph for integrity. A's HS -then replicates the message to B's HS by performing an HTTP PUT using the -server-server API. B's HS authenticates the request, validates the event's -signature, authorises the event's contents and then adds it to its copy of the -room's event graph. Client B then receives the message from his homeserver via -a long-lived GET request. - -:: - - How data flows between clients - ============================== - - { Matrix client A } { Matrix client B } - ^ | ^ | - | events | Client-Server API | events | - | V | V - +------------------+ +------------------+ - | |---------( HTTPS )--------->| | - | homeserver | | homeserver | - | |<--------( HTTPS )----------| | - +------------------+ Server-Server API +------------------+ - History Synchronisation - (Federation) - - -Users -~~~~~ - -Each client is associated with a user account, which is identified in Matrix -using a unique "user ID". This ID is namespaced to the homeserver which -allocated the account and has the form:: - - @localpart:domain - -See `'Identifier Grammar' in the appendices `_ for full details of -the structure of user IDs. - -Devices -~~~~~~~ - -The Matrix specification has a particular meaning for the term "device". As a -user, I might have several devices: a desktop client, some web browsers, an -Android device, an iPhone, etc. They broadly relate to a real device in the -physical world, but you might have several browsers on a physical device, or -several Matrix client applications on a mobile device, each of which would be -its own device. - -Devices are used primarily to manage the keys used for end-to-end encryption -(each device gets its own copy of the decryption keys), but they also help -users manage their access - for instance, by revoking access to particular -devices. - -When a user first uses a client, it registers itself as a new device. The -longevity of devices might depend on the type of client. A web client will -probably drop all of its state on logout, and create a new device every time -you log in, to ensure that cryptography keys are not leaked to a new user. In -a mobile client, it might be acceptable to reuse the device if a login session -expires, provided the user is the same. - -Devices are identified by a ``device_id``, which is unique within the scope of -a given user. - -A user may assign a human-readable display name to a device, to help them -manage their devices. - -Events -~~~~~~ - -All data exchanged over Matrix is expressed as an "event". Typically each client -action (e.g. sending a message) correlates with exactly one event. Each event -has a ``type`` which is used to differentiate different kinds of data. ``type`` -values MUST be uniquely globally namespaced following Java's `package naming -conventions`_, e.g. -``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved -for events defined in the Matrix specification - for instance ``m.room.message`` -is the event type for instant messages. Events are usually sent in the context -of a "Room". - -.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions - -Event Graphs -~~~~~~~~~~~~ - -.. _sect:event-graph: - -Events exchanged in the context of a room are stored in a directed acyclic graph -(DAG) called an "event graph". The partial ordering of this graph gives the -chronological ordering of events within the room. Each event in the graph has a -list of zero or more "parent" events, which refer to any preceding events -which have no chronological successor from the perspective of the homeserver -which created the event. - -Typically an event has a single parent: the most recent message in the room at -the point it was sent. However, homeservers may legitimately race with each -other when sending messages, resulting in a single event having multiple -successors. The next event added to the graph thus will have multiple parents. -Every event graph has a single root event with no parent. - -To order and ease chronological comparison between the events within the graph, -homeservers maintain a ``depth`` metadata field on each event. An event's -``depth`` is a positive integer that is strictly greater than the depths of any -of its parents. The root event should have a depth of 1. Thus if one event is -before another, then it must have a strictly smaller depth. - -Room structure -~~~~~~~~~~~~~~ - -A room is a conceptual place where users can send and receive events. Events are -sent to a room, and all participants in that room with sufficient access will -receive the event. Rooms are uniquely identified internally via "Room IDs", -which have the form:: - - !opaque_id:domain - -There is exactly one room ID for each room. Whilst the room ID does contain a -domain, it is simply for globally namespacing room IDs. The room does NOT -reside on the domain specified. - -See `'Identifier Grammar' in the appendices `_ for full details of -the structure of a room ID. - -The following conceptual diagram shows an -``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: - - { @alice:matrix.org } { @bob:example.org } - | ^ - | | - [HTTP POST] [HTTP GET] - Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org - Event type: m.room.message Event type: m.room.message - Content: { JSON object } Content: { JSON object } - | | - V | - +------------------+ +------------------+ - | homeserver | | homeserver | - | matrix.org | | example.org | - +------------------+ +------------------+ - | ^ - | [HTTP PUT] | - | Room ID: !qporfwt:matrix.org | - | Event type: m.room.message | - | Content: { JSON object } | - `-------> Pointer to the preceding message ------` - PKI signature from matrix.org - Transaction-layer metadata - PKI Authorization header - - .................................... - | Shared Data | - | State: | - | Room ID: !qporfwt:matrix.org | - | Servers: matrix.org, example.org | - | Members: | - | - @alice:matrix.org | - | - @bob:example.org | - | Messages: | - | - @alice:matrix.org | - | Content: { JSON object } | - |....................................| - -Federation maintains *shared data structures* per-room between multiple -homeservers. The data is split into ``message events`` and ``state events``. - -Message events: - These describe transient 'once-off' activity in a room such as an - instant messages, VoIP call setups, file transfers, etc. They generally - describe communication activity. - -State events: - These describe updates to a given piece of persistent information - ('state') related to a room, such as the room's name, topic, membership, - participating servers, etc. State is modelled as a lookup table of key/value - pairs per room, with each key being a tuple of ``state_key`` and ``event type``. - Each state event updates the value of a given key. - -The state of the room at a given point is calculated by considering all events -preceding and including a given event in the graph. Where events describe the -same state, a merge conflict algorithm is applied. The state resolution -algorithm is transitive and does not depend on server state, as it must -consistently select the same event irrespective of the server or the order the -events were received in. Events are signed by the originating server (the -signature includes the parent relations, type, depth and payload hash) and are -pushed over federation to the participating servers in a room, currently using -full mesh topology. Servers may also request backfill of events over federation -from the other servers participating in a room. - -.. Note:: - Events are not limited to the types defined in this specification. New or custom - event types can be created on a whim using the Java package naming convention. - For example, a ``com.example.game.score`` event can be sent by clients and other - clients would receive it through Matrix, assuming the client has access to the - ``com.example`` namespace. - -Room Aliases -++++++++++++ - -Each room can also have multiple "Room Aliases", which look like:: - - #room_alias:domain - -See `'Identifier Grammar' in the appendices `_ for full details of -the structure of a room alias. - -A room alias "points" to a room ID and is the human-readable label by which -rooms are publicised and discovered. The room ID the alias is pointing to can -be obtained by visiting the domain specified. Note that the mapping from a room -alias to a room ID is not fixed, and may change over time to point to a -different room ID. For this reason, Clients SHOULD resolve the room alias to a -room ID once and then use that ID on subsequent requests. - -When resolving a room alias the server will also respond with a list of servers -that are in the room that can be used to join via. - -:: - - HTTP GET - #matrix:example.org !aaabaa:matrix.org - | ^ - | | - _______V____________________|____ - | example.org | - | Mappings: | - | #matrix >> !aaabaa:matrix.org | - | #golf >> !wfeiofh:sport.com | - | #bike >> !4rguxf:matrix.org | - |________________________________| - -Identity -~~~~~~~~ - -Users in Matrix are identified via their Matrix user ID. However, -existing 3rd party ID namespaces can also be used in order to identify Matrix -users. A Matrix "Identity" describes both the user ID and any other existing IDs -from third party namespaces *linked* to their account. -Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social -network accounts and phone numbers to their user ID. Linking 3PIDs creates a -mapping from a 3PID to a user ID. This mapping can then be used by Matrix -users in order to discover the user IDs of their contacts. -In order to ensure that the mapping from 3PID to user ID is genuine, a globally -federated cluster of trusted "identity servers" (IS) are used to verify the 3PID -and persist and replicate the mappings. - -Usage of an IS is not required in order for a client application to be part of -the Matrix ecosystem. However, without one clients will not be able to look up -user IDs using 3PIDs. - - -Profiles -~~~~~~~~ - -Users may publish arbitrary key/value data associated with their account - such -as a human-readable display name, a profile photo URL, contact information -(email address, phone numbers, website URLs etc). - -.. TODO - Actually specify the different types of data - e.g. what format are display - names allowed to be? - -Private User Data -~~~~~~~~~~~~~~~~~ - -Users may also store arbitrary private key/value data in their account - such as -client preferences, or server configuration settings which lack any other -dedicated API. The API is symmetrical to managing Profile data. - -.. TODO - Would it really be overengineered to use the same API for both profile & - private user data, but with different ACLs? - - -Common concepts ---------------- - -Various things are common throughout all of the Matrix APIs. They are -documented here. - -.. TODO: Some words about trailing slashes. See https://github.com/matrix-org/matrix-doc/issues/2107 - -Namespacing -~~~~~~~~~~~ - -Namespacing helps prevent conflicts between multiple applications and the specification -itself. Where namespacing is used, ``m.`` prefixes are used by the specification to -indicate that the field is controlled by the specification. Custom or non-specified -namespaces used in the wild MUST use the Java package naming convention to prevent -conflicts. - -As an example, event types defined in the specification are namespaced under the -special ``m.`` prefix, however any client can send a custom event type, such as -``com.example.game.score`` (assuming the client has rights to the ``com.example`` -namespace) without needing to put the event into the ``m.`` namespace. - -Timestamps -~~~~~~~~~~ - -Unless otherwise stated, timestamps are measured as milliseconds since the Unix epoch. -Throughout the specification this may be referred to as POSIX, Unix, or just "time in -milliseconds". - - -.. _`room versions`: - -Room Versions -------------- - -Rooms are central to how Matrix operates, and have strict rules for what -is allowed to be contained within them. Rooms can also have various -algorithms that handle different tasks, such as what to do when two or -more events collide in the underlying DAG. To allow rooms to be improved -upon through new algorithms or rules, "room versions" are employed to -manage a set of expectations for each room. New room versions are assigned -as needed. - -There is no implicit ordering or hierarchy to room versions, and their principles -are immutable once placed in the specification. Although there is a recommended -set of versions, some rooms may benefit from features introduced by other versions. -Rooms move between different versions by "upgrading" to the desired version. Due -to versions not being ordered or hierarchical, this means a room can "upgrade" -from version 2 to version 1, if it is so desired. - -Room version grammar -~~~~~~~~~~~~~~~~~~~~ - -Room versions are used to change properties of rooms that may not be compatible -with other servers. For example, changing the rules for event authorization would -cause older servers to potentially end up in a split-brain situation due to not -understanding the new rules. - -A room version is defined as a string of characters which MUST NOT exceed 32 -codepoints in length. Room versions MUST NOT be empty and SHOULD contain only -the characters ``a-z``, ``0-9``, ``.``, and ``-``. - -Room versions are not intended to be parsed and should be treated as opaque -identifiers. Room versions consisting only of the characters ``0-9`` and ``.`` -are reserved for future versions of the Matrix protocol. - -The complete grammar for a legal room version is:: - - room_version = 1*room_version_char - room_version_char = DIGIT - / %x61-7A ; a-z - / "-" / "." - -Examples of valid room versions are: - -* ``1`` (would be reserved by the Matrix protocol) -* ``1.2`` (would be reserved by the Matrix protocol) -* ``1.2-beta`` -* ``com.example.version`` - -Complete list of room versions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Room versions are divided into two distinct groups: stable and unstable. Stable -room versions may be used by rooms safely. Unstable room versions are everything -else which is either not listed in the specification or flagged as unstable for -some other reason. Versions can switch between stable and unstable periodically -for a variety of reasons, including discovered security vulnerabilities and age. - -Clients should not ask room administrators to upgrade their rooms if the room is -running a stable version. Servers SHOULD use room version 6 as the default room -version when creating new rooms. - -The available room versions are: - -* `Version 1 `_ - **Stable**. The current version of most rooms. -* `Version 2 `_ - **Stable**. Implements State Resolution Version 2. -* `Version 3 `_ - **Stable**. Introduces events whose IDs are the event's hash. -* `Version 4 `_ - **Stable**. Builds on v3 by using URL-safe base64 for event IDs. -* `Version 5 `_ - **Stable**. Introduces enforcement of signing key validity periods. -* `Version 6 `_ - **Stable**. Alters several authorization rules for events. - -Specification Versions ----------------------- - -The specification for each API is versioned in the form ``rX.Y.Z``. - * A change to ``X`` reflects a breaking change: a client implemented against - ``r1.0.0`` may need changes to work with a server which supports (only) - ``r2.0.0``. - * A change to ``Y`` represents a change which is backwards-compatible for - existing clients, but not necessarily existing servers: a client implemented - against ``r1.1.0`` will work without changes against a server which supports - ``r1.2.0``; but a client which requires ``r1.2.0`` may not work correctly - with a server which implements only ``r1.1.0``. - * A change to ``Z`` represents a change which is backwards-compatible on both - sides. Typically this implies a clarification to the specification, rather - than a change which must be implemented. - -License -------- - -The Matrix specification is licensed under the `Apache License, Version 2.0 -`_. diff --git a/specification/modules.rst b/specification/modules.rst deleted file mode 100644 index 1bc88445..00000000 --- a/specification/modules.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Modules -======= - -Modules are parts of the Client-Server API which are not universal to all -endpoints. Modules are strictly defined within this specification and -should not be mistaken for experimental extensions or optional features. -A compliant server implementation MUST support all modules and supporting -specification (unless the implementation only targets clients of certain -profiles, in which case only the required modules for those feature profiles -MUST be implemented). A compliant client implementation MUST support all -the required modules and supporting specification for the `Feature Profile <#feature-profiles>`_ -it targets. diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst deleted file mode 100644 index d1fef7f5..00000000 --- a/specification/modules/_template.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Module Heading -============== - -.. NOTE: Prefer to identify-modules-with-dashes despite historical examples. -.. _module:short-name: - -A short summary of the module. What features does this module provide? An anchor -should be specified at the top of the module using the format ``module:name``. - -Complicated modules may wish to have architecture diagrams or event flows -(e.g. VoIP call flows) here. Custom subsections can be included but they should -be used *sparingly* to reduce the risk of putting client or server behaviour -information in these custom sections. - -Events ------- -List the new event types introduced by this module, if any. If there are no -new events, this section can be omitted. Event types should be done as -subsections. This section is intended to document the "common shared event -structure" between client and server. Deviations from this shared structure -should be documented in the relevant behaviour section. - -``m.example.event.type`` -~~~~~~~~~~~~~~~~~~~~~~~~ -There should be JSON Schema docs for this event. Once there is JSON schema, -there will be a template variable with dots in the event type replaced with -underscores and the suffix ``_event``. You can insert a template like so: - -{{m_example_event_type_event}} - -Client behaviour ----------------- -List any new HTTP endpoints. These endpoints should be documented using Swagger. -Once there is Swagger, there will be a template variable based on the name of -the YAML file with the suffix ``_cs_http_api``. You can insert a template for -swagger docs like so: - -{{name-of-yaml-file-without-file-ext_cs_http_api}} - -List the steps the client needs to take to -correctly process this module. List what data structures the client should be -storing in order to aid implementation. - -Server behaviour ----------------- -Does the server need to handle any of the new events in a special way (e.g. -typing timeouts, presence). Advice on how to persist events and/or requests are -recommended to aid implementation. Federation-specific logic should be included -here. - -Security considerations ------------------------ -This includes privacy leaks: for example leaking presence info. How do -misbehaving clients or servers impact this module? This section should always be -included, if only to say "we've thought about it but there isn't anything to do -here". diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst deleted file mode 100644 index a67e503a..00000000 --- a/specification/modules/account_data.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Client Config -============= - -.. _module:account_data: - -Clients can store custom config data for their account on their homeserver. -This account data will be synced between different devices and can persist -across installations on a particular device. Users may only view the account -data for their own account - -The account_data may be either global or scoped to a particular rooms. - -Events ------- - -The client receives the account data as events in the ``account_data`` sections -of a ``/sync``. - -These events can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/sync``. ``m.tag`` -events appearing in ``/events`` will have a ``room_id`` with the room -the tags are for. - -Client Behaviour ----------------- - -{{account_data_cs_http_api}} - - -Server Behaviour ----------------- - -Servers MUST reject clients from setting account data for event types that -the server manages. Currently, this only includes `m.fully_read`_. diff --git a/specification/modules/admin.rst b/specification/modules/admin.rst deleted file mode 100644 index c4465675..00000000 --- a/specification/modules/admin.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Server Administration -===================== - -.. _module:admin: - -This module adds capabilities for server administrators to inspect server state -and data. - -Client Behaviour ----------------- - -{{admin_cs_http_api}} diff --git a/specification/modules/anonymous_access.rst b/specification/modules/anonymous_access.rst deleted file mode 100644 index a6ffbfb6..00000000 --- a/specification/modules/anonymous_access.rst +++ /dev/null @@ -1,64 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Guest access -================ - -.. _module:guest-access: - -It may be desirable to allow users without a fully registered user account to -ephemerally access Matrix rooms. This module specifies limited ways of doing so. - -Note that this is not currently a complete anonymous access solution; in -particular, it only allows servers to provided anonymous access to rooms in -which they are already participating, and relies on individual homeservers to -adhere to the conventions which this module sets, rather than allowing all -participating homeservers to enforce them. - -Events ------- - -{{m_room_guest_accessibility}} - -Client behaviour ----------------- -A client can register for guest access using the FOO endpoint. From that point -on, they can interact with a limited subset of the existing client-server API, -as if they were a fully registered user, using the access token granted to them -by the server. - -These users are only allowed to make calls in relation to rooms which have the -``m.room.history_visibility`` event set to ``world_readable``. - -The APIs they are allowed to hit are: - -/rooms/{roomId}/messages -/rooms/{roomId}/state -/rooms/{roomId}/state/{eventType}/{stateKey} -/events - -Server behaviour ----------------- -Does the server need to handle any of the new events in a special way (e.g. -typing timeouts, presence). Advice on how to persist events and/or requests are -recommended to aid implementation. Federation-specific logic should be included -here. - -Security considerations ------------------------ -This includes privacy leaks: for example leaking presence info. How do -misbehaving clients or servers impact this module? This section should always be -included, if only to say "we've thought about it but there isn't anything to do -here". - diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst deleted file mode 100644 index 192250d2..00000000 --- a/specification/modules/content_repo.rst +++ /dev/null @@ -1,136 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Content repository -================== - -.. _module:content: - -The content repository (or "media repository") allows users to upload -files to their homeserver for later use. For example, files which the -user wants to send to a room would be uploaded here, as would an avatar -the user wants to use. - -Uploads are POSTed to a resource on the user's local homeserver which -returns a MXC URI which can later be used to GET the download. Content -is downloaded from the recipient's local homeserver, which must first -transfer the content from the origin homeserver using the same API -(unless the origin and destination homeservers are the same). - -When serving content, the server SHOULD provide a ``Content-Security-Policy`` -header. The recommended policy is ``sandbox; default-src 'none'; script-src -'none'; plugin-types application/pdf; style-src 'unsafe-inline'; object-src -'self';``. - -Matrix Content (MXC) URIs -------------------------- - -.. _`MXC URI`: - -Content locations are represented as Matrix Content (MXC) URIs. They look -like:: - - mxc:/// - - : The name of the homeserver where this content originated, e.g. matrix.org - : An opaque ID which identifies the content. - - -Client behaviour ----------------- - -Clients can upload and download content using the following HTTP APIs. - -{{content_repo_cs_http_api}} - -Thumbnails -~~~~~~~~~~ -The homeserver SHOULD be able to supply thumbnails for uploaded images and -videos. The exact file types which can be thumbnailed are not currently -specified - see `Issue #1938 `_ -for more information. - -The thumbnail methods are "crop" and "scale". "scale" tries to return an -image where either the width or the height is smaller than the requested -size. The client should then scale and letterbox the image if it needs to -fit within a given rectangle. "crop" tries to return an image where the -width and height are close to the requested size and the aspect matches -the requested size. The client should scale the image if it needs to fit -within a given rectangle. - -The dimensions given to the thumbnail API are the minimum size the client -would prefer. Servers must never return thumbnails smaller than the client's -requested dimensions, unless the content being thumbnailed is smaller than -the dimensions. When the content is smaller than the requested dimensions, -servers should return the original content rather than thumbnail it. - -Servers SHOULD produce thumbnails with the following dimensions and methods: - -* 32x32, crop -* 96x96, crop -* 320x240, scale -* 640x480, scale -* 800x600, scale - -In summary: - * "scale" maintains the original aspect ratio of the image - * "crop" provides an image in the aspect ratio of the sizes given in the request - * The server will return an image larger than or equal to the dimensions requested - where possible. - -Servers MUST NOT upscale thumbnails under any circumstance. Servers MUST NOT -return a smaller thumbnail than requested, unless the original content makes -that impossible. - -Security considerations ------------------------ - -The HTTP GET endpoint does not require any authentication. Knowing the URL of -the content is sufficient to retrieve the content, even if the entity isn't in -the room. - -MXC URIs are vulnerable to directory traversal attacks such as -``mxc://127.0.0.1/../../../some_service/etc/passwd``. This would cause the target -homeserver to try to access and return this file. As such, homeservers MUST -sanitise MXC URIs by allowing only alphanumeric (``A-Za-z0-9``), ``_`` -and ``-`` characters in the ``server-name`` and ``media-id`` values. This set -of whitelisted characters allows URL-safe base64 encodings specified in RFC 4648. -Applying this character whitelist is preferable to blacklisting ``.`` and ``/`` -as there are techniques around blacklisted characters (percent-encoded characters, -UTF-8 encoded traversals, etc). - -Homeservers have additional content-specific concerns: - -- Clients may try to upload very large files. Homeservers should not store files - that are too large and should not serve them to clients, returning a HTTP 413 - error with the ``M_TOO_LARGE`` code. - -- Clients may try to upload very large images. Homeservers should not attempt to - generate thumbnails for images that are too large, returning a HTTP 413 error - with the ``M_TOO_LARGE`` code. - -- Remote homeservers may host very large files or images. Homeservers should not - proxy or thumbnail large files or images from remote homeservers, returning a - HTTP 502 error with the ``M_TOO_LARGE`` code. - -- Clients may try to upload a large number of files. Homeservers should limit the - number and total size of media that can be uploaded by clients, returning a - HTTP 403 error with the ``M_FORBIDDEN`` code. - -- Clients may try to access a large number of remote files through a homeserver. - Homeservers should restrict the number and size of remote files that it caches. - -- Clients or remote homeservers may try to upload malicious files targeting - vulnerabilities in either the homeserver thumbnailing or the client decoders. diff --git a/specification/modules/device_management.rst b/specification/modules/device_management.rst deleted file mode 100644 index 3f9c8452..00000000 --- a/specification/modules/device_management.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Device Management -================= - -.. _module:device-management: - -This module provides a means for a user to manage their `devices`_. - -Client behaviour ----------------- -Clients that implement this module should offer the user a list of registered -devices, as well as the means to update their display names. Clients should -also allow users to delete disused devices. - -{{device_management_cs_http_api}} - -Security considerations ------------------------ - -Deleting devices has security implications: it invalidates the access_token -assigned to the device, so an attacker could use it to log out the real user -(and do it repeatedly every time the real user tries to log in to block the -attacker). Servers should require additional authentication beyond the access -token when deleting devices (for example, requiring that the user resubmit -their password). - -The display names of devices are publicly visible. Clients should consider -advising the user of this. diff --git a/specification/modules/dm.rst b/specification/modules/dm.rst deleted file mode 100644 index a89d3522..00000000 --- a/specification/modules/dm.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Direct Messaging -================ - -.. _module:dm: - -All communication over Matrix happens within a room. It is sometimes -desirable to offer users the concept of speaking directly to one -particular person. This module defines a way of marking certain rooms -as 'direct chats' with a given person. This does not restrict the chat -to being between exactly two people since this would preclude the -presence of automated 'bot' users or even a 'personal assistant' who is -able to answer direct messages on behalf of the user in their absence. - -A room may not necessarily be considered 'direct' by all members of the -room, but a signalling mechanism exists to propagate the information of -whether a chat is 'direct' to an invitee. - -Events ------- - -{{m_direct_event}} - -Client behaviour ----------------- -To start a direct chat with another user, the inviting user's client -should set the ``is_direct`` flag to |/createRoom|_. The client should do -this whenever the flow the user has followed is one where their -intention is to speak directly with another person, as opposed to bringing that -person in to a shared room. For example, clicking on 'Start Chat' beside a -person's profile picture would imply the ``is_direct`` flag should be set. - -The invitee's client may use the ``is_direct`` flag in the `m.room.member`_ -event to automatically mark the room as a direct chat but this is not -required: it may for example, prompt the user, or ignore the flag altogether. - -Both the inviting client and the invitee's client should record the fact that -the room is a direct chat by storing an ``m.direct`` event in the account data -using |/user//account_data/|_. - -Server behaviour ----------------- -When the ``is_direct`` flag is given to |/createRoom|_, the home -server must set the ``is_direct`` flag in the invite member event for any users -invited in the |/createRoom|_ call. diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst deleted file mode 100644 index 6c51ebbd..00000000 --- a/specification/modules/end_to_end_encryption.rst +++ /dev/null @@ -1,1537 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2019-2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -End-to-End Encryption -===================== - -.. _module:e2e: - -Matrix optionally supports end-to-end encryption, allowing rooms to be created -whose conversation contents are not decryptable or interceptable on any of the -participating homeservers. - -Key Distribution ----------------- -Encryption and Authentication in Matrix is based around public-key -cryptography. The Matrix protocol provides a basic mechanism for exchange of -public keys, though an out-of-band channel is required to exchange fingerprints -between users to build a web of trust. - -Overview -~~~~~~~~ - -.. code:: - - 1) Bob publishes the public keys and supported algorithms for his - device. This may include long-term identity keys, and/or one-time - keys. - - +----------+ +--------------+ - | Bob's HS | | Bob's Device | - +----------+ +--------------+ - | | - |<=============| - /keys/upload - - 2) Alice requests Bob's public identity keys and supported algorithms. - - +----------------+ +------------+ +----------+ - | Alice's Device | | Alice's HS | | Bob's HS | - +----------------+ +------------+ +----------+ - | | | - |=================>|==============>| - /keys/query - - 3) Alice selects an algorithm and claims any one-time keys needed. - - +----------------+ +------------+ +----------+ - | Alice's Device | | Alice's HS | | Bob's HS | - +----------------+ +------------+ +----------+ - | | | - |=================>|==============>| - /keys/claim - - -Key algorithms -~~~~~~~~~~~~~~ - -The name ``ed25519`` corresponds to the `Ed25519`_ signature algorithm. The key -is a 32-byte Ed25519 public key, encoded using `unpadded Base64`_. Example: - -.. code:: json - - "SogYyrkTldLz0BXP+GYWs0qaYacUI0RleEqNT8J3riQ" - -The name ``curve25519`` corresponds to the `Curve25519`_ ECDH algorithm. The -key is a 32-byte Curve25519 public key, encoded using `unpadded -Base64`_. Example: - -.. code:: json - - "JGLn/yafz74HB2AbPLYJWIVGnKAtqECOBf11yyXac2Y" - -The name ``signed_curve25519`` also corresponds to the Curve25519 algorithm, -but a key using this algorithm is represented by an object with the following -properties: - -``KeyObject`` - -========== ================ ===================================================== -Parameter Type Description -========== ================ ===================================================== -key string **Required.** The unpadded Base64-encoded 32-byte - Curve25519 public key. -signatures Signatures **Required.** Signatures of the key object. - - The signature is calculated using the process described - at `Signing JSON`_. -========== ================ ===================================================== - -Example: - -.. code:: json - - { - "key":"06UzBknVHFMwgi7AVloY7ylC+xhOhEX4PkNge14Grl8", - "signatures": { - "@user:example.com": { - "ed25519:EGURVBUNJP": "YbJva03ihSj5mPk+CHMJKUKlCXCPFXjXOK6VqBnN9nA2evksQcTGn6hwQfrgRHIDDXO2le49x7jnWJHMJrJoBQ" - } - } - } - -Device keys -~~~~~~~~~~~ - -Each device should have one Ed25519 signing key. This key should be generated -on the device from a cryptographically secure source, and the private part of -the key should never be exported from the device. This key is used as the -fingerprint for a device by other clients. - -A device will generally need to generate a number of additional keys. Details -of these will vary depending on the messaging algorithm in use. - -Algorithms generally require device identity keys as well as signing keys. Some -algorithms also require one-time keys to improve their secrecy and deniability. -These keys are used once during session establishment, and are then thrown -away. - -For Olm version 1, each device requires a single Curve25519 identity key, and a -number of signed Curve25519 one-time keys. - -Uploading keys -~~~~~~~~~~~~~~ - -A device uploads the public parts of identity keys to their homeserver as a -signed JSON object, using the |/keys/upload|_ API. -The JSON object must include the public part of the device's Ed25519 key, and -must be signed by that key, as described in `Signing JSON`_. - -One-time keys are also uploaded to the homeserver using the |/keys/upload|_ -API. - -Devices must store the private part of each key they upload. They can -discard the private part of a one-time key when they receive a message using -that key. However it's possible that a one-time key given out by a homeserver -will never be used, so the device that generates the key will never know that -it can discard the key. Therefore a device could end up trying to store too -many private keys. A device that is trying to store too many private keys may -discard keys starting with the oldest. - -Tracking the device list for a user -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Before Alice can send an encrypted message to Bob, she needs a list of each of -his devices and the associated identity keys, so that she can establish an -encryption session with each device. This list can be obtained by calling -|/keys/query|_, passing Bob's user ID in the ``device_keys`` parameter. - -From time to time, Bob may add new devices, and Alice will need to know this so -that she can include his new devices for later encrypted messages. A naive -solution to this would be to call |/keys/query|_ before sending each message - -however, the number of users and devices may be large and this would be -inefficient. - -It is therefore expected that each client will maintain a list of devices for a -number of users (in practice, typically each user with whom we share an -encrypted room). Furthermore, it is likely that this list will need to be -persisted between invocations of the client application (to preserve device -verification data and to alert Alice if Bob suddenly gets a new -device). - -Alice's client can maintain a list of Bob's devices via the following -process: - -#. It first sets a flag to record that it is now tracking Bob's device list, - and a separate flag to indicate that its list of Bob's devices is - outdated. Both flags should be in storage which persists over client - restarts. - -#. It then makes a request to |/keys/query|_, passing Bob's user ID in the - ``device_keys`` parameter. When the request completes, it stores the - resulting list of devices in persistent storage, and clears the 'outdated' - flag. - -#. During its normal processing of responses to |/sync|_, Alice's client - inspects the ``changed`` property of the |device_lists|_ field. If it is - tracking the device lists of any of the listed users, then it marks the - device lists for those users outdated, and initiates another request to - |/keys/query|_ for them. - -#. Periodically, Alice's client stores the ``next_batch`` field of the result - from |/sync|_ in persistent storage. If Alice later restarts her client, it - can obtain a list of the users who have updated their device list while it - was offline by calling |/keys/changes|_, passing the recorded ``next_batch`` - field as the ``from`` parameter. If the client is tracking the device list - of any of the users listed in the response, it marks them as outdated. It - combines this list with those already flagged as outdated, and initiates a - |/keys/query|_ request for all of them. - -.. Warning:: - - Bob may update one of his devices while Alice has a request to - ``/keys/query`` in flight. Alice's client may therefore see Bob's user ID in - the ``device_lists`` field of the ``/sync`` response while the first request - is in flight, and initiate a second request to ``/keys/query``. This may - lead to either of two related problems. - - The first problem is that, when the first request completes, the client will - clear the 'outdated' flag for Bob's devices. If the second request fails, or - the client is shut down before it completes, this could lead to Alice using - an outdated list of Bob's devices. - - The second possibility is that, under certain conditions, the second request - may complete *before* the first one. When the first request completes, the - client could overwrite the later results from the second request with those - from the first request. - - Clients MUST guard against these situations. For example, a client could - ensure that only one request to ``/keys/query`` is in flight at a time for - each user, by queuing additional requests until the first completes. - Alternatively, the client could make a new request immediately, but ensure - that the first request's results are ignored (possibly by cancelling the - request). - -.. Note:: - - When Bob and Alice share a room, with Bob tracking Alice's devices, she may leave - the room and then add a new device. Bob will not be notified of this change, - as he doesn't share a room anymore with Alice. When they start sharing a - room again, Bob has an out-of-date list of Alice's devices. In order to address - this issue, Bob's homeserver will add Alice's user ID to the ``changed`` property of - the ``device_lists`` field, thus Bob will update his list of Alice's devices as part - of his normal processing. Note that Bob can also be notified when he stops sharing - any room with Alice by inspecting the ``left`` property of the ``device_lists`` - field, and as a result should remove her from its list of tracked users. - -.. |device_lists| replace:: ``device_lists`` -.. _`device_lists`: `device_lists_sync`_ - - -Sending encrypted attachments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When encryption is enabled in a room, files should be uploaded encrypted on -the homeserver. - -In order to achieve this, a client should generate a single-use 256-bit AES -key, and encrypt the file using AES-CTR. The counter should be 64-bit long, -starting at 0 and prefixed by a random 64-bit Initialization Vector (IV), which -together form a 128-bit unique counter block. - -.. Warning:: - An IV must never be used multiple times with the same key. This implies that - if there are multiple files to encrypt in the same message, typically an - image and its thumbnail, the files must not share both the same key and IV. - -Then, the encrypted file can be uploaded to the homeserver. -The key and the IV must be included in the room event along with the resulting -``mxc://`` in order to allow recipients to decrypt the file. As the event -containing those will be Megolm encrypted, the server will never have access to -the decrypted file. - -A hash of the ciphertext must also be included, in order to prevent the homeserver from -changing the file content. - -A client should send the data as an encrypted ``m.room.message`` event, using -either ``m.file`` as the msgtype, or the appropriate msgtype for the file -type. The key is sent using the `JSON Web Key`_ format, with a `W3C -extension`_. - -.. anchor for link from m.room.message api spec -.. |encrypted_files| replace:: End-to-end encryption -.. _encrypted_files: - -Extensions to ``m.room.message`` msgtypes -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -This module adds ``file`` and ``thumbnail_file`` properties, of type -``EncryptedFile``, to ``m.room.message`` msgtypes that reference files, such as -`m.file`_ and `m.image`_, replacing the ``url`` and ``thumbnail_url`` -properties. - -.. todo: generate this from a swagger definition? - -``EncryptedFile`` - -========= ================ ===================================================== -Parameter Type Description -========= ================ ===================================================== -url string **Required.** The URL to the file. -key JWK **Required.** A `JSON Web Key`_ object. -iv string **Required.** The 128-bit unique counter block used by - AES-CTR, encoded as unpadded base64. -hashes {string: string} **Required.** A map from an algorithm name to a hash - of the ciphertext, encoded as unpadded base64. Clients - should support the SHA-256 hash, which uses the key - ``sha256``. -v string **Required.** Version of the encrypted attachments - protocol. Must be ``v2``. -========= ================ ===================================================== - -``JWK`` - -========= ========= ============================================================ -Parameter Type Description -========= ========= ============================================================ -kty string **Required.** Key type. Must be ``oct``. -key_ops [string] **Required.** Key operations. Must at least contain - ``encrypt`` and ``decrypt``. -alg string **Required.** Algorithm. Must be ``A256CTR``. -k string **Required.** The key, encoded as urlsafe unpadded base64. -ext boolean **Required.** Extractable. Must be ``true``. This is a - `W3C extension`_. -========= ========= ============================================================ - -Example: - -.. code :: json - - { - "content": { - "body": "something-important.jpg", - "file": { - "url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe", - "mimetype": "image/jpeg", - "v": "v2", - "key": { - "alg": "A256CTR", - "ext": true, - "k": "aWF6-32KGYaC3A_FEUCk1Bt0JA37zP0wrStgmdCaW-0", - "key_ops": ["encrypt","decrypt"], - "kty": "oct" - }, - "iv": "w+sE15fzSc0AAAAAAAAAAA", - "hashes": { - "sha256": "fdSLu/YkRx3Wyh3KQabP3rd6+SFiKg5lsJZQHtkSAYA" - } - }, - "info": { - "mimetype": "image/jpeg", - "h": 1536, - "size": 422018, - "thumbnail_file": { - "hashes": { - "sha256": "/NogKqW5bz/m8xHgFiH5haFGjCNVmUIPLzfvOhHdrxY" - }, - "iv": "U+k7PfwLr6UAAAAAAAAAAA", - "key": { - "alg": "A256CTR", - "ext": true, - "k": "RMyd6zhlbifsACM1DXkCbioZ2u0SywGljTH8JmGcylg", - "key_ops": ["encrypt", "decrypt"], - "kty": "oct" - }, - "mimetype": "image/jpeg", - "url": "mxc://example.org/pmVJxyxGlmxHposwVSlOaEOv", - "v": "v2" - }, - "thumbnail_info": { - "h": 768, - "mimetype": "image/jpeg", - "size": 211009, - "w": 432 - }, - "w": 864 - }, - "msgtype": "m.image" - }, - "event_id": "$143273582443PhrSn:example.org", - "origin_server_ts": 1432735824653, - "room_id": "!jEsUZKDJdhlrceRyVU:example.org", - "sender": "@example:example.org", - "type": "m.room.message", - "unsigned": { - "age": 1234 - } - } - -Claiming one-time keys -~~~~~~~~~~~~~~~~~~~~~~ - -A client wanting to set up a session with another device can claim a one-time -key for that device. This is done by making a request to the |/keys/claim|_ -API. - -A homeserver should rate-limit the number of one-time keys that a given user or -remote server can claim. A homeserver should discard the public part of a one -time key once it has given that key to another user. - -Device verification -------------------- - -Before Alice sends Bob encrypted data, or trusts data received from him, she -may want to verify that she is actually communicating with him, rather than a -man-in-the-middle. This verification process requires an out-of-band channel: -there is no way to do it within Matrix without trusting the administrators of -the homeservers. - -In Matrix, verification works by Alice meeting Bob in person, or contacting him -via some other trusted medium, and use `SAS Verification`_ to interactively -verify Bob's devices. Alice and Bob may also read aloud their unpadded base64 -encoded Ed25519 public key, as returned by ``/keys/query``. - -Device verification may reach one of several conclusions. For example: - -* Alice may "accept" the device. This means that she is satisfied that the - device belongs to Bob. She can then encrypt sensitive material for that - device, and knows that messages received were sent from that device. - -* Alice may "reject" the device. She will do this if she knows or suspects - that Bob does not control that device (or equivalently, does not trust - Bob). She will not send sensitive material to that device, and cannot trust - messages apparently received from it. - -* Alice may choose to skip the device verification process. She is not able - to verify that the device actually belongs to Bob, but has no reason to - suspect otherwise. The encryption protocol continues to protect against - passive eavesdroppers. - -.. NOTE:: - - Once the signing key has been verified, it is then up to the encryption - protocol to verify that a given message was sent from a device holding that - Ed25519 private key, or to encrypt a message so that it may only be - decrypted by such a device. For the Olm protocol, this is documented at - https://matrix.org/docs/olm_signing.html. - - -Key verification framework -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Verifying keys manually by reading out the Ed25519 key is not very user-friendly, -and can lead to errors. In order to help mitigate errors, and to make the process -easier for users, some verification methods are supported by the specification. -The methods all use a common framework for negotiating the key verification. - -To use this framework, Alice's client would send ``m.key.verification.request`` -events to Bob's devices. All of the ``to_device`` messages sent to Bob MUST have -the same ``transaction_id`` to indicate they are part of the same request. This -allows Bob to reject the request on one device, and have it apply to all of his -devices. Similarly, it allows Bob to process the verification on one device without -having to involve all of his devices. - -When Bob's device receives an ``m.key.verification.request``, it should prompt Bob -to verify keys with Alice using one of the supported methods in the request. If -Bob's device does not understand any of the methods, it should not cancel the request -as one of his other devices may support the request. Instead, Bob's device should -tell Bob that an unsupported method was used for starting key verification. The -prompt for Bob to accept/reject Alice's request (or the unsupported method prompt) -should be automatically dismissed 10 minutes after the ``timestamp`` field or 2 -minutes after Bob's client receives the message, whichever comes first, if Bob -does not interact with the prompt. The prompt should additionally be hidden if -an appropriate ``m.key.verification.cancel`` message is received. - -If Bob rejects the request, Bob's client must send an ``m.key.verification.cancel`` -message to Alice's device. Upon receipt, Alice's device should tell her that Bob -does not want to verify her device and send ``m.key.verification.cancel`` messages -to all of Bob's devices to notify them that the request was rejected. - -If Bob accepts the request, Bob's device starts the key verification process by -sending an ``m.key.verification.start`` message to Alice's device. Upon receipt -of this message, Alice's device should send an ``m.key.verification.cancel`` message -to all of Bob's other devices to indicate the process has been started. The start -message must use the same ``transaction_id`` from the original key verification -request if it is in response to the request. The start message can be sent independently -of any request. - -Individual verification methods may add additional steps, events, and properties to -the verification messages. Event types for methods defined in this specification must -be under the ``m.key.verification`` namespace and any other event types must be namespaced -according to the Java package naming convention. - -Any of Alice's or Bob's devices can cancel the key verification request or process -at any time with an ``m.key.verification.cancel`` message to all applicable devices. - -This framework yields the following handshake, assuming both Alice and Bob each have -2 devices, Bob's first device accepts the key verification request, and Alice's second -device initiates the request. Note how Alice's first device is not involved in the -request or verification process. - -:: - - +---------------+ +---------------+ +-------------+ +-------------+ - | AliceDevice1 | | AliceDevice2 | | BobDevice1 | | BobDevice2 | - +---------------+ +---------------+ +-------------+ +-------------+ - | | | | - | | m.key.verification.request | | - | |---------------------------------->| | - | | | | - | | m.key.verification.request | | - | |-------------------------------------------------->| - | | | | - | | m.key.verification.start | | - | |<----------------------------------| | - | | | | - | | m.key.verification.cancel | | - | |-------------------------------------------------->| - | | | | - - -After the handshake, the verification process begins. - -{{m_key_verification_request_event}} - -{{m_key_verification_start_event}} - -{{m_key_verification_cancel_event}} - - -.. _`SAS Verification`: - -Short Authentication String (SAS) verification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -SAS verification is a user-friendly key verification process built off the common -framework outlined above. SAS verification is intended to be a highly interactive -process for users, and as such exposes verification methods which are easier for -users to use. - -The verification process is heavily inspired by Phil Zimmermann's ZRTP key agreement -handshake. A key part of key agreement in ZRTP is the hash commitment: the party that -begins the Diffie-Hellman key sharing sends a hash of their part of the Diffie-Hellman -exchange, and does not send their part of the Diffie-Hellman exchange until they have -received the other party's part. Thus an attacker essentially only has one attempt to -attack the Diffie-Hellman exchange, and hence we can verify fewer bits while still -achieving a high degree of security: if we verify n bits, then an attacker has a 1 in -2\ :sup:`n` chance of success. For example, if we verify 40 bits, then an attacker has -a 1 in 1,099,511,627,776 chance (or less than 1 in 10\ :sup:`12` chance) of success. A failed -attack would result in a mismatched Short Authentication String, alerting users to the -attack. - -The verification process takes place over `to-device`_ messages in two phases: - -1. Key agreement phase (based on `ZRTP key agreement `_). -#. Key verification phase (based on HMAC). - -The process between Alice and Bob verifying each other would be: - -.. |AlicePublicKey| replace:: :math:`K_{A}^{public}` -.. |AlicePrivateKey| replace:: :math:`K_{A}^{private}` -.. |AliceCurve25519| replace:: :math:`K_{A}^{private},K_{A}^{public}` -.. |BobPublicKey| replace:: :math:`K_{B}^{public}` -.. |BobPrivateKey| replace:: :math:`K_{B}^{private}` -.. |BobCurve25519| replace:: :math:`K_{B}^{private},K_{B}^{public}` -.. |BobAliceCurve25519| replace:: :math:`K_{B}^{private}K_{A}^{public}` -.. |AliceBobECDH| replace:: :math:`ECDH(K_{A}^{private},K_{B}^{public})` - -1. Alice and Bob establish a secure out-of-band connection, such as meeting - in-person or a video call. "Secure" here means that either party cannot be - impersonated, not explicit secrecy. -#. Alice and Bob communicate which devices they'd like to verify with each other. -#. Alice selects Bob's device from the device list and begins verification. -#. Alice's client ensures it has a copy of Bob's device key. -#. Alice's device sends Bob's device an ``m.key.verification.start`` message. -#. Bob's device receives the message and selects a key agreement protocol, hash - algorithm, message authentication code, and SAS method supported by Alice's - device. -#. Bob's device ensures it has a copy of Alice's device key. -#. Bob's device creates an ephemeral Curve25519 key pair (|BobCurve25519|), and - calculates the hash (using the chosen algorithm) of the public key |BobPublicKey|. -#. Bob's device replies to Alice's device with an ``m.key.verification.accept`` message. -#. Alice's device receives Bob's message and stores the commitment hash for later use. -#. Alice's device creates an ephemeral Curve25519 key pair (|AliceCurve25519|) and - replies to Bob's device with an ``m.key.verification.key``, sending only the public - key |AlicePublicKey|. -#. Bob's device receives Alice's message and replies with its own ``m.key.verification.key`` - message containing its public key |BobPublicKey|. -#. Alice's device receives Bob's message and verifies the commitment hash from earlier - matches the hash of the key Bob's device just sent and the content of Alice's - ``m.key.verification.start`` message. -#. Both Alice and Bob's devices perform an Elliptic-curve Diffie-Hellman (|AliceBobECDH|), - using the result as the shared secret. -#. Both Alice and Bob's devices display a SAS to their users, which is derived - from the shared key using one of the methods in this section. If multiple SAS - methods are available, clients should allow the users to select a method. -#. Alice and Bob compare the strings shown by their devices, and tell their devices if - they match or not. -#. Assuming they match, Alice and Bob's devices calculate the HMAC of their own device keys - and a comma-separated sorted list of the key IDs that they wish the other user - to verify, using SHA-256 as the hash function. HMAC is defined in `RFC 2104 `_. - The key for the HMAC is different for each item and is calculated by generating - 32 bytes (256 bits) using `the key verification HKDF <#sas-hkdf>`_. -#. Alice's device sends Bob's device an ``m.key.verification.mac`` message containing the - MAC of Alice's device keys and the MAC of her key IDs to be verified. Bob's device does - the same for Bob's device keys and key IDs concurrently with Alice. -#. When the other device receives the ``m.key.verification.mac`` message, the device - calculates the HMAC of its copies of the other device's keys given in the message, - as well as the HMAC of the comma-separated, sorted, list of key IDs in the message. - The device compares these with the HMAC values given in the message, and if everything - matches then the device keys are verified. - -The wire protocol looks like the following between Alice and Bob's devices:: - - +-------------+ +-----------+ - | AliceDevice | | BobDevice | - +-------------+ +-----------+ - | | - | m.key.verification.start | - |-------------------------------->| - | | - | m.key.verification.accept | - |<--------------------------------| - | | - | m.key.verification.key | - |-------------------------------->| - | | - | m.key.verification.key | - |<--------------------------------| - | | - | m.key.verification.mac | - |-------------------------------->| - | | - | m.key.verification.mac | - |<--------------------------------| - | | - -Error and exception handling -<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -At any point the interactive verification can go wrong. The following describes what -to do when an error happens: - -* Alice or Bob can cancel the verification at any time. An ``m.key.verification.cancel`` - message must be sent to signify the cancellation. -* The verification can time out. Clients should time out a verification that does not - complete within 10 minutes. Additionally, clients should expire a ``transaction_id`` - which goes unused for 10 minutes after having last sent/received it. The client should - inform the user that the verification timed out, and send an appropriate - ``m.key.verification.cancel`` message to the other device. -* When the same device attempts to initiate multiple verification attempts, the recipient - should cancel all attempts with that device. -* When a device receives an unknown ``transaction_id``, it should send an appropriate - ``m.key.verification.cancel`` message to the other device indicating as such. This - does not apply for inbound ``m.key.verification.start`` or ``m.key.verification.cancel`` - messages. -* If the two devices do not share a common key share, hash, HMAC, or SAS method then - the device should notify the other device with an appropriate ``m.key.verification.cancel`` - message. -* If the user claims the Short Authentication Strings do not match, the device should - send an appropriate ``m.key.verification.cancel`` message to the other device. -* If the device receives a message out of sequence or that it was not expecting, it should - notify the other device with an appropriate ``m.key.verification.cancel`` message. - - -Verification messages specific to SAS -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -Building off the common framework, the following events are involved in SAS verification. - -The ``m.key.verification.cancel`` event is unchanged, however the following error codes -are used in addition to those already specified: - -* ``m.unknown_method``: The devices are unable to agree on the key agreement, hash, MAC, - or SAS method. -* ``m.mismatched_commitment``: The hash commitment did not match. -* ``m.mismatched_sas``: The SAS did not match. - - -{{m_key_verification_start_m_sas_v1_event}} - -{{m_key_verification_accept_event}} - -{{m_key_verification_key_event}} - -{{m_key_verification_mac_event}} - - -.. _sas-hkdf: - -HKDF calculation -<<<<<<<<<<<<<<<< - -In all of the SAS methods, HKDF is as defined in `RFC 5869 `_ -and uses the previously agreed-upon hash function for the hash function. The shared -secret is supplied as the input keying material. No salt is used. When the -``key_agreement_protocol`` is ``curve25519-hkdf-sha256``, the info parameter is -the concatenation of: - - * The string ``MATRIX_KEY_VERIFICATION_SAS|``. - * The Matrix ID of the user who sent the ``m.key.verification.start`` message, - followed by ``|``. - * The Device ID of the device which sent the ``m.key.verification.start`` - message, followed by ``|``. - * The public key from the ``m.key.verification.key`` message sent by the device - which sent the ``m.key.verification.start`` message, followed by ``|``. - * The Matrix ID of the user who sent the ``m.key.verification.accept`` message, - followed by ``|``. - * The Device ID of the device which sent the ``m.key.verification.accept`` - message, followed by ``|``. - * The public key from the ``m.key.verification.key`` message sent by the device - which sent the ``m.key.verification.accept`` message, followed by ``|``. - * The ``transaction_id`` being used. - -When the ``key_agreement_protocol`` is the deprecated method ``curve25519``, -the info parameter is the concatenation of: - - * The string ``MATRIX_KEY_VERIFICATION_SAS``. - * The Matrix ID of the user who sent the ``m.key.verification.start`` message. - * The Device ID of the device which sent the ``m.key.verification.start`` message. - * The Matrix ID of the user who sent the ``m.key.verification.accept`` message. - * The Device ID of the device which sent the ``m.key.verification.accept`` message. - * The ``transaction_id`` being used. - -New implementations are discouraged from implementing the ``curve25519`` method. - -.. admonition:: Rationale - - HKDF is used over the plain shared secret as it results in a harder attack - as well as more uniform data to work with. - -For verification of each party's device keys, HKDF is as defined in RFC 5869 and -uses SHA-256 as the hash function. The shared secret is supplied as the input keying -material. No salt is used, and in the info parameter is the concatenation of: - - * The string ``MATRIX_KEY_VERIFICATION_MAC``. - * The Matrix ID of the user whose key is being MAC-ed. - * The Device ID of the device sending the MAC. - * The Matrix ID of the other user. - * The Device ID of the device receiving the MAC. - * The ``transaction_id`` being used. - * The Key ID of the key being MAC-ed, or the string ``KEY_IDS`` if the item - being MAC-ed is the list of key IDs. - -SAS method: ``decimal`` -<<<<<<<<<<<<<<<<<<<<<<< - -Generate 5 bytes using `HKDF <#sas-hkdf>`_ then take sequences of 13 bits to -convert to decimal numbers (resulting in 3 numbers between 0 and 8191 inclusive -each). Add 1000 to each calculated number. - -The bitwise operations to get the numbers given the 5 bytes -:math:`B_{0}, B_{1}, B_{2}, B_{3}, B_{4}` would be: - -* First: :math:`(B_{0} \ll 5 | B_{1} \gg 3) + 1000` -* Second: :math:`((B_{1} \& 0x7) \ll 10 | B_{2} \ll 2 | B_{3} \gg 6) + 1000` -* Third: :math:`((B_{3} \& 0x3F) \ll 7 | B_{4} \gg 1) + 1000` - -The digits are displayed to the user either with an appropriate separator, -such as dashes, or with the numbers on individual lines. - -SAS method: ``emoji`` -<<<<<<<<<<<<<<<<<<<<< - -Generate 6 bytes using `HKDF <#sas-hkdf>`_ then split the first 42 bits into -7 groups of 6 bits, similar to how one would base64 encode something. Convert -each group of 6 bits to a number and use the following table to get the corresponding -emoji: - -{{sas_emoji_table}} - -.. Note:: - This table is available as JSON at - https://github.com/matrix-org/matrix-doc/blob/master/data-definitions/sas-emoji.json - -.. admonition:: Rationale - - The emoji above were chosen to: - - * Be recognisable without colour. - * Be recognisable at a small size. - * Be recognisable by most cultures. - * Be distinguishable from each other. - * Easily described by a few words. - * Avoid symbols with negative connotations. - * Be likely similar across multiple platforms. - -Clients SHOULD show the emoji with the descriptions from the table, or appropriate -translation of those descriptions. Client authors SHOULD collaborate to create a -common set of translations for all languages. - -.. Note:: - Known translations for the emoji are available from - https://github.com/matrix-org/matrix-doc/blob/master/data-definitions/ and can be - translated online: https://translate.riot.im/projects/matrix-doc/sas-emoji-v1 - - -Cross-signing -~~~~~~~~~~~~~ - -Rather than requiring Alice to verify each of Bob's devices with each of her -own devices and vice versa, the cross-signing feature allows users to sign their -device keys such that Alice and Bob only need to verify once. With -cross-signing, each user has a set of cross-signing keys that are used to sign -their own device keys and other users' keys, and can be used to trust device -keys that were not verified directly. - -Each user has three ed25519 key pairs for cross-signing: - -* a master key (MSK) that serves as the user's identity in cross-signing and signs - their other cross-signing keys; -* a user-signing key (USK) -- only visible to the user that it belongs to -- - that signs other users' master keys; and -* a self-signing key (SSK) that signs the user's own device keys. - -The master key may also be used to sign other items such as the backup key. The -master key may also be signed by the user's own device keys to aid in migrating -from device verifications: if Alice's device had previously verified Bob's -device and Bob's device has signed his master key, then Alice's device can -trust Bob's master key, and she can sign it with her user-signing key. - -Users upload their cross-signing keys to the server using `POST -/_matrix/client/r0/keys/device_signing/upload`_. When Alice uploads new -cross-signing keys, her user ID will appear in the ``changed`` property of the -``device_lists`` field of the ``/sync`` of response of all users who share an -encrypted room with her. When Bob sees Alice's user ID in his ``/sync``, he -will call `POST /_matrix/client/r0/keys/query`_ to retrieve Alice's device and -cross-signing keys. - -If Alice has a device and wishes to send an encrypted message to Bob, she can -trust Bob's device if: - -- Alice's device is using a master key that has signed her user-signing key, -- Alice's user-signing key has signed Bob's master key, -- Bob's master key has signed Bob's self-signing key, and -- Bob's self-signing key has signed Bob's device key. - -The following diagram illustrates how keys are signed: - -.. code:: - - +------------------+ .................. +----------------+ - | +--------------+ | .................. : | +------------+ | - | | v v v : : v v v | | - | | +-----------+ : : +-----------+ | | - | | | Alice MSK | : : | Bob MSK | | | - | | +-----------+ : : +-----------+ | | - | | | : : : : | | | - | | +--+ :... : : ...: +--+ | | - | | v v : : v v | | - | | +-----------+ ............. : : ............. +-----------+ | | - | | | Alice SSK | : Alice USK : : : : Bob USK : | Bob SSK | | | - | | +-----------+ :...........: : : :...........: +-----------+ | | - | | | ... | : : : : | ... | | | - | | V V :........: :........: V V | | - | | +---------+ -+ +---------+ -+ | | - | | | Devices | ...| | Devices | ...| | | - | | +---------+ -+ +---------+ -+ | | - | | | ... | | ... | | | - | +------+ | | +----+ | - +----------------+ +--------------+ - -.. based on https://jcg.re/blog/quick-overview-matrix-cross-signing/ - -In the diagram, boxes represent keys and lines represent signatures with the -arrows pointing from the signing key to the key being signed. Dotted boxes and -lines represent keys and signatures that are only visible to the user who -created them. - -The following diagram illustrates Alice's view, hiding the keys and signatures -that she cannot see: - -.. code:: - - +------------------+ +----------------+ +----------------+ - | +--------------+ | | | | +------------+ | - | | v v | v v v | | - | | +-----------+ | +-----------+ | | - | | | Alice MSK | | | Bob MSK | | | - | | +-----------+ | +-----------+ | | - | | | | | | | | - | | +--+ +--+ | +--+ | | - | | v v | v | | - | | +-----------+ +-----------+ | +-----------+ | | - | | | Alice SSK | | Alice USK | | | Bob SSK | | | - | | +-----------+ +-----------+ | +-----------+ | | - | | | ... | | | | ... | | | - | | V V +--------+ V V | | - | | +---------+ -+ +---------+ -+ | | - | | | Devices | ...| | Devices | ...| | | - | | +---------+ -+ +---------+ -+ | | - | | | ... | | ... | | | - | +------+ | | +----+ | - +----------------+ +--------------+ - -`Verification methods <#device-verification>`_ can be used to verify a user's -master key by using the master public key, encoded using unpadded base64, as -the device ID, and treating it as a normal device. For example, if Alice and -Bob verify each other using SAS, Alice's ``m.key.verification.mac`` message to -Bob may include ``"ed25519:alices+master+public+key": -"alices+master+public+key"`` in the ``mac`` property. Servers therefore must -ensure that device IDs will not collide with cross-signing public keys. - -Key and signature security -<<<<<<<<<<<<<<<<<<<<<<<<<< - -A user's master key could allow an attacker to impersonate that user to other -users, or other users to that user. Thus clients must ensure that the private -part of the master key is treated securely. If clients do not have a secure -means of storing the master key (such as a secret storage system provided by -the operating system), then clients must not store the private part. - -If a user's client sees that any other user has changed their master key, that -client must notify the user about the change before allowing communication -between the users to continue. - -A user's user-signing and self-signing keys are intended to be easily -replaceable if they are compromised by re-issuing a new key signed by the -user's master key and possibly by re-verifying devices or users. However, -doing so relies on the user being able to notice when their keys have been -compromised, and it involves extra work for the user, and so although clients -do not have to treat the private parts as sensitively as the master key, -clients should still make efforts to store the private part securely, or not -store it at all. Clients will need to balance the security of the keys with -the usability of signing users and devices when performing key verification. - -To avoid leaking of social graphs, servers will only allow users to see: - -* signatures made by the user's own master, self-signing or user-signing keys, -* signatures made by the user's own devices about their own master key, -* signatures made by other users' self-signing keys about their respective - devices, -* signatures made by other users' master keys about their respective - self-signing key, or -* signatures made by other users' devices about their respective master keys. - -Users will not be able to see signatures made by other users' user-signing keys. - -{{cross_signing_cs_http_api}} - -.. section name changed, so make sure that old links keep working -.. _key-sharing: - -Sharing keys between devices ----------------------------- - -If Bob has an encrypted conversation with Alice on his computer, and then logs in -through his phone for the first time, he may want to have access to the previously -exchanged messages. To address this issue, several methods are provided to -allow users to transfer keys from one device to another. - -Key requests -~~~~~~~~~~~~ - -When a device is missing keys to decrypt messages, it can request the keys by -sending `m.room_key_request`_ to-device messages to other devices with -``action`` set to ``request``. - -If a device wishes to share the keys with that device, it can forward the keys -to the first device by sending an encrypted `m.forwarded_room_key`_ to-device -message. The first device should then send an `m.room_key_request`_ to-device -message with ``action`` set to ``request_cancellation`` to the other devices -that it had originally sent the key request to; a device that receives a -``request_cancellation`` should disregard any previously-received ``request`` -message with the same ``request_id`` and ``requesting_device_id``. - -If a device does not wish to share keys with that device, it can indicate this -by sending an `m.room_key.withheld`_ to-device message, as described in -`Reporting that decryption keys are withheld`_. - -.. NOTE:: - - Key sharing can be a big attack vector, thus it must be done very carefully. - A reasonable strategy is for a user's client to only send keys requested by the - verified devices of the same user. - -Server-side key backups -~~~~~~~~~~~~~~~~~~~~~~~ - -Devices may upload encrypted copies of keys to the server. When a device tries -to read a message that it does not have keys for, it may request the key from -the server and decrypt it. Backups are per-user, and users may replace backups -with new backups. - -In contrast with `Key requests`_, Server-side key backups do not require another -device to be online from which to request keys. However, as the session keys are -stored on the server encrypted, it requires users to enter a decryption key to -decrypt the session keys. - -To create a backup, a client will call `POST -/_matrix/client/r0/room_keys/version`_ and define how the keys are to be -encrypted through the backup's ``auth_data``; other clients can discover the -backup by calling `GET /_matrix/client/r0/room_keys/version`_. Keys are -encrypted according to the backup's ``auth_data`` and added to the backup by -calling `PUT /_matrix/client/r0/room_keys/keys`_ or one of its variants, and -can be retrieved by calling `GET /_matrix/client/r0/room_keys/keys`_ or one of -its variants. Keys can only be written to the most recently created version of -the backup. Backups can also be deleted using `DELETE -/_matrix/client/r0/room_keys/version/{version}`_, or individual keys can be -deleted using `DELETE /_matrix/client/r0/room_keys/keys`_ or one of its -variants. - -Clients must only store keys in backups after they have ensured that the -``auth_data`` is trusted, either by checking the signatures on it, or by -deriving the public key from a private key that it obtained from a trusted -source. - -When a client uploads a key for a session that the server already has a key -for, the server will choose to either keep the existing key or replace it with -the new key based on the key metadata as follows: - -- if the keys have different values for ``is_verified``, then it will keep the - key that has ``is_verified`` set to ``true``; -- if they have the same values for ``is_verified``, then it will keep the key - with a lower ``first_message_index``; -- and finally, is ``is_verified`` and ``first_message_index`` are equal, then - it will keep the key with a lower ``forwarded_count``. - -Recovery key -<<<<<<<<<<<< - -If the recovery key (the private half of the backup encryption key) is -presented to the user to save, it is presented as a string constructed as -follows: - -1. The 256-bit curve25519 private key is prepended by the bytes ``0x8B`` and - ``0x01`` -2. All the bytes in the string above, including the two header bytes, are XORed - together to form a parity byte. This parity byte is appended to the byte - string. -3. The byte string is encoded using base58, using the same `mapping as is used - for Bitcoin addresses - `_, - that is, using the alphabet - ``123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz``. -4. A space should be added after every 4th character. - -When reading in a recovery key, clients must disregard whitespace, and perform -the reverse of steps 1 through 3. - -Backup algorithm: ``m.megolm_backup.v1.curve25519-aes-sha2`` -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -When a backup is created with the ``algorithm`` set to -``m.megolm_backup.v1.curve25519-aes-sha2``, the ``auth_data`` should have the -following format: - -``AuthData`` - -.. table:: - :widths: auto - - ========== =========== ====================================================== - Parameter Type Description - ========== =========== ====================================================== - public_key string **Required.** The curve25519 public key used to encrypt - the backups, encoded in unpadded base64. - signatures Signatures Optional. Signatures of the ``auth_data``, as Signed - JSON - ========== =========== ====================================================== - -The ``session_data`` field in the backups is constructed as follows: - -1. Encode the session key to be backed up as a JSON object with the properties: - - .. table:: - :widths: auto - - =============================== ======== ========================================= - Parameter Type Description - =============================== ======== ========================================= - algorithm string **Required.** The end-to-end message - encryption algorithm that the key is - for. Must be ``m.megolm.v1.aes-sha2``. - forwarding_curve25519_key_chain [string] **Required.** Chain of Curve25519 keys - through which this session was - forwarded, via - `m.forwarded_room_key`_ events. - sender_key string **Required.** Unpadded base64-encoded - device curve25519 key. - sender_claimed_keys {string: **Required.** A map from algorithm name - string} (``ed25519``) to the identity key - for the sending device. - session_key string **Required.** Unpadded base64-encoded - session key in `session-sharing format - `_. - =============================== ======== ========================================= - -2. Generate an ephemeral curve25519 key, and perform an ECDH with the ephemeral - key and the backup's public key to generate a shared secret. The public - half of the ephemeral key, encoded using unpadded base64, becomes the ``ephemeral`` - property of the ``session_data``. -3. Using the shared secret, generate 80 bytes by performing an HKDF using - SHA-256 as the hash, with a salt of 32 bytes of 0, and with the empty string - as the info. The first 32 bytes are used as the AES key, the next 32 bytes - are used as the MAC key, and the last 16 bytes are used as the AES - initialization vector. -4. Stringify the JSON object, and encrypt it using AES-CBC-256 with PKCS#7 - padding. This encrypted data, encoded using unpadded base64, becomes the - ``ciphertext`` property of the ``session_data``. -5. Pass the raw encrypted data (prior to base64 encoding) through HMAC-SHA-256 - using the MAC key generated above. The first 8 bytes of the resulting MAC - are base64-encoded, and become the ``mac`` property of the ``session_data``. - -{{key_backup_cs_http_api}} - -Key exports -~~~~~~~~~~~ - -Keys can be manually exported from one device to an encrypted file, copied to -another device, and imported. The file is encrypted using a user-supplied -passphrase, and is created as follows: - -1. Encode the sessions as a JSON object, formatted as described in `Key export - format`_. -2. Generate a 512-bit key from the user-entered passphrase by computing - `PBKDF2`_\(HMAC-SHA-512, passphrase, S, N, 512), where S is a 128-bit - cryptographically-random salt and N is the number of rounds. N should be at - least 100,000. The keys K and K' are set to the first and last 256 bits of - this generated key, respectively. K is used as an AES-256 key, and K' is - used as an HMAC-SHA-256 key. -3. Serialize the JSON object as a UTF-8 string, and encrypt it using - AES-CTR-256 with the key K generated above, and with a 128-bit - cryptographically-random initialization vector, IV, that has bit 63 set to - zero. (Setting bit 63 to zero in IV is needed to work around differences in - implementations of AES-CTR.) -4. Concatenate the following data: - - ============ =============================================================== - Size (bytes) Description - ============ =============================================================== - 1 Export format version, which must be ``0x01``. - 16 The salt S. - 16 The initialization vector IV. - 4 The number of rounds N, as a big-endian unsigned 32-bit integer. - variable The encrypted JSON object. - 32 The HMAC-SHA-256 of all the above string concatenated together, - using K' as the key. - ============ =============================================================== - -5. Base64-encode the string above. Newlines may be added to avoid overly long - lines. -6. Prepend the resulting string with ``-----BEGIN MEGOLM SESSION DATA-----``, - with a trailing newline, and append ``-----END MEGOLM SESSION DATA-----``, - with a leading and trailing newline. - -Key export format -<<<<<<<<<<<<<<<<< - -The exported sessions are formatted as a JSON array of ``SessionData`` objects -described as follows: - -``SessionData`` - -.. table:: - :widths: auto - - =============================== =========== ==================================== - Parameter Type Description - =============================== =========== ==================================== - algorithm string Required. The encryption algorithm - that the session uses. Must be - ``m.megolm.v1.aes-sha2``. - forwarding_curve25519_key_chain [string] Required. Chain of Curve25519 keys - through which this session was - forwarded, via - `m.forwarded_room_key`_ events. - room_id string Required. The room where the - session is used. - sender_key string Required. The Curve25519 key of the - device which initiated the session - originally. - sender_claimed_keys {string: Required. The Ed25519 key of the - string} device which initiated the session - originally. - session_id string Required. The ID of the session. - session_key string Required. The key for the session. - =============================== =========== ==================================== - -This is similar to the format before encryption used for the session keys in -`Server-side key backups`_ but adds the ``room_id`` and ``session_id`` fields. - -Example: - -.. code:: json - - [ - { - "algorithm": "m.megolm.v1.aes-sha2", - "forwarding_curve25519_key_chain": [ - "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" - ], - "room_id": "!Cuyf34gef24t:localhost", - "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU", - "sender_claimed_keys": { - "ed25519": "", - }, - "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", - "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8Llf..." - }, - ... - ] - -Messaging Algorithms --------------------- - -Messaging Algorithm Names -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with ``m.`` are reserved for -algorithms defined by this specification. Implementations wanting to experiment -with new algorithms must be uniquely globally namespaced following Java's package -naming conventions. - -Algorithm names should be short and meaningful, and should list the primitives -used by the algorithm so that it is easier to see if the algorithm is using a -broken primitive. - -A name of ``m.olm.v1`` is too short: it gives no information about the primitives -in use, and is difficult to extend for different primitives. However a name of -``m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256`` -is too long despite giving a more precise description of the algorithm: it adds -to the data transfer overhead and sacrifices clarity for human readers without -adding any useful extra information. - -``m.olm.v1.curve25519-aes-sha2`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The name ``m.olm.v1.curve25519-aes-sha2`` corresponds to version 1 of the Olm -ratchet, as defined by the `Olm specification`_. This uses: - -* Curve25519 for the initial key agreement. -* HKDF-SHA-256 for ratchet key derivation. -* Curve25519 for the root key ratchet. -* HMAC-SHA-256 for the chain key ratchet. -* HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. - -Devices that support Olm must include "m.olm.v1.curve25519-aes-sha2" in their -list of supported messaging algorithms, must list a Curve25519 device key, and -must publish Curve25519 one-time keys. - -An event encrypted using Olm has the following format: - -.. code:: json - - { - "type": "m.room.encrypted", - "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": "", - "ciphertext": { - "": { - "type": 0, - "body": "" - } - } - } - } - -``ciphertext`` is a mapping from device Curve25519 key to an encrypted payload -for that device. ``body`` is a Base64-encoded Olm message body. ``type`` is an -integer indicating the type of the message body: 0 for the initial pre-key -message, 1 for ordinary messages. - -Olm sessions will generate messages with a type of 0 until they receive a -message. Once a session has decrypted a message it will produce messages with -a type of 1. - -When a client receives a message with a type of 0 it must first check if it -already has a matching session. If it does then it will use that session to -try to decrypt the message. If there is no existing session then the client -must create a new session and use the new session to decrypt the message. A -client must not persist a session or remove one-time keys used by a session -until it has successfully decrypted a message using that session. - -Messages with type 1 can only be decrypted with an existing session. If there -is no matching session, the client must treat this as an invalid message. - -The plaintext payload is of the form: - -.. code:: json - - { - "type": "", - "content": "", - "sender": "", - "recipient": "", - "recipient_keys": { - "ed25519": "" - }, - "keys": { - "ed25519": "" - } - } - -The type and content of the plaintext message event are given in the payload. - -Other properties are included in order to prevent an attacker from publishing -someone else's curve25519 keys as their own and subsequently claiming to have -sent messages which they didn't. -``sender`` must correspond to the user who sent the event, ``recipient`` to -the local user, and ``recipient_keys`` to the local ed25519 key. - -Clients must confirm that the ``sender_key`` and the ``ed25519`` field value -under the ``keys`` property match the keys returned by |/keys/query|_ for -the given user, and must also verify the signature of the payload. Without -this check, a client cannot be sure that the sender device owns the private -part of the ed25519 key it claims to have in the Olm payload. -This is crucial when the ed25519 key corresponds to a verified device. - -If a client has multiple sessions established with another device, it should -use the session from which it last received and successfully decrypted a -message. For these purposes, a session that has not received any messages -should use its creation time as the time that it last received a message. -A client may expire old sessions by defining a maximum number of olm sessions -that it will maintain for each device, and expiring sessions on a Least Recently -Used basis. The maximum number of olm sessions maintained per device should -be at least 4. - -Recovering from undecryptable messages -<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - -Occasionally messages may be undecryptable by clients due to a variety of reasons. -When this happens to an Olm-encrypted message, the client should assume that the Olm -session has become corrupted and create a new one to replace it. - -.. Note:: - Megolm-encrypted messages generally do not have the same problem. Usually the key - for an undecryptable Megolm-encrypted message will come later, allowing the client - to decrypt it successfully. Olm does not have a way to recover from the failure, - making this session replacement process required. - -To establish a new session, the client sends an `m.dummy <#m-dummy>`_ to-device event -to the other party to notify them of the new session details. - -Clients should rate-limit the number of sessions it creates per device that it receives -a message from. Clients should not create a new session with another device if it has -already created one for that given device in the past 1 hour. - -Clients should attempt to mitigate loss of the undecryptable messages. For example, -Megolm sessions that were sent using the old session would have been lost. The client -can attempt to retrieve the lost sessions through ``m.room_key_request`` messages. - - -``m.megolm.v1.aes-sha2`` -~~~~~~~~~~~~~~~~~~~~~~~~ - -The name ``m.megolm.v1.aes-sha2`` corresponds to version 1 of the Megolm -ratchet, as defined by the `Megolm specification`_. This uses: - -* HMAC-SHA-256 for the hash ratchet. -* HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -* Ed25519 for message authenticity. - -Devices that support Megolm must support Olm, and include "m.megolm.v1.aes-sha2" in -their list of supported messaging algorithms. - -An event encrypted using Megolm has the following format: - -.. code:: json - - { - "type": "m.room.encrypted", - "content": { - "algorithm": "m.megolm.v1.aes-sha2", - "sender_key": "", - "device_id": "", - "session_id": "", - "ciphertext": "" - } - } - -The encrypted payload can contain any message event. The plaintext is of the form: - -.. code:: json - - { - "type": "", - "content": "", - "room_id": "" - } - -We include the room ID in the payload, because otherwise the homeserver would -be able to change the room a message was sent in. - -Clients must guard against replay attacks by keeping track of the ratchet indices -of Megolm sessions. They should reject messages with a ratchet index that they -have already decrypted. Care should be taken in order to avoid false positives, as a -client may decrypt the same event twice as part of its normal processing. - -As with Olm events, clients must confirm that the ``sender_key`` belongs to the user -who sent the message. The same reasoning applies, but the sender ed25519 key has to be -inferred from the ``keys.ed25519`` property of the event which established the Megolm -session. - -In order to enable end-to-end encryption in a room, clients can send an -``m.room.encryption`` state event specifying ``m.megolm.v1.aes-sha2`` as its -``algorithm`` property. - -When creating a Megolm session in a room, clients must share the corresponding session -key using Olm with the intended recipients, so that they can decrypt future messages -encrypted using this session. An ``m.room_key`` event is used to do this. Clients -must also handle ``m.room_key`` events sent by other devices in order to decrypt their -messages. - -Protocol definitions --------------------- - -Events -~~~~~~ - -{{m_room_encryption_event}} - -{{m_room_encrypted_event}} - -{{m_room_key_event}} - -{{m_room_key_request_event}} - -{{m_forwarded_room_key_event}} - -{{m_dummy_event}} - -Key management API -~~~~~~~~~~~~~~~~~~ - -{{keys_cs_http_api}} - - -.. anchor for link from /sync api spec -.. |device_lists_sync| replace:: End-to-end encryption -.. _device_lists_sync: - -Extensions to /sync -~~~~~~~~~~~~~~~~~~~ - -This module adds an optional ``device_lists`` property to the |/sync|_ -response, as specified below. The server need only populate this property for -an incremental ``/sync`` (i.e., one where the ``since`` parameter was -specified). The client is expected to use |/keys/query|_ or |/keys/changes|_ -for the equivalent functionality after an initial sync, as documented in -`Tracking the device list for a user`_. - -It also adds a ``one_time_keys_count`` property. Note the spelling difference -with the ``one_time_key_counts`` property in the |/keys/upload|_ response. - -.. todo: generate this from a swagger definition? - -.. device_lists: { changed: ["@user:server", ... ]}, - -============ =========== ===================================================== -Parameter Type Description -============ =========== ===================================================== -device_lists DeviceLists Optional. Information on e2e device updates. Note: - only present on an incremental sync. -|device_otk| {string: Optional. For each key algorithm, the number of - integer} unclaimed one-time keys currently held on the server - for this device. -============ =========== ===================================================== - -``DeviceLists`` - -========= ========= ============================================= -Parameter Type Description -========= ========= ============================================= -changed [string] List of users who have updated their device identity or - cross-signing keys, - or who now share an encrypted room with the client since - the previous sync response. -left [string] List of users with whom we do not share any encrypted rooms - anymore since the previous sync response. -========= ========= ============================================= - -.. NOTE:: - - For optimal performance, Alice should be added to ``changed`` in Bob's sync only - when she updates her devices or cross-signing keys, or when Alice and Bob now - share a room but didn't - share any room previously. However, for the sake of simpler logic, a server - may add Alice to ``changed`` when Alice and Bob share a new room, even if they - previously already shared a room. - -Example response: - -.. code:: json - - { - "next_batch": "s72595_4483_1934", - "rooms": {"leave": {}, "join": {}, "invite": {}}, - "device_lists": { - "changed": [ - "@alice:example.com", - ], - "left": [ - "@bob:example.com", - ], - }, - "device_one_time_keys_count": { - "curve25519": 10, - "signed_curve25519": 20 - } - } - -Reporting that decryption keys are withheld -------------------------------------------- - -When sending an encrypted event to a room, a client can optionally signal to -other devices in that room that it is not sending them the keys needed to -decrypt the event. In this way, the receiving client can indicate to the user -why it cannot decrypt the event, rather than just showing a generic error -message. - -In the same way, when one device requests keys from another using `Key -requests`_, the device from which the key is being requested may want to tell -the requester that it is purposely not sharing the key. - -If Alice withholds a megolm session from Bob for some messages in a room, and -then later on decides to allow Bob to decrypt later messages, she can send Bob -the megolm session, ratcheted up to the point at which she allows Bob to -decrypt the messages. If Bob logs into a new device and uses key sharing to -obtain the decryption keys, the new device will be sent the megolm sessions -that have been ratcheted up. Bob's old device can include the reason that the -session was initially not shared by including a ``withheld`` property in the -``m.forwarded_room_key`` message that is an object with the ``code`` and -``reason`` properties from the ``m.room_key.withheld`` message. - -{{m_room_key_withheld_event}} - - -.. References - -.. _ed25519: http://ed25519.cr.yp.to/ -.. _curve25519: https://cr.yp.to/ecdh.html -.. _`Olm specification`: http://matrix.org/docs/spec/olm.html -.. _`Megolm specification`: http://matrix.org/docs/spec/megolm.html -.. _`JSON Web Key`: https://tools.ietf.org/html/rfc7517#appendix-A.3 -.. _`W3C extension`: https://w3c.github.io/webcrypto/#iana-section-jwk -.. _`PBKDF2`: https://tools.ietf.org/html/rfc2898#section-5.2 - -.. _`Signing JSON`: ../appendices.html#signing-json - -.. |m.olm.v1.curve25519-aes-sha2| replace:: ``m.olm.v1.curve25519-aes-sha2`` -.. |device_otk| replace:: device_one_time_keys_count - -.. |/keys/upload| replace:: ``/keys/upload`` -.. _/keys/upload: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-upload - -.. |/keys/query| replace:: ``/keys/query`` -.. _/keys/query: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-query - -.. |/keys/claim| replace:: ``/keys/claim`` -.. _/keys/claim: #post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-claim - -.. |/keys/changes| replace:: ``/keys/changes`` -.. _/keys/changes: #get-matrix-client-%CLIENT_MAJOR_VERSION%-keys-changes diff --git a/specification/modules/event_context.rst b/specification/modules/event_context.rst deleted file mode 100644 index 2d5f22b1..00000000 --- a/specification/modules/event_context.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Event Context -============= - -.. _module:event-context: - -This API returns a number of events that happened just before and after the -specified event. This allows clients to get the context surrounding an event. - -Client behaviour ----------------- - -There is a single HTTP API for retrieving event context, documented below. - -{{event_context_cs_http_api}} - -Security considerations ------------------------ - -The server must only return results that the user has permission to see. diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst deleted file mode 100644 index fde38a2a..00000000 --- a/specification/modules/guest_access.rst +++ /dev/null @@ -1,103 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Guest Access -============ - -.. _module:guest-access: - -There are times when it is desirable for clients to be able to interact with -rooms without having to fully register for an account on a homeserver or join -the room. This module specifies how these clients should interact with servers -in order to participate in rooms as guests. - -Guest users retrieve access tokens from a homeserver using the ordinary -`register endpoint <#post-matrix-client-%CLIENT_MAJOR_VERSION%-register>`_, specifying -the ``kind`` parameter as ``guest``. They may then interact with the -client-server API as any other user would, but will only have access to a subset -of the API as described the Client behaviour subsection below. -Homeservers may choose not to allow this access at all to their local users, but -have no information about whether users on other homeservers are guests or not. - -Guest users can also upgrade their account by going through the ordinary -``register`` flow, but specifying the additional POST parameter -``guest_access_token`` containing the guest's access token. They are also -required to specify the ``username`` parameter to the value of the local part of -their username, which is otherwise optional. - -This module does not fully factor in federation; it relies on individual -homeservers properly adhering to the rules set out in this module, rather than -allowing all homeservers to enforce the rules on each other. - -Events ------- -{{m_room_guest_access_event}} - -Client behaviour ----------------- -The following API endpoints are allowed to be accessed by guest accounts for -retrieving events: - -* `GET /rooms/:room_id/state <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-state>`_ -* `GET /rooms/:room_id/context/:event_id <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-context-eventid>`_ -* `GET /rooms/:room_id/event/:event_id <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-event-eventid>`_ -* `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-state-eventtype-statekey>`_ -* `GET /rooms/:room_id/messages <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-messages>`_ -* `GET /rooms/:room_id/members <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-members>`_ -* `GET /rooms/:room_id/initialSync <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-initialsync>`_ -* `GET /sync <#get-matrix-client-%CLIENT_MAJOR_VERSION%-sync>`_ -* `GET /events`__ as used for room previews. - -__ `peeking_events_api`_ - -The following API endpoints are allowed to be accessed by guest accounts for -sending events: - -* `POST /rooms/:room_id/join <#post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-join>`_ -* `POST /rooms/:room_id/leave <#post-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-leave>`_ -* `PUT /rooms/:room_id/send/m.room.message/:txn_id <#put-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-send-eventtype-txnid>`_ -* `PUT /sendToDevice/{eventType}/{txnId} <#put-matrix-client-%CLIENT_MAJOR_VERSION%-sendtodevice-eventtype-txnid>`_ - -The following API endpoints are allowed to be accessed by guest accounts for -their own account maintenance: - -* `PUT /profile/:user_id/displayname <#put-matrix-client-%CLIENT_MAJOR_VERSION%-profile-userid-displayname>`_ -* `GET /devices <#get-matrix-client-%CLIENT_MAJOR_VERSION%-devices>`_ -* `GET /devices/{deviceId} <#get-matrix-client-%CLIENT_MAJOR_VERSION%-devices-deviceid>`_ -* `PUT /devices/{deviceId} <#put-matrix-client-%CLIENT_MAJOR_VERSION%-devices-deviceid>`_ - -The following API endpoints are allowed to be accessed by guest accounts for -end-to-end encryption: - -* `POST /keys/upload <#post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-upload>`_ -* `POST /keys/query <#post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-query>`_ -* `POST /keys/claim <#post-matrix-client-%CLIENT_MAJOR_VERSION%-keys-claim>`_ - -Server behaviour ----------------- -Servers MUST only allow guest users to join rooms if the ``m.room.guest_access`` -state event is present on the room, and has the ``guest_access`` value -``can_join``. If the ``m.room.guest_access`` event is changed to stop this from -being the case, the server MUST set those users' ``m.room.member`` state to -``leave``. - -Security considerations ------------------------ -Each homeserver manages its own guest accounts itself, and whether an account -is a guest account or not is not information passed from server to server. -Accordingly, any server participating in a room is trusted to properly enforce -the permissions outlined in this section. - -Homeservers may want to enable protections such as captchas for guest -registration to prevent spam, denial of service, and similar attacks. diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst deleted file mode 100644 index 84435edb..00000000 --- a/specification/modules/history_visibility.rst +++ /dev/null @@ -1,101 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room History Visibility -======================= - -.. _module:history-visibility: - -This module adds support for controlling the visibility of previous events in a -room. - -In all cases except ``world_readable``, a user needs to join a room to view events in that room. Once they -have joined a room, they will gain access to a subset of events in the room. How -this subset is chosen is controlled by the ``m.room.history_visibility`` event -outlined below. After a user has left a room, they may see any events which they -were allowed to see before they left the room, but no events received after they -left. - -The four options for the ``m.room.history_visibility`` event are: - -- ``world_readable`` - All events while this is the - ``m.room.history_visibility`` value may be shared by any participating - homeserver with anyone, regardless of whether they have ever joined the room. -- ``shared`` - Previous events are always accessible to newly joined members. All - events in the room are accessible, even those sent when the member was not a part - of the room. -- ``invited`` - Events are accessible to newly joined members from the point - they were invited onwards. Events stop being accessible when the member's state - changes to something other than ``invite`` or ``join``. -- ``joined`` - Events are accessible to newly joined members from the point - they joined the room onwards. Events stop being accessible when the member's state - changes to something other than ``join``. - -.. WARNING:: - These options are applied at the point an event is *sent*. Checks are - performed with the state of the ``m.room.history_visibility`` event when the - event in question is added to the DAG. This means clients cannot - retrospectively choose to show or hide history to new users if the setting at - that time was more restrictive. - -Events ------- - -{{m_room_history_visibility_event}} - -Client behaviour ----------------- - -Clients that implement this module MUST present to the user the possible options -for setting history visibility when creating a room. - -Clients may want to display a notice that their events may be read by non-joined -people if the value is set to ``world_readable``. - -Server behaviour ----------------- - -By default if no ``history_visibility`` is set, or if the value is not understood, the visibility is assumed to be -``shared``. The rules governing whether a user is allowed to see an event depend -on the state of the room *at that event*. - -1. If the ``history_visibility`` was set to ``world_readable``, allow. -2. If the user's ``membership`` was ``join``, allow. -3. If ``history_visibility`` was set to ``shared``, and the user joined the - room at any point after the event was sent, allow. -4. If the user's ``membership`` was ``invite``, and the ``history_visibility`` - was set to ``invited``, allow. -5. Otherwise, deny. - -For ``m.room.history_visibility`` events themselves, the user should be allowed -to see the event if the ``history_visibility`` before *or* after the event -would allow them to see it. (For example, a user should be able to see -``m.room.history_visibility`` events which change the ``history_visibility`` -from ``world_readable`` to ``joined`` *or* from ``joined`` to -``world_readable``, even if that user was not a member of the room.) - -Likewise, for the user's own ``m.room.member`` events, the user should be -allowed to see the event if their ``membership`` before *or* after the event -would allow them to see it. (For example, a user can always see -``m.room.member`` events which set their membership to ``join``, or which -change their membership from ``join`` to any other value, even if -``history_visibility`` is ``joined``.) - -Security considerations ------------------------ - -The default value for ``history_visibility`` is ``shared`` for -backwards-compatibility reasons. Clients need to be aware that by not setting -this event they are exposing all of their room history to anyone in the room. - diff --git a/specification/modules/ignore_users.rst b/specification/modules/ignore_users.rst deleted file mode 100644 index 56a410d1..00000000 --- a/specification/modules/ignore_users.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. Copyright 2018 Travis Ralston -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Ignoring Users -============== - -.. _module:ignore_users: - -With all the communication through Matrix it may be desirable to ignore a -particular user for whatever reason. This module defines how clients and -servers can implement the ignoring of users. - -Events ------- - -{{m_ignored_user_list_event}} - -Client behaviour ----------------- -To ignore a user, effectively blocking them, the client should add the target -user to the ``m.ignored_user_list`` event in their account data using -|/user//account_data/|_. Once ignored, the client will no longer -receive events sent by that user, with the exception of state events. The client -should either hide previous content sent by the newly ignored user or perform -a new ``/sync`` with no previous token. - -Invites to new rooms by ignored users will not be sent to the client. The server -may optionally reject the invite on behalf of the client. - -State events will still be sent to the client, even if the user is ignored. -This is to ensure parts, such as the room name, do not appear different to the -user just because they ignored the sender. - -To remove a user from the ignored users list, remove them from the account data -event. The server will resume sending events from the previously ignored user, -however it should not send events that were missed while the user was ignored. -To receive the events that were sent while the user was ignored the client -should perform a fresh sync. The client may also un-hide any events it previously -hid due to the user becoming ignored. - -Server behaviour ----------------- -Following an update of the ``m.ignored_user_list``, the sync API for all clients -should immediately start ignoring (or un-ignoring) the user. Clients are responsible -for determining if they should hide previously sent events or to start a new sync -stream. - -Servers must still send state events sent by ignored users to clients. - -Servers must not send room invites from ignored users to clients. Servers may -optionally decide to reject the invite, however. diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst deleted file mode 100644 index 704277a1..00000000 --- a/specification/modules/instant_messaging.rst +++ /dev/null @@ -1,515 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Instant Messaging -================= - -.. _module:im: - -This module adds support for sending human-readable messages to a room. It also -adds support for associating human-readable information with the room itself -such as a room name and topic. - -Events ------- - -{{m_room_message_event}} - -{{m_room_message_feedback_event}} - -Usage of this event is discouraged for several reasons: - - The number of feedback events will grow very quickly with the number of users - in the room. This event provides no way to "batch" feedback, unlike the - `receipts module`_. - - Pairing feedback to messages gets complicated when paginating as feedback - arrives before the message it is acknowledging. - - There are no guarantees that the client has seen the event ID being - acknowledged. - - -.. _`receipts module`: `module:receipts`_ - -{{m_room_name_event}} - -{{m_room_topic_event}} - -{{m_room_avatar_event}} - -{{m_room_pinned_events_event}} - -m.room.message msgtypes -~~~~~~~~~~~~~~~~~~~~~~~ - -Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type -of message being sent. Each type has their own required and optional keys, as -outlined below. If a client cannot display the given ``msgtype`` then it SHOULD -display the fallback plain text ``body`` key instead. - -Some message types support HTML in the event content that clients should prefer -to display if available. Currently ``m.text``, ``m.emote``, and ``m.notice`` -support an additional ``format`` parameter of ``org.matrix.custom.html``. When -this field is present, a ``formatted_body`` with the HTML must be provided. The -plain text version of the HTML should be provided in the ``body``. - -Clients should limit the HTML they render to avoid Cross-Site Scripting, HTML -injection, and similar attacks. The strongly suggested set of HTML tags to permit, -denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, -``h2``, ``h3``, ``h4``, ``h5``, ``h6``, ``blockquote``, ``p``, ``a``, ``ul``, -``ol``, ``sup``, ``sub``, ``li``, ``b``, ``i``, ``u``, ``strong``, ``em``, -``strike``, ``code``, ``hr``, ``br``, ``div``, ``table``, ``thead``, ``tbody``, -``tr``, ``th``, ``td``, ``caption``, ``pre``, ``span``, ``img``. - -Not all attributes on those tags should be permitted as they may be avenues for -other disruption attempts, such as adding ``onclick`` handlers or excessively -large text. Clients should only permit the attributes listed for the tags below. -Where ``data-mx-bg-color`` and ``data-mx-color`` are listed, clients should -translate the value (a 6-character hex color code) to the appropriate CSS/attributes -for the tag. - - -:``font``: - ``data-mx-bg-color``, ``data-mx-color`` - -:``span``: - ``data-mx-bg-color``, ``data-mx-color`` - -:``a``: - ``name``, ``target``, ``href`` (provided the value is not relative and has a scheme - matching one of: ``https``, ``http``, ``ftp``, ``mailto``, ``magnet``) - -:``img``: - ``width``, ``height``, ``alt``, ``title``, ``src`` (provided it is a `Matrix Content (MXC) URI`_) - -:``ol``: - ``start`` - -:``code``: - ``class`` (only classes which start with ``language-`` for syntax highlighting) - - -Additionally, web clients should ensure that *all* ``a`` tags get a ``rel="noopener"`` -to prevent the target page from referencing the client's tab/window. - -Tags must not be nested more than 100 levels deep. Clients should only support the subset -of tags they can render, falling back to other representations of the tags where possible. -For example, a client may not be able to render tables correctly and instead could fall -back to rendering tab-delimited text. - -In addition to not rendering unsafe HTML, clients should not emit unsafe HTML in events. -Likewise, clients should not generate HTML that is not needed, such as extra paragraph tags -surrounding text due to Rich Text Editors. HTML included in events should otherwise be valid, -such as having appropriate closing tags, appropriate attributes (considering the custom ones -defined in this specification), and generally valid structure. - -A special tag, ``mx-reply``, may appear on rich replies (described below) and should be -allowed if, and only if, the tag appears as the very first tag in the ``formatted_body``. -The tag cannot be nested and cannot be located after another tag in the tree. Because the -tag contains HTML, an ``mx-reply`` is expected to have a partner closing tag and should -be treated similar to a ``div``. Clients that support rich replies will end up stripping -the tag and its contents and therefore may wish to exclude the tag entirely. - -.. Note:: - A future iteration of the specification will support more powerful and extensible - message formatting options, such as the proposal `MSC1767 `_. - -{{msgtype_events}} - - -Client behaviour ----------------- - -Clients SHOULD verify the structure of incoming events to ensure that the -expected keys exist and that they are of the right type. Clients can discard -malformed events or display a placeholder message to the user. Redacted -``m.room.message`` events MUST be removed from the client. This can either be -replaced with placeholder text (e.g. "[REDACTED]") or the redacted message can -be removed entirely from the messages view. - -Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be -uploaded using the `content repository module`_ where available. The -resulting ``mxc://`` URI can then be used in the ``url`` key. - -Clients MAY include a client generated thumbnail image for an attachment under -a ``info.thumbnail_url`` key. The thumbnail SHOULD also be a ``mxc://`` URI. -Clients displaying events with attachments can either use the client generated -thumbnail or ask its homeserver to generate a thumbnail from the original -attachment using the `content repository module`_. - -.. _`content repository module`: `module:content`_ - -Recommendations when sending messages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the event of send failure, clients SHOULD retry requests using an -exponential-backoff algorithm for a -certain amount of time T. It is recommended that T is no longer than 5 minutes. -After this time, the client should stop retrying and mark the message as "unsent". -Users should be able to manually resend unsent messages. - -Users may type several messages at once and send them all in quick succession. -Clients SHOULD preserve the order in which they were sent by the user. This -means that clients should wait for the response to the previous request before -sending the next request. This can lead to head-of-line blocking. In order to -reduce the impact of head-of-line blocking, clients should use a queue per room -rather than a global queue, as ordering is only relevant within a single room -rather than between rooms. - -Local echo -~~~~~~~~~~ - -Messages SHOULD appear immediately in the message view when a user presses the -"send" button. This should occur even if the message is still sending. This is -referred to as "local echo". Clients SHOULD implement "local echo" of messages. -Clients MAY display messages in a different format to indicate that the server -has not processed the message. This format should be removed when the server -responds. - -Clients need to be able to match the message they are sending with the same -message which they receive from the event stream. The echo of the same message -from the event stream is referred to as "remote echo". Both echoes need to be -identified as the same message in order to prevent duplicate messages being -displayed. Ideally this pairing would occur transparently to the user: the UI -would not flicker as it transitions from local to remote. Flickering can be -reduced through clients making use of the transaction ID they used to send -a particular event. The transaction ID used will be included in the event's -``unsigned`` data as ``transaction_id`` when it arrives through the event stream. - -Clients unable to make use of the transaction ID are likely to experience -flickering when the remote echo arrives on the event stream *before* -the request to send the message completes. In that case the event -arrives before the client has obtained an event ID, making it impossible to -identify it as a remote echo. This results in the client displaying the message -twice for some time (depending on the server responsiveness) before the original -request to send the message completes. Once it completes, the client can take -remedial actions to remove the duplicate event by looking for duplicate event IDs. - - -Calculating the display name for a user -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Clients may wish to show the human-readable display name of a room member as -part of a membership list, or when they send a message. However, different -members may have conflicting display names. Display names MUST be disambiguated -before showing them to the user, in order to prevent spoofing of other users. - -To ensure this is done consistently across clients, clients SHOULD use the -following algorithm to calculate a disambiguated display name for a given user: - -1. Inspect the ``m.room.member`` state event for the relevant user id. -2. If the ``m.room.member`` state event has no ``displayname`` field, or if - that field has a ``null`` value, use the raw user id as the display - name. Otherwise: -3. If the ``m.room.member`` event has a ``displayname`` which is unique among - members of the room with ``membership: join`` or ``membership: invite``, use - the given ``displayname`` as the user-visible display name. Otherwise: -4. The ``m.room.member`` event has a non-unique ``displayname``. This should be - disambiguated using the user id, for example "display name - (@id:homeserver.org)". - - .. TODO-spec - what does it mean for a ``displayname`` to be 'unique'? Are we - case-sensitive? Do we care about homograph attacks? See - https://matrix.org/jira/browse/SPEC-221. - -Developers should take note of the following when implementing the above -algorithm: - -* The user-visible display name of one member can be affected by changes in the - state of another member. For example, if ``@user1:matrix.org`` is present in - a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins - the room, also with ``displayname: Alice``, *both* users must be given - disambiguated display names. Similarly, when one of the users then changes - their display name, there is no longer a clash, and *both* users can be given - their chosen display name. Clients should be alert to this possibility and - ensure that all affected users are correctly renamed. - -* The display name of a room may also be affected by changes in the membership - list. This is due to the room name sometimes being based on user display - names (see `Calculating the display name for a room`_). - -* If the entire membership list is searched for clashing display names, this - leads to an O(N^2) implementation for building the list of room members. This - will be very inefficient for rooms with large numbers of members. It is - recommended that client implementations maintain a hash table mapping from - ``displayname`` to a list of room members using that name. Such a table can - then be used for efficient calculation of whether disambiguation is needed. - - -Displaying membership information with messages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Clients may wish to show the display name and avatar URL of the room member who -sent a message. This can be achieved by inspecting the ``m.room.member`` state -event for that user ID (see `Calculating the display name for a user`_). - -When a user paginates the message history, clients may wish to show the -**historical** display name and avatar URL for a room member. This is possible -because older ``m.room.member`` events are returned when paginating. This can -be implemented efficiently by keeping two sets of room state: old and current. -As new events arrive and/or the user paginates back in time, these two sets of -state diverge from each other. New events update the current state and paginated -events update the old state. When paginated events are processed sequentially, -the old state represents the state of the room *at the time the event was sent*. -This can then be used to set the historical display name and avatar URL. - - -Calculating the display name for a room -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Clients may wish to show a human-readable name for a room. There are a number -of possibilities for choosing a useful name. To ensure that rooms are named -consistently across clients, clients SHOULD use the following algorithm to -choose a name: - -1. If the room has an `m.room.name`_ state event with a non-empty ``name`` - field, use the name given by that field. - -#. If the room has an `m.room.canonical_alias`_ state event with a valid - ``alias`` field, use the alias given by that field as the name. Note that - clients should avoid using ``alt_aliases`` when calculating the room name. - -#. If none of the above conditions are met, a name should be composed based - on the members of the room. Clients should consider `m.room.member`_ events - for users other than the logged-in user, as defined below. - - i. If the number of ``m.heroes`` for the room are greater or equal to - ``m.joined_member_count + m.invited_member_count - 1``, then use the - membership events for the heroes to calculate display names for the - users (`disambiguating them if required`_) and concatenating them. For - example, the client may choose to show "Alice, Bob, and Charlie - (@charlie:example.org)" as the room name. The client may optionally - limit the number of users it uses to generate a room name. - - #. If there are fewer heroes than ``m.joined_member_count + m.invited_member_count - - 1``, and ``m.joined_member_count + m.invited_member_count`` is greater - than 1, the client should use the heroes to calculate display names for - the users (`disambiguating them if required`_) and concatenating them - alongside a count of the remaining users. For example, "Alice, Bob, and - 1234 others". - - #. If ``m.joined_member_count + m.invited_member_count`` is less than or - equal to 1 (indicating the member is alone), the client should use the - rules above to indicate that the room was empty. For example, "Empty - Room (was Alice)", "Empty Room (was Alice and 1234 others)", or - "Empty Room" if there are no heroes. - -Clients SHOULD internationalise the room name to the user's language when using -the ``m.heroes`` to calculate the name. Clients SHOULD use minimum 5 heroes to -calculate room names where possible, but may use more or less to fit better with -their user experience. - -.. _`disambiguating them if required`: `Calculating the display name for a user`_ - -Forming relationships between events -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In some cases, events may wish to reference other events. This could be to form -a thread of messages for the user to follow along with, or to provide more context -as to what a particular event is describing. Currently, the only kind of relation -defined is a "rich reply" where a user may reference another message to create a -thread-like conversation. - -Relationships are defined under an ``m.relates_to`` key in the event's ``content``. -If the event is of the type ``m.room.encrypted``, the ``m.relates_to`` key MUST NOT -be covered by the encryption and instead be put alongside the encryption information -held in the ``content``. - - -Rich replies -++++++++++++ - -Users may wish to reference another message when forming their own message, and -clients may wish to better embed the referenced message for the user to have a -better context for the conversation being had. This sort of embedding another -message in a message is known as a "rich reply", or occasionally just a "reply". - -A rich reply is formed through use of an ``m.relates_to`` relation for ``m.in_reply_to`` -where a single key, ``event_id``, is used to reference the event being replied to. -The referenced event ID SHOULD belong to the same room where the reply is being sent. -Clients should be cautious of the event ID belonging to another room, or being invalid -entirely. Rich replies can only be constructed in the form of ``m.room.message`` events -with a ``msgtype`` of ``m.text`` or ``m.notice``. Due to the fallback requirements, rich -replies cannot be constructed for types of ``m.emote``, ``m.file``, etc. Rich replies -may reference any other ``m.room.message`` event, however. Rich replies may reference -another event which also has a rich reply, infinitely. - -An ``m.in_reply_to`` relationship looks like the following:: - - { - ... - "type": "m.room.message", - "content": { - "msgtype": "m.text", - "body": "", - "format": "org.matrix.custom.html", - "formatted_body": "", - "m.relates_to": { - "m.in_reply_to": { - "event_id": "$another:event.com" - } - } - } - } - - -Fallbacks and event representation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some clients may not have support for rich replies and therefore need a fallback -to use instead. Clients that do not support rich replies should render the event -as if rich replies were not special. - -Clients that do support rich replies MUST provide the fallback format on replies, -and MUST strip the fallback before rendering the reply. Rich replies MUST have -a ``format`` of ``org.matrix.custom.html`` and therefore a ``formatted_body`` -alongside the ``body`` and appropriate ``msgtype``. The specific fallback text -is different for each ``msgtype``, however the general format for the ``body`` is: - -.. code-block:: text - - > <@alice:example.org> This is the original body - - This is where the reply goes - - -The ``formatted_body`` should use the following template: - -.. code-block:: html - - -
- In reply to - @alice:example.org -
- -
- - This is where the reply goes. - - -If the related event does not have a ``formatted_body``, the event's ``body`` should -be considered after encoding any HTML special characters. Note that the ``href`` in -both of the anchors use a `matrix.to URI <../appendices.html#matrix-to-navigation>`_. - -Stripping the fallback -`````````````````````` - -Clients which support rich replies MUST strip the fallback from the event before -rendering the event. This is because the text provided in the fallback cannot be -trusted to be an accurate representation of the event. After removing the fallback, -clients are recommended to represent the event referenced by ``m.in_reply_to`` -similar to the fallback's representation, although clients do have creative freedom -for their user interface. Clients should prefer the ``formatted_body`` over the -``body``, just like with other ``m.room.message`` events. - -To strip the fallback on the ``body``, the client should iterate over each line of -the string, removing any lines that start with the fallback prefix ("> ", -including the space, without quotes) and stopping when a line is encountered without -the prefix. This prefix is known as the "fallback prefix sequence". - -To strip the fallback on the ``formatted_body``, the client should remove the -entirety of the ``mx-reply`` tag. - -Fallback for ``m.text``, ``m.notice``, and unrecognised message types -````````````````````````````````````````````````````````````````````` - -Using the prefix sequence, the first line of the related event's ``body`` should -be prefixed with the user's ID, followed by each line being prefixed with the fallback -prefix sequence. For example:: - - > <@alice:example.org> This is the first line - > This is the second line - - This is the reply - - -The ``formatted_body`` uses the template defined earlier in this section. - -Fallback for ``m.emote`` -```````````````````````` - -Similar to the fallback for ``m.text``, each line gets prefixed with the fallback -prefix sequence. However an asterisk should be inserted before the user's ID, like -so:: - - > * <@alice:example.org> feels like today is going to be a great day - - This is the reply - - -The ``formatted_body`` has a subtle difference for the template where the asterisk -is also inserted ahead of the user's ID: - -.. code-block:: html - - -
- In reply to - * @alice:example.org -
- -
- - This is where the reply goes. - - -Fallback for ``m.image``, ``m.video``, ``m.audio``, and ``m.file`` -`````````````````````````````````````````````````````````````````` - -The related event's ``body`` would be a file name, which may not be very descriptive. -The related event should additionally not have a ``format`` or ``formatted_body`` -in the ``content`` - if the event does have a ``format`` and/or ``formatted_body``, -those fields should be ignored. Because the filename alone may not be descriptive, -the related event's ``body`` should be considered to be ``"sent a file."`` such that -the output looks similar to the following:: - - > <@alice:example.org> sent a file. - - This is the reply - - -.. code-block:: html - - -
- In reply to - @alice:example.org -
- sent a file. -
- - This is where the reply goes. - - -For ``m.image``, the text should be ``"sent an image."``. For ``m.video``, the text -should be ``"sent a video."``. For ``m.audio``, the text should be ``"sent an audio file"``. - - -Server behaviour ----------------- - -Homeservers SHOULD reject ``m.room.message`` events which don't have a -``msgtype`` key, or which don't have a textual ``body`` key, with an HTTP status -code of 400. - -Security considerations ------------------------ - -Messages sent using this module are not encrypted, although end to end encryption is in development (see `E2E module`_). - -Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross-Site -Scripting (XSS) attacks. This includes room names and topics. - -.. _`E2E module`: `module:e2e`_ -.. _`Matrix Content (MXC) URI`: `module:content`_ diff --git a/specification/modules/mentions.rst b/specification/modules/mentions.rst deleted file mode 100644 index dc078f3b..00000000 --- a/specification/modules/mentions.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. Copyright 2018 New Vector Ltd. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -User, room, and group mentions -============================== - -.. _module:mentions: - -This module allows users to mention other users, rooms, and groups within -a room message. This is achieved by including a `matrix.to URI`_ in the HTML -body of an `m.room.message`_ event. This module does not have any server-specific -behaviour to it. - -Mentions apply only to `m.room.message`_ events where the ``msgtype`` is ``m.text``, -``m.emote``, or ``m.notice``. The ``format`` for the event must be ``org.matrix.custom.html`` -and therefore requires a ``formatted_body``. - -To make a mention, reference the entity being mentioned in the ``formatted_body`` -using an anchor, like so:: - - { - "body": "Hello Alice!", - "msgtype": "m.text", - "format": "org.matrix.custom.html", - "formatted_body": "Hello Alice!" - } - - -Client behaviour ----------------- - -In addition to using the appropriate ``matrix.to URI`` for the mention, -clients should use the following guidelines when making mentions in events -to be sent: - -* When mentioning users, use the user's potentially ambiguous display name for - the anchor's text. If the user does not have a display name, use the user's - ID. - -* When mentioning rooms, use the canonical alias for the room. If the room - does not have a canonical alias, prefer one of the aliases listed on the - room. If no alias can be found, fall back to the room ID. In all cases, - use the alias/room ID being linked to as the anchor's text. - -* When referencing groups, use the group ID as the anchor's text. - -The text component of the anchor should be used in the event's ``body`` where -the mention would normally be represented, as shown in the example above. - -Clients should display mentions differently from other elements. For example, -this may be done by changing the background color of the mention to indicate -that it is different from a normal link. - -If the current user is mentioned in a message (either by a mention as defined -in this module or by a push rule), the client should show that mention differently -from other mentions, such as by using a red background color to signify to the -user that they were mentioned. - -When clicked, the mention should navigate the user to the appropriate room, group, -or user information. - - -.. _`matrix.to URI`: ../appendices.html#matrix-to-navigation \ No newline at end of file diff --git a/specification/modules/moderation_policies.rst b/specification/modules/moderation_policies.rst deleted file mode 100644 index 5092c895..00000000 --- a/specification/modules/moderation_policies.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. Copyright 2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Moderation policy lists -======================= - -.. _module:moderation-policies: - -With Matrix being an open network where anyone can participate, a very wide -range of content exists and it is important that users are empowered to select -which content they wish to see, and which content they wish to block. By -extension, room moderators and server admins should also be able to select -which content they do not wish to host in their rooms and servers. - -The protocol's position on this is one of neutrality: it should not be deciding -what content is undesirable for any particular entity and should instead be -empowering those entities to make their own decisions. As such, a generic -framework for communicating "moderation policy lists" or "moderation policy rooms" -is described. Note that this module only describes the data structures and not -how they should be interpreting: the entity making the decisions on filtering -is best positioned to interpret the rules how it sees fit. - -Moderation policy lists are stored as room state events. There are no restrictions -on how the rooms can be configured (they could be public, private, encrypted, etc). - -There are currently 3 kinds of entities which can be affected by rules: ``user``, -``server``, and ``room``. All 3 are described with ``m.policy.rule.`` state -events. The ``state_key`` for a policy rule is an arbitrary string decided by the -sender of the rule. - -Rules contain recommendations and reasons for the rule existing. The ``reason`` -is a human-readable string which describes the ``recommendation``. Currently only -one recommendation, ``m.ban``, is specified. - -``m.ban`` recommendation ------------------------- - -When this recommendation is used, the entities affected by the rule should be -banned from participation where possible. The enforcement of this is deliberately -left as an implementation detail to avoid the protocol imposing its opinion on how -the policy list is to be interpreted. However, a suggestion for a simple implementation -is as follows: - -* Is a ``user`` rule... - - * Applied to a user: The user should be added to the subscriber's ignore list. - * Applied to a room: The user should be banned from the room (either on sight or immediately). - * Applied to a server: The user should not be allowed to send invites to users on the server. - -* Is a ``room`` rule... - - * Applied to a user: The user should leave the room and not join it - (`MSC2270 `_-style ignore). - * Applied to a room: No-op because a room cannot ban itself. - * Applied to a server: The server should prevent users from joining the room and from receiving - invites to it. - -* Is a ``server`` rule... - - * Applied to a user: The user should not receive events or invites from the server. - * Applied to a room: The server is added as a denied server in the ACLs. - * Applied to a server: The subscriber should avoid federating with the server as much as - possible by blocking invites from the server and not sending traffic unless strictly - required (no outbound invites). - -Subscribing to policy lists ---------------------------- - -This is deliberately left as an implementation detail. For implementations using the -Client-Server API, this could be as easy as joining or peeking the room. Joining or peeking -is not required, however: an implementation could poll for updates or use a different -technique for receiving updates to the policy's rules. - -Sharing -------- - -In addition to sharing a direct reference to the room which contains the policy's rules, -plain http or https URLs can be used to share links to the list. When the URL is approached -with a ``Accept: application/json`` header or has ``.json`` appended to the end of the URL, it -should return a JSON object containing a ``room_uri`` property which references the room. -Currently this would be a ``matrix.to`` URI, however in future it could be a Matrix-schemed -URI instead. When not approached with the intent of JSON, the service could return a -user-friendly page describing what is included in the ban list. - -Events ------- - -The ``entity`` described by the state events can contain ``*`` and ``?`` to match zero or more -and one or more characters respectively. Note that rules against rooms can describe a room ID -or room alias - the subscriber is responsible for resolving the alias to a room ID if desired. - -{{m_policy_rule_user_event}} - -{{m_policy_rule_room_event}} - -{{m_policy_rule_server_event}} - -Client behaviour ----------------- -As described above, the client behaviour is deliberately left undefined. - -Server behaviour ----------------- -Servers have no additional requirements placed on them by this module. - -Security considerations ------------------------ -This module could be used to build a system of shared blacklists, which may create -a divide within established communities if not carefully deployed. This may well not -be a suitable solution for all communities. - -Depending on how implementations handle subscriptions, user IDs may be linked to -policy lists and therefore expose the views of that user. For example, a client implementation -which joins the user to the policy room would expose the user's ID to observers of the -policy room. In future, `MSC1228 `_ -and `MSC1777 `_ (or similar) could -help solve this concern. diff --git a/specification/modules/openid.rst b/specification/modules/openid.rst deleted file mode 100644 index 63406719..00000000 --- a/specification/modules/openid.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. Copyright 2018 New Vector Ltd. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -OpenID -====== - -.. _module:openid: - -This module allows users to verify their identity with a third party service. The -third party service does need to be matrix-aware in that it will need to know to -resolve matrix homeservers to exchange the user's token for identity information. - -{{openid_cs_http_api}} diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst deleted file mode 100644 index 53a33550..00000000 --- a/specification/modules/presence.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Presence -======== - -.. _module:presence: - -Each user has the concept of presence information. This encodes: - -* Whether the user is currently online -* How recently the user was last active (as seen by the server) -* Whether a given client considers the user to be currently idle -* Arbitrary information about the user's current status (e.g. "in a meeting"). - -This information is collated from both per-device (``online``, ``idle``, -``last_active``) and per-user (status) data, aggregated by the user's homeserver -and transmitted as an ``m.presence`` event. Presence events are sent to -interested parties where users share a room membership. - -User's presence state is represented by the ``presence`` key, which is an enum -of one of the following: - -- ``online`` : The default state when the user is connected to an event - stream. -- ``unavailable`` : The user is not reachable at this time e.g. they are - idle. -- ``offline`` : The user is not connected to an event stream or is - explicitly suppressing their profile information from being sent. - -Events ------- - -{{presence_events}} - -Client behaviour ----------------- - -Clients can manually set/get their presence using the HTTP APIs listed below. - -{{presence_cs_http_api}} - -Last active ago -~~~~~~~~~~~~~~~ -The server maintains a timestamp of the last time it saw a pro-active event from -the user. A pro-active event may be sending a message to a room or changing -presence state to ``online``. This timestamp is presented via a key called -``last_active_ago`` which gives the relative number of milliseconds since the -pro-active event. - -To reduce the number of presence updates sent to clients the server may include -a ``currently_active`` boolean field when the presence state is ``online``. When -true, the server will not send further updates to the last active time until an -update is sent to the client with either a) ``currently_active`` set to false or -b) a presence state other than ``online``. During this period clients must -consider the user to be currently active, irrespective of the last active time. - -The last active time must be up to date whenever the server gives a presence -event to the client. The ``currently_active`` mechanism should purely be used by -servers to stop sending continuous presence updates, as opposed to disabling -last active tracking entirely. Thus clients can fetch up to date last active -times by explicitly requesting the presence for a given user. - -Idle timeout -~~~~~~~~~~~~ - -The server will automatically set a user's presence to ``unavailable`` if their -last active time was over a threshold value (e.g. 5 minutes). Clients can -manually set a user's presence to ``unavailable``. Any activity that bumps the -last active time on any of the user's clients will cause the server to -automatically set their presence to ``online``. - -Security considerations ------------------------ - -Presence information is shared with all users who share a room with the target -user. In large public rooms this could be undesirable. diff --git a/specification/modules/push.rst b/specification/modules/push.rst deleted file mode 100644 index ec855e6b..00000000 --- a/specification/modules/push.rst +++ /dev/null @@ -1,768 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Push Notifications -================== - -.. _module:push: - -:: - - +--------------------+ +-------------------+ - Matrix HTTP | | | | - Notification Protocol | App Developer | | Device Vendor | - | | | | - +-------------------+ | +----------------+ | | +---------------+ | - | | | | | | | | | | - | Matrix homeserver +-----> Push Gateway +------> Push Provider | | - | | | | | | | | | | - +-^-----------------+ | +----------------+ | | +----+----------+ | - | | | | | | - Matrix | | | | | | - Client/Server API + | | | | | - | | +--------------------+ +-------------------+ - | +--+-+ | - | | <-------------------------------------------+ - +---+ | - | | Provider Push Protocol - +----+ - - Mobile Device or Client - - -This module adds support for push notifications. Homeservers send notifications -of events to user-configured HTTP endpoints. Users may also configure a -number of rules that determine which events generate notifications. These are -all stored and managed by the user's homeserver. This allows user-specific push -settings to be reused between client applications. - -The above diagram shows the flow of push notifications being sent to a handset -where push notifications are submitted via the handset vendor, such as Apple's -APNS or Google's GCM. This happens as follows: - -1. The client app signs in to a homeserver. -2. The client app registers with its vendor's Push Provider and - obtains a routing token of some kind. -3. The mobile app uses the Client/Server API to add a 'pusher', providing the - URL of a specific Push Gateway which is configured for that - application. It also provides the routing token it has acquired from the - Push Provider. -4. The homeserver starts sending HTTP requests to the Push Gateway using the - supplied URL. The Push Gateway relays this notification to - the Push Provider, passing the routing token along with any - necessary private credentials the provider requires to send push - notifications. -5. The Push Provider sends the notification to the device. - -Definitions for terms used in this section are below: - -Push Provider - A push provider is a service managed by the device vendor which can send - notifications directly to the device. Google Cloud Messaging (GCM) and Apple - Push Notification Service (APNS) are two examples of push providers. - -Push Gateway - A push gateway is a server that receives HTTP event notifications from - homeservers and passes them on to a different protocol such as APNS for iOS - devices or GCM for Android devices. Clients inform the homeserver which - Push Gateway to send notifications to when it sets up a Pusher. - -.. _def:pushers: - -Pusher - A pusher is a worker on the homeserver that manages the sending - of HTTP notifications for a user. A user can have multiple pushers: one per - device. - -Push Rule - A push rule is a single rule that states under what *conditions* an event should - be passed onto a push gateway and *how* the notification should be presented. - These rules are stored on the user's homeserver. They are manually configured - by the user, who can create and view them via the Client/Server API. - -Push Ruleset - A push ruleset *scopes a set of rules according to some criteria*. For example, - some rules may only be applied for messages from a particular sender, - a particular room, or by default. The push ruleset contains the entire set - of scopes and rules. - -Client behaviour ----------------- - -Clients MUST configure a Pusher before they will receive push notifications. -There is a single API endpoint for this, as described below. - -{{pusher_cs_http_api}} - -.. _pushers: `def:pushers`_ - -Listing Notifications -~~~~~~~~~~~~~~~~~~~~~ - -A client can retrieve a list of events that it has been notified about. This -may be useful so that users can see a summary of what important messages they -have received. - -{{notifications_cs_http_api}} - -Receiving notifications -~~~~~~~~~~~~~~~~~~~~~~~ - -Servers MUST include the number of unread notifications in a client's ``/sync`` -stream, and MUST update it as it changes. Notifications are determined by the -push rules which apply to an event. - -When the user updates their read receipt (either by using the API or by sending an -event), notifications prior to and including that event MUST be marked as read. - -Push Rules -~~~~~~~~~~ -A push rule is a single rule that states under what *conditions* an event should -be passed onto a push gateway and *how* the notification should be presented. -There are different "kinds" of push rules and each rule has an associated -priority. Every push rule MUST have a ``kind`` and ``rule_id``. The ``rule_id`` -is a unique string within the kind of rule and its' scope: ``rule_ids`` do not -need to be unique between rules of the same kind on different devices. Rules may -have extra keys depending on the value of ``kind``. - -The different ``kind``\ s of rule, in the order that they are checked, are: - -Override Rules ``override`` - The highest priority rules are user-configured overrides. -Content-specific Rules ``content`` - These configure behaviour for (unencrypted) messages that match certain - patterns. Content rules take one parameter: ``pattern``, that gives the glob - pattern to match against. This is treated in the same way as ``pattern`` for - ``event_match``. -Room-specific Rules ``room`` - These rules change the behaviour of all messages for a given room. The - ``rule_id`` of a room rule is always the ID of the room that it affects. -Sender-specific rules ``sender`` - These rules configure notification behaviour for messages from a specific - Matrix user ID. The ``rule_id`` of Sender rules is always the Matrix user - ID of the user whose messages they'd apply to. -Underride rules ``underride`` - These are identical to ``override`` rules, but have a lower priority than - ``content``, ``room`` and ``sender`` rules. - -Rules with the same ``kind`` can specify an ordering priority. This determines -which rule is selected in the event of multiple matches. For example, a rule -matching "tea" and a separate rule matching "time" would both match the sentence -"It's time for tea". The ordering of the rules would then resolve the tiebreak -to determine which rule is executed. Only ``actions`` for highest priority rule -will be sent to the Push Gateway. - -Each rule can be enabled or disabled. Disabled rules never match. If no rules -match an event, the homeserver MUST NOT notify the Push Gateway for that event. -Homeservers MUST NOT notify the Push Gateway for events that the user has sent -themselves. - -Actions -+++++++ -All rules have an associated list of ``actions``. An action affects if and how a -notification is delivered for a matching event. The following actions are defined: - -``notify`` - This causes each matching event to generate a notification. -``dont_notify`` - This prevents each matching event from generating a notification -``coalesce`` - This enables notifications for matching events but activates homeserver - specific behaviour to intelligently coalesce multiple events into a single - notification. Not all homeservers may support this. Those that do not support - it should treat it as the ``notify`` action. -``set_tweak`` - Sets an entry in the ``tweaks`` dictionary key that is sent in the notification - request to the Push Gateway. This takes the form of a dictionary with a - ``set_tweak`` key whose value is the name of the tweak to set. It may also - have a ``value`` key which is the value to which it should be set. - -Actions that have no parameters are represented as a string. Otherwise, they are -represented as a dictionary with a key equal to their name and other keys as -their parameters, e.g. ``{ "set_tweak": "sound", "value": "default" }`` - -Tweaks -^^^^^^ -The ``set_tweak`` action is used to add an entry to the 'tweaks' dictionary -that is sent in the notification request to the Push Gateway. The following -tweaks are defined: - -``sound`` - A string representing the sound to be played when this notification arrives. - A value of ``default`` means to play a default sound. A device may choose to - alert the user by some other means if appropriate, eg. vibration. -``highlight`` - A boolean representing whether or not this message should be highlighted in - the UI. This will normally take the form of presenting the message in a - different colour and/or style. The UI might also be adjusted to draw - particular attention to the room in which the event occurred. If a - ``highlight`` tweak is given with no value, its value is defined to be - ``true``. If no highlight tweak is given at all then the value of - ``highlight`` is defined to be false. - -Tweaks are passed transparently through the homeserver so client applications -and Push Gateways may agree on additional tweaks. For example, a tweak may be -added to specify how to flash the notification light on a mobile device. - -Conditions -++++++++++ - -``override`` and ``underride`` rules MAY have a list of 'conditions'. -All conditions must hold true for an event in order for the rule to match. -A rule with no conditions always matches. The following conditions are defined: - -``event_match`` - This is a glob pattern match on a field of the event. Parameters: - - * ``key``: The dot-separated field of the event to match, e.g. ``content.body`` - * ``pattern``: The glob-style pattern to match against. Patterns with no - special glob characters should be treated as having asterisks - prepended and appended when testing the condition. - -``contains_display_name`` - This matches unencrypted messages where ``content.body`` contains the owner's - display name in that room. This is a separate rule because display names may - change and as such it would be hard to maintain a rule that matched the user's - display name. This condition has no parameters. - -``room_member_count`` - This matches the current number of members in the room. Parameters: - - * ``is``: A decimal integer optionally prefixed by one of, ``==``, ``<``, - ``>``, ``>=`` or ``<=``. A prefix of ``<`` matches rooms where the member - count is strictly less than the given number and so forth. If no prefix is - present, this parameter defaults to ``==``. - -``sender_notification_permission`` - This takes into account the current power levels in the room, ensuring the - sender of the event has high enough power to trigger the notification. - - Parameters: - - * ``key``: A string that determines the power level the sender must have to trigger - notifications of a given type, such as ``room``. Refer to the `m.room.power_levels`_ - event schema for information about what the defaults are and how to interpret the event. - The ``key`` is used to look up the power level required to send a notification type - from the ``notifications`` object in the power level event content. - -Unrecognised conditions MUST NOT match any events, effectively making the push -rule disabled. - -``room``, ``sender`` and ``content`` rules do not have conditions in the same -way, but instead have predefined conditions. In the cases of ``room`` and -``sender`` rules, the ``rule_id`` of the rule determines its behaviour. - - -Predefined Rules -++++++++++++++++ -Homeservers can specify "server-default rules" which operate at a lower priority -than "user-defined rules". The ``rule_id`` for all server-default rules MUST -start with a dot (".") to identify them as "server-default". The following -server-default rules are specified: - - -Default Override Rules -^^^^^^^^^^^^^^^^^^^^^^ - -``.m.rule.master`` -`````````````````` -Matches all events. This can be enabled to turn off all push notifications -other than those generated by override rules set by the user. By default this -rule is disabled. - -Definition - -.. code:: json - - { - "rule_id": ".m.rule.master", - "default": true, - "enabled": false, - "conditions": [], - "actions": [ - "dont_notify" - ] - } - -``.m.rule.suppress_notices`` -```````````````````````````` -Matches messages with a ``msgtype`` of ``notice``. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.suppress_notices", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "event_match", - "key": "content.msgtype", - "pattern": "m.notice", - } - ], - "actions": [ - "dont_notify", - ] - } - -``.m.rule.invite_for_me`` -````````````````````````` -Matches any invites to a new room for this user. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.invite_for_me", - "default": true, - "enabled": true, - "conditions": [ - { - "key": "type", - "kind": "event_match", - "pattern": "m.room.member" - }, - { - "key": "content.membership", - "kind": "event_match", - "pattern": "invite" - }, - { - "key": "state_key", - "kind": "event_match", - "pattern": "[the user's Matrix ID]" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ] - } - -``.m.rule.member_event`` -```````````````````````` - -Matches any ``m.room.member_event``. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.member_event", - "default": true, - "enabled": true, - "conditions": [ - { - "key": "type", - "kind": "event_match", - "pattern": "m.room.member" - } - ], - "actions": [ - "dont_notify" - ] - } - - -``.m.rule.contains_display_name`` -````````````````````````````````` -Matches any message whose content is unencrypted and contains the user's -current display name in the room in which it was sent. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.contains_display_name", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "contains_display_name" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - }, - { - "set_tweak": "highlight" - } - ] - } - - -``.m.rule.tombstone`` -````````````````````` -Matches any state event whose type is ``m.room.tombstone``. This is intended -to notify users of a room when it is upgraded, similar to what an -``@room`` notification would accomplish. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.tombstone", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.tombstone" - }, - { - "kind": "event_match", - "key": "state_key", - "pattern": "" - } - ], - "actions": [ - "notify", - { - "set_tweak": "highlight" - } - ] - } - - -``.m.rule.roomnotif`` -````````````````````` -Matches any message whose content is unencrypted and contains the -text ``@room``, signifying the whole room should be notified of -the event. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.roomnotif", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "event_match", - "key": "content.body", - "pattern": "@room" - }, - { - "kind": "sender_notification_permission", - "key": "room" - } - ], - "actions": [ - "notify", - { - "set_tweak": "highlight" - } - ] - } - - -Default Content Rules -^^^^^^^^^^^^^^^^^^^^^ - -``.m.rule.contains_user_name`` -`````````````````````````````` -Matches any message whose content is unencrypted and contains the local part -of the user's Matrix ID, separated by word boundaries. - -Definition (as a ``content`` rule): - -.. code:: json - - { - "rule_id": ".m.rule.contains_user_name", - "default": true, - "enabled": true, - "pattern": "[the local part of the user's Matrix ID]", - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - }, - { - "set_tweak": "highlight" - } - ] - } - -Default Underride Rules -^^^^^^^^^^^^^^^^^^^^^^^ - -``.m.rule.call`` -```````````````` -Matches any incoming VOIP call. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.call", - "default": true, - "enabled": true, - "conditions": [ - { - "key": "type", - "kind": "event_match", - "pattern": "m.call.invite" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "ring" - } - ] - } - -``.m.rule.encrypted_room_one_to_one`` -````````````````````````````````````` -Matches any encrypted event sent in a room with exactly two members. -Unlike other push rules, this rule cannot be matched against the content -of the event by nature of it being encrypted. This causes the rule to -be an "all or nothing" match where it either matches *all* events that -are encrypted (in 1:1 rooms) or none. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.encrypted_room_one_to_one", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "room_member_count", - "is": "2" - }, - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.encrypted" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ] - } - -``.m.rule.room_one_to_one`` -``````````````````````````` -Matches any message sent in a room with exactly two members. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.room_one_to_one", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "room_member_count", - "is": "2" - }, - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.message" - } - ], - "actions": [ - "notify", - { - "set_tweak": "sound", - "value": "default" - } - ] - } - -``.m.rule.message`` -``````````````````` -Matches all chat messages. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.message", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.message" - } - ], - "actions": [ - "notify" - ] - } - -``.m.rule.encrypted`` -````````````````````` -Matches all encrypted events. Unlike other push rules, this rule cannot -be matched against the content of the event by nature of it being encrypted. -This causes the rule to be an "all or nothing" match where it either -matches *all* events that are encrypted (in group rooms) or none. - -Definition: - -.. code:: json - - { - "rule_id": ".m.rule.encrypted", - "default": true, - "enabled": true, - "conditions": [ - { - "kind": "event_match", - "key": "type", - "pattern": "m.room.encrypted" - } - ], - "actions": [ - "notify" - ] - } - -Push Rules: API -~~~~~~~~~~~~~~~ - -Clients can retrieve, add, modify and remove push rules globally or per-device -using the APIs below. - -{{pushrules_cs_http_api}} - - -Push Rules: Events -~~~~~~~~~~~~~~~~~~ - -When a user changes their push rules a ``m.push_rules`` event is sent to all -clients in the ``account_data`` section of their next ``/sync`` request. The -content of the event is the current push rules for the user. - -{{m_push_rules_event}} - -Examples -++++++++ - -To create a rule that suppresses notifications for the room with ID -``!dj234r78wl45Gh4D:matrix.org``:: - - curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ - '{ - "actions" : ["dont_notify"] - }' - -To suppress notifications for the user ``@spambot:matrix.org``:: - - curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ - '{ - "actions" : ["dont_notify"] - }' - -To always notify for messages that contain the work 'cake' and set a specific -sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``):: - - curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ - '{ - "pattern": "cake", - "actions" : ["notify", {"set_sound":"cakealarm.wav"}] - }' - -To add a rule suppressing notifications for messages starting with 'cake' but -ending with 'lie', superseding the previous rule:: - - curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ - '{ - "pattern": "cake*lie", - "actions" : ["notify"] - }' - -To add a custom sound for notifications messages containing the word 'beer' in -any rooms with 10 members or fewer (with greater importance than the room, -sender and content rules):: - - curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ - '{ - "conditions": [ - {"kind": "event_match", "key": "content.body", "pattern": "beer" }, - {"kind": "room_member_count", "is": "<=10"} - ], - "actions" : [ - "notify", - {"set_sound":"beeroclock.wav"} - ] - }' - -Server behaviour ----------------- - -Push Gateway behaviour ----------------------- - -Recommendations for APNS -~~~~~~~~~~~~~~~~~~~~~~~~ -The exact format for sending APNS notifications is flexible and up to the -client app and its' push gateway to agree on. As APNS requires that the sender -has a private key owned by the app developer, each app must have its own push -gateway. It is recommended that: - -* The APNS token be base64 encoded and used as the pushkey. -* A different app_id be used for apps on the production and sandbox - APS environments. -* APNS push gateways do not attempt to wait for errors from the APNS - gateway before returning and instead to store failures and return - 'rejected' responses next time that pushkey is used. - -Security considerations ------------------------ - -Clients specify the Push Gateway URL to use to send event notifications to. This -URL should be over HTTPS and *never* over HTTP. - -As push notifications will pass through a Push Provider, message content -shouldn't be sent in the push itself where possible. Instead, Push Gateways -should send a "sync" command to instruct the client to get new events from the -homeserver directly. - - -.. _`Push Gateway Specification`: ../push_gateway/%PUSH_GATEWAY_RELEASE_LABEL%.html diff --git a/specification/modules/read_markers.rst b/specification/modules/read_markers.rst deleted file mode 100644 index be06d037..00000000 --- a/specification/modules/read_markers.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Fully read markers -================== - -.. _module:read-markers: - -The history for a given room may be split into three sections: messages the -user has read (or indicated they aren't interested in them), messages the user -might have read some but not others, and messages the user hasn't seen yet. -The "fully read marker" (also known as a "read marker") marks the last event -of the first section, whereas the user's read receipt marks the last event of -the second section. - -Events ------- -The user's fully read marker is kept as an event in the room's `account data`_. -The event may be read to determine the user's current fully read marker location -in the room, and just like other account data events the event will be pushed down -the event stream when updated. - -The fully read marker is kept under an ``m.fully_read`` event. If the event does -not exist on the user's account data, the fully read marker should be considered -to be the user's read receipt location. - -{{m_fully_read_event}} - -Client behaviour ----------------- -The client cannot update fully read markers by directly modifying the ``m.fully_read`` -account data event. Instead, the client must make use of the read markers API -to change the values. - -The read markers API can additionally update the user's read receipt (``m.read``) -location in the same operation as setting the fully read marker location. This is -because read receipts and read markers are commonly updated at the same time, -and therefore the client might wish to save an extra HTTP call. Providing an -``m.read`` location performs the same task as a request to ``/receipt/m.read/$event:example.org``. - -{{read_markers_cs_http_api}} - -Server behaviour ----------------- -The server MUST prevent clients from setting ``m.fully_read`` directly in -room account data. The server must additionally ensure that it treats the -presence of ``m.read`` in the ``/read_markers`` request the same as how it -would for a request to ``/receipt/m.read/$event:example.org``. - -Upon updating the ``m.fully_read`` event due to a request to ``/read_markers``, -the server MUST send the updated account data event through to the client via -the event stream (eg: ``/sync``), provided any applicable filters are also -satisfied. - - -.. _`account data`: #client-config diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst deleted file mode 100644 index 4630091f..00000000 --- a/specification/modules/receipts.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Receipts -======== - -.. _module:receipts: - -This module adds in support for receipts. These receipts are a form of -acknowledgement of an event. This module defines a single acknowledgement: -``m.read`` which indicates that the user has read up to a given event. - -Sending a receipt for each event can result in sending large amounts of traffic -to a homeserver. To prevent this from becoming a problem, receipts are implemented -using "up to" markers. This marker indicates that the acknowledgement applies -to all events "up to and including" the event specified. For example, marking -an event as "read" would indicate that the user had read all events *up to* the -referenced event. See the `Receiving notifications <#receiving-notifications>`_ -section for more information on how read receipts affect notification counts. - -Events ------- -Each ``user_id``, ``receipt_type`` pair must be associated with only a -single ``event_id``. - -{{m_receipt_event}} - -Client behaviour ----------------- - -In ``/sync``, receipts are listed under the ``ephemeral`` array of events -for a given room. New receipts that come down the event streams are deltas -which update existing mappings. Clients should replace older receipt acknowledgements -based on ``user_id`` and ``receipt_type`` pairs. For example:: - - Client receives m.receipt: - user = @alice:example.com - receipt_type = m.read - event_id = $aaa:example.com - - Client receives another m.receipt: - user = @alice:example.com - receipt_type = m.read - event_id = $bbb:example.com - - The client should replace the older acknowledgement for $aaa:example.com with - this one for $bbb:example.com - -Clients should send read receipts when there is some certainty that the event in -question has been **displayed** to the user. Simply receiving an event does not -provide enough certainty that the user has seen the event. The user SHOULD need -to *take some action* such as viewing the room that the event was sent to or -dismissing a notification in order for the event to count as "read". Clients -SHOULD NOT send read receipts for events sent by their own user. - -A client can update the markers for its user by interacting with the following -HTTP APIs. - -{{receipts_cs_http_api}} - -Server behaviour ----------------- - -For efficiency, receipts SHOULD be batched into one event per room before -delivering them to clients. - -Receipts are sent across federation as EDUs with type ``m.receipt``. The -format of the EDUs are:: - - { - : { - : { - : { } - }, - ... - }, - ... - } - -These are always sent as deltas to previously sent receipts. Currently only a -single ```` should be used: ``m.read``. - -Security considerations ------------------------ - -As receipts are sent outside the context of the event graph, there are no -integrity checks performed on the contents of ``m.receipt`` events. diff --git a/specification/modules/report_content.rst b/specification/modules/report_content.rst deleted file mode 100644 index 5eca69cd..00000000 --- a/specification/modules/report_content.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. Copyright 2018 Travis Ralston -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Reporting Content -================= - -.. _module:report_content: - -Users may encounter content which they find inappropriate and should be able -to report it to the server administrators or room moderators for review. This -module defines a way for users to report content. - -Content is reported based upon a negative score, where -100 is "most offensive" -and 0 is "inoffensive". - -Client behaviour ----------------- -{{report_content_cs_http_api}} - -Server behaviour ----------------- -Servers are free to handle the reported content however they desire. This may -be a dedicated room to alert server administrators to the reported content or -some other mechanism for notifying the appropriate people. diff --git a/specification/modules/room_previews.rst b/specification/modules/room_previews.rst deleted file mode 100644 index f40c8972..00000000 --- a/specification/modules/room_previews.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Previews -============= - -.. _module:room-previews: - -It is sometimes desirable to offer a preview of a room, where a user can "lurk" -and read messages posted to the room, without joining the room. This can be -particularly effective when combined with `Guest Access`_. - -Previews are implemented via the ``world_readable`` `Room History Visibility`_. -setting, along with a special version of the -`GET /events <#get-matrix-client-%CLIENT_MAJOR_VERSION%-events>`_ endpoint. - -Client behaviour ----------------- -A client wishing to view a room without joining it should call -`GET /rooms/:room_id/initialSync <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-initialsync>`_, -followed by `GET /events`__. Clients will need to do this -in parallel for each room they wish to view. - -__ `peeking_events_api`_ - -Clients can of course also call other endpoints such as -`GET /rooms/:room_id/messages <#get-matrix-client-%CLIENT_MAJOR_VERSION%-rooms-roomid-messages>`_ -and `GET /search <#get-matrix-client-%CLIENT_MAJOR_VERSION%-search>`_ to access -events outside the ``/events`` stream. - -.. _peeking_events_api: - -{{peeking_events_cs_http_api}} - -Server behaviour ----------------- -For clients which have not joined a room, servers are required to only return -events where the room state at the event had the ``m.room.history_visibility`` -state event present with ``history_visibility`` value ``world_readable``. - -Security considerations ------------------------ -Clients may wish to display to their users that rooms which are -``world_readable`` *may* be showing messages to non-joined users. There is no -way using this module to find out whether any non-joined guest users *do* see -events in the room, or to list or count any lurking users. - diff --git a/specification/modules/room_upgrades.rst b/specification/modules/room_upgrades.rst deleted file mode 100644 index f1861f72..00000000 --- a/specification/modules/room_upgrades.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. Copyright 2019 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Upgrades -============= - -.. _module:room-upgrades: - -From time to time, a room may need to be upgraded to a different room version for a -variety for reasons. This module defines a way for rooms to upgrade to a different -room version when needed. - -Events ------- - -{{m_room_tombstone_event}} - -Client behaviour ----------------- - -Clients which understand ``m.room.tombstone`` events and the ``predecessor`` field on -``m.room.create`` events should communicate to the user that the room was upgraded. -One way of accomplishing this would be hiding the old room from the user's room list -and showing banners linking between the old and new room - ensuring that permalinks -work when referencing the old room. Another approach may be to virtually merge the -rooms such that the old room's timeline seamlessly continues into the new timeline -without the user having to jump between the rooms. - -{{room_upgrades_cs_http_api}} - -Server behaviour ----------------- - -When the client requests to upgrade a known room to a known version, the server: - -1. Checks that the user has permission to send ``m.room.tombstone`` events in the room. -2. Creates a replacement room with a ``m.room.create`` event containing a ``predecessor`` - field and the applicable ``room_version``. -3. Replicates transferable state events to the new room. The exact details for what is - transferred is left as an implementation detail, however the recommended state events - to transfer are: - - * ``m.room.server_acl`` - * ``m.room.encryption`` - * ``m.room.name`` - * ``m.room.avatar`` - * ``m.room.topic`` - * ``m.room.guest_access`` - * ``m.room.history_visibility`` - * ``m.room.join_rules`` - * ``m.room.power_levels`` - - Membership events should not be transferred to the new room due to technical limitations - of servers not being able to impersonate people from other homeservers. Additionally, - servers should not transfer state events which are sensitive to who sent them, such as - events outside of the Matrix namespace where clients may rely on the sender to match - certain criteria. - -4. Moves any local aliases to the new room. -5. Sends a ``m.room.tombstone`` event to the old room to indicate that it is not intended - to be used any further. -6. If possible, the power levels in the old room should also be modified to prevent sending - of events and inviting new users. For example, setting ``events_default`` and ``invite`` - to the greater of ``50`` and ``users_default + 1``. - -When a user joins the new room, the server should automatically transfer/replicate some of -the user's personalized settings such as notifications, tags, etc. diff --git a/specification/modules/search.rst b/specification/modules/search.rst deleted file mode 100644 index 08926552..00000000 --- a/specification/modules/search.rst +++ /dev/null @@ -1,109 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Server Side Search -================== - -.. _module:search: - -The search API allows clients to perform full text search across events in all -rooms that the user has been in, including those that they have left. Only -events that the user is allowed to see will be searched, e.g. it won't include -events in rooms that happened after you left. - -Client behaviour ----------------- -There is a single HTTP API for performing server-side search, documented below. - -{{search_cs_http_api}} - -Search Categories ------------------ - -The search API allows clients to search in different categories of items. -Currently the only specified category is ``room_events``. - -``room_events`` -~~~~~~~~~~~~~~~ - -This category covers all events that the user is allowed to see, including -events in rooms that they have left. The search is performed on certain keys of -certain event types. - -The supported keys to search over are: - -- ``content.body`` in ``m.room.message`` -- ``content.name`` in ``m.room.name`` -- ``content.topic`` in ``m.room.topic`` - -The search will *not* include rooms that are end to end encrypted. - -The results include a ``rank`` key that can be used to sort the results by -relevancy. The higher the ``rank`` the more relevant the result is. - -The value of ``count`` gives an approximation of the total number of -results. Homeservers may give an estimate rather than an exact value for this -field. - -Ordering --------- - -The client can specify the ordering that the server returns results in. The two -allowed orderings are: - -- ``rank``, which returns the most relevant results first. -- ``recent``, which returns the most recent results first. - -The default ordering is ``rank``. - -Groups ------- - -The client can request that the results are returned along with grouping -information, e.g. grouped by ``room_id``. In this case the response will -contain a group entry for each distinct value of ``room_id``. Each group entry -contains at least a list of the ``event_ids`` that are in that group, as well -as potentially other metadata about the group. - -The current required supported groupings are: - -- ``room_id`` -- ``sender`` - - -Pagination ----------- - -The server may return a ``next_batch`` key at various places in the response. -These are used to paginate the results. To fetch more results, the client -should send the *same* request to the server with a ``next_batch`` query -parameter set to that of the token. - -The scope of the pagination is defined depending on where the ``next_batch`` -token was returned. For example, using a token inside a group will return more -results from within that group. - -The currently supported locations for the ``next_batch`` token are: - -- ``search_categories..next_batch`` -- ``search_categories..groups...next_batch`` - -A server need not support pagination, even if there are more matching results. -In that case, they must not return a ``next_batch`` token in the response. - - -Security considerations ------------------------ -The server must only return results that the user has permission to see. - diff --git a/specification/modules/secrets.rst b/specification/modules/secrets.rst deleted file mode 100644 index 2e8e3886..00000000 --- a/specification/modules/secrets.rst +++ /dev/null @@ -1,361 +0,0 @@ -.. Copyright 2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Secrets -======= - -Clients may have secret information that they wish to be made available to -other authorised clients, but that the server should not be able to see, so the -information must be encrypted as it passes through the server. This can be done -either asynchronously, by storing encrypted data on the server for later -retrieval, or synchronously, by sending messages to each other. - -Each secret has an identifier that is used by clients to refer to the secret -when storing, fetching, requesting, or sharing the secret. Secrets are plain -strings; structured data can be stored by encoding it as a string. - -Storage -------- - -When secrets are stored on the server, they are stored in the user's -`account-data <#module-account-data>`_, using an event type equal to the -secret's identifier. The keys that secrets are encrypted with are described by -data that is also stored in the user's account-data. Users can have multiple -keys, allowing them to control what sets of secrets clients can access, -depending on what keys are given to them. - -Key storage -~~~~~~~~~~~ - -Each key has an ID, and the description of the key is stored in the user's -account_data using the event type ``m.secret_storage.key.[key ID]``. The -contents of the account data for the key will include an ``algorithm`` -property, which indicates the encryption algorithm used, as well as a ``name`` -property, which is a human-readable name. Key descriptions may also have a -``passphrase`` property for generating the key from a user-entered -passphrase, as described in `deriving keys from passphrases`_. - -``KeyDescription`` - -============ =========== ======================================================= -Parameter Type Description -============ =========== ======================================================= -name string **Required.** The name of the key. -algorithm string **Required.** The encryption algorithm to be used for - this key. Currently, only - ``m.secret_storage.v1.aes-hmac-sha2`` is supported. -passphrase string See `deriving keys from passphrases`_ section for a - description of this property. -============ =========== ======================================================= - -Other properties depend on the encryption algorithm, and are described below. - -A key can be marked as the "default" key by setting the user's account_data -with event type ``m.secret_storage.default_key`` to an object that has the ID -of the key as its ``key`` property. The default key will be used to encrypt -all secrets that the user would expect to be available on all their clients. -Unless the user specifies otherwise, clients will try to use the default key to -decrypt secrets. - -Secret storage -~~~~~~~~~~~~~~ - -Encrypted data is stored in the user's account_data using the event type -defined by the feature that uses the data. The account_data will have an -``encrypted`` property that is a map from key ID to an object. The algorithm -from the ``m.secret_storage.key.[key ID]`` data for the given key defines how -the other properties are interpreted, though it's expected that most encryption -schemes would have ``ciphertext`` and ``mac`` properties, where the -``ciphertext`` property is the unpadded base64-encoded ciphertext, and the -``mac`` is used to ensure the integrity of the data. - -``Secret`` - -============ =========== ======================================================= -Parameter Type Description -============ =========== ======================================================= -encrypted {string: **Required.** Map from key ID the encrypted data. The - object} exact format for the encrypted data is dependent on the - key algorithm. See the definition of - ``AesHmacSha2EncryptedData`` in the - `m.secret_storage.v1.aes-hmac-sha2`_ section. -============ =========== ======================================================= - -Example: - -Some secret is encrypted using keys with ID ``key_id_1`` and ``key_id_2``: - -``org.example.some.secret``: - -.. code:: json - - { - "encrypted": { - "key_id_1": { - "ciphertext": "base64+encoded+encrypted+data", - "mac": "base64+encoded+mac", - // ... other properties according to algorithm property in - // m.secret_storage.key.key_id_1 - }, - "key_id_2": { - // ... - } - } - } - -and the key descriptions for the keys would be: - -``m.secret_storage.key.key_id_1``: - -.. code:: json - - { - "name": "Some key", - "algorithm": "m.secret_storage.v1.aes-hmac-sha2", - // ... other properties according to algorithm - } - -``m.secret_storage.key.key_id_2``: - -.. code:: json - - { - "name": "Some other key", - "algorithm": "m.secret_storage.v1.aes-hmac-sha2", - // ... other properties according to algorithm - } - -``m.secret_storage.v1.aes-hmac-sha2`` -+++++++++++++++++++++++++++++++++++++ - -Secrets encrypted using the ``m.secret_storage.v1.aes-hmac-sha2`` algorithm are -encrypted using AES-CTR-256, and authenticated using HMAC-SHA-256. The secret is -encrypted as follows: - -1. Given the secret storage key, generate 64 bytes by performing an HKDF with - SHA-256 as the hash, a salt of 32 bytes of 0, and with the secret name as - the info. The first 32 bytes are used as the AES key, and the next 32 bytes - are used as the MAC key -2. Generate 16 random bytes, set bit 63 to 0 (in order to work around - differences in AES-CTR implementations), and use this as the AES - initialization vector. This becomes the ``iv`` property, encoded using base64. -3. Encrypt the data using AES-CTR-256 using the AES key generated above. This - encrypted data, encoded using base64, becomes the ``ciphertext`` property. -4. Pass the raw encrypted data (prior to base64 encoding) through HMAC-SHA-256 - using the MAC key generated above. The resulting MAC is base64-encoded and - becomes the ``mac`` property. - -``AesHmacSha2EncryptedData`` - -============ =========== ======================================================= -Parameter Type Description -============ =========== ======================================================= -iv string **Required.** The 16-byte initialization vector, - encoded as base64. -ciphertext string **Required.** The AES-CTR-encrypted data, encoded as - base64. -mac string **Required.** The MAC, encoded as base64. -============ =========== ======================================================= - -For the purposes of allowing clients to check whether a user has correctly -entered the key, clients should: - -1. encrypt and MAC a message consisting of 32 bytes of 0 as described above, - using the empty string as the info parameter to the HKDF in step 1. -2. store the ``iv`` and ``mac`` in the ``m.secret_storage.key.[key ID]`` - account-data. - -``AesHmacSha2KeyDescription`` - -============ =========== ======================================================= -Parameter Type Description -============ =========== ======================================================= -name string **Required.** The name of the key. -algorithm string **Required.** The encryption algorithm to be used for - this key. Currently, only - ``m.secret_storage.v1.aes-hmac-sha2`` is supported. -passphrase object See `deriving keys from passphrases`_ section for a - description of this property. -iv string The 16-byte initialization vector, encoded as base64. -mac string The MAC of the result of encrypting 32 bytes of 0, - encoded as base64. -============ =========== ======================================================= - -For example, the ``m.secret_storage.key.key_id`` for a key using this algorithm -could look like: - -.. code:: json - - { - "name": "m.default", - "algorithm": "m.secret_storage.v1.aes-hmac-sha2", - "iv": "random+data", - "mac": "mac+of+encrypted+zeros" - } - -and data encrypted using this algorithm could look like this: - -.. code:: json - - { - "encrypted": { - "key_id": { - "iv": "16+bytes+base64", - "ciphertext": "base64+encoded+encrypted+data", - "mac": "base64+encoded+mac" - } - } - } - -Key representation -++++++++++++++++++ - -When a user is given a raw key for ``m.secret_storage.v1.aes-hmac-sha2``, -it will be presented as a string constructed as follows: - -1. The key is prepended by the two bytes ``0x8b`` and ``0x01`` -2. All the bytes in the string above, including the two header bytes, are - XORed together to form a parity byte. This parity byte is appended to the byte - string. -3. The byte string is encoded using base58, using the same `mapping as is used - for Bitcoin addresses - `_, - that is, using the alphabet - ``123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz``. -4. The string is formatted into groups of four characters separated by spaces. - -When decoding a raw key, the process should be reversed, with the exception -that whitespace is insignificant in the user's input. - -Deriving keys from passphrases -++++++++++++++++++++++++++++++ - -A user may wish to use a chosen passphrase rather than a randomly generated -key. In this case, information on how to generate the key from a passphrase -will be stored in the ``passphrase`` property of the ``m.secret_storage.key.[key -ID]`` account-data. The ``passphrase`` property has an ``algorithm`` property -that indicates how to generate the key from the passphrase. Other properties of -the ``passphrase`` property are defined by the ``algorithm`` specified. - -``m.pbkdf2`` -<<<<<<<<<<<< - -For the ``m.pbkdf2`` algorithm, the ``passphrase`` property has the following -properties: - -============ =========== ======================================================== -Parameter Type Description -============ =========== ======================================================== -algorithm string **Required.** Must be ``m.pbkdf2`` -salt string **Required.** The salt used in PBKDF2. -iterations integer **Required.** The number of iterations to use in PBKDF2. -bits integer Optional. The number of bits to generate for the key. - Defaults to 256. -============ =========== ======================================================== - -The key is generated using PBKDF2 with SHA-512 as the hash, using the salt -given in the ``salt`` parameter, and the number of iterations given in the -``iterations`` parameter. - -Example: - -.. code:: json - - { - "passphrase": { - "algorithm": "m.pbkdf2", - "salt": "MmMsAlty", - "iterations": 100000, - "bits": 256 - }, - ... - } - -Sharing -------- - -To request a secret from other devices, a client sends an ``m.secret.requests`` -device event with ``action`` set to ``request`` and ``name`` set to the -identifier of the secret. A device that wishes to share the secret will reply -with an ``m.secret.send`` event, encrypted using olm. When the original client -obtains the secret, it sends an ``m.secret.request`` event with ``action`` set -to ``request_cancellation`` to all devices other than the one that it received -the secret from. Clients should ignore ``m.secret.send`` events received from -devices that it did not send an ``m.secret.request`` event to. - -Clients must ensure that they only share secrets with other devices that are -allowed to see them. For example, clients should only share secrets with the -user’s own devices that are verified and may prompt the user to confirm sharing -the secret. - -Event definitions -~~~~~~~~~~~~~~~~~ - -``m.secret.request`` -++++++++++++++++++++ - -Sent by a client to request a secret from another device or to cancel a -previous request. It is sent as an unencrypted to-device event. - -.. table:: - :widths: auto - - ===================== =========== ===================================================== - Parameter Type Description - ===================== =========== ===================================================== - name string Required if ``action`` is ``request``. The name of - the secret that is being requested. - action enum **Required.** One of ["request", "request_cancellation"]. - requesting_device_id string **Required.** The ID of the device requesting the secret. - request_id string **Required.** A random string uniquely identifying (with - respect to the requester and the target) the target - for a secret. If the secret is requested from - multiple devices at the same time, the same ID may - be used for every target. The same ID is also used - in order to cancel a previous request. - ===================== =========== ===================================================== - -Example: - -.. code:: json - - { - "name": "org.example.some.secret", - "action": "request", - "requesting_device_id": "ABCDEFG", - "request_id": "randomly_generated_id_9573" - } - -``m.secret.send`` -+++++++++++++++++ - -Sent by a client to share a secret with another device, in response to an -``m.secret.request`` event. It must be encrypted as an ``m.room.encrypted`` event, -then sent as a to-device event. - -============ =========== ======================================================== -Parameter Type Description -============ =========== ======================================================== -request_id string **Required.** The ID of the request that this a response to. -secret string **Required.** The contents of the secret. -============ =========== ======================================================== - -Example: - -.. code:: json - - { - "request_id": "randomly_generated_id_9573", - "secret": "ThisIsASecretDon'tTellAnyone" - } diff --git a/specification/modules/send_to_device.rst b/specification/modules/send_to_device.rst deleted file mode 100644 index 7ab622bc..00000000 --- a/specification/modules/send_to_device.rst +++ /dev/null @@ -1,150 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -.. _module:to_device: -.. _`to-device`: - -Send-to-Device messaging -======================== - -This module provides a means by which clients can exchange signalling messages -without them being stored permanently as part of a shared communication -history. A message is delivered exactly once to each client device. - -The primary motivation for this API is exchanging data that is meaningless or -undesirable to persist in the room DAG - for example, one-time authentication -tokens or key data. It is not intended for conversational data, which should be -sent using the normal |/rooms//send|_ API for consistency throughout -Matrix. - -Client behaviour ----------------- -To send a message to other devices, a client should call |/sendToDevice|_. -Only one message can be sent to each device per transaction, and they must all -have the same event type. The device ID in the request body can be set to ``*`` -to request that the message be sent to all known devices. - -If there are send-to-device messages waiting for a client, they will be -returned by |/sync|_, as detailed in |Extensions|_. Clients should -inspect the ``type`` of each returned event, and ignore any they do not -understand. - -.. |Extensions| replace:: Extensions to /sync -.. _Extensions: `send_to_device_sync`_ - -Server behaviour ----------------- -Servers should store pending messages for local users until they are -successfully delivered to the destination device. When a client calls |/sync|_ -with an access token which corresponds to a device with pending messages, the -server should list the pending messages, in order of arrival, in the response -body. - -When the client calls ``/sync`` again with the ``next_batch`` token from the -first response, the server should infer that any send-to-device messages in -that response have been delivered successfully, and delete them from the store. - -If there is a large queue of send-to-device messages, the server should -limit the number sent in each ``/sync`` response. 100 messages is recommended -as a reasonable limit. - -If the client sends messages to users on remote domains, those messages should -be sent on to the remote servers via -`federation`_. - -.. _`federation`: ../server_server/%SERVER_RELEASE_LABEL%.html#send-to-device-messaging - -.. TODO-spec: - - * Is a server allowed to delete undelivered messages? After how long? What - about if the device is deleted? - - * If the destination HS doesn't support the ``m.direct_to_device`` EDU, it - will just get dumped. Should we indicate that to the client? - - -Protocol definitions --------------------- - -{{to_device_cs_http_api}} - -.. TODO-spec: - - * What should a server do if the user id or device id is unknown? Presumably - it shouldn't reject the request outright, because some of the destinations - may be valid. Should we add something to the response? - -.. anchor for link from /sync api spec -.. |send_to_device_sync| replace:: Send-to-Device messaging -.. _send_to_device_sync: - -Extensions to /sync -~~~~~~~~~~~~~~~~~~~ - -This module adds the following properties to the |/sync|_ response: - -.. todo: generate this from a swagger definition? - -========= ========= ======================================================= -Parameter Type Description -========= ========= ======================================================= -to_device ToDevice Optional. Information on the send-to-device messages - for the client device. -========= ========= ======================================================= - -``ToDevice`` - -========= ========= ============================================= -Parameter Type Description -========= ========= ============================================= -events [Event] List of send-to-device messages. -========= ========= ============================================= - -``Event`` - -================ ============ ================================================== -Parameter Type Description -================ ============ ================================================== -content EventContent The content of this event. The fields in this - object will vary depending on the type of event. -sender string The Matrix user ID of the user who sent this - event. -type string The type of event. -================ ============ ================================================== - - -Example response: - -.. code:: json - - { - "next_batch": "s72595_4483_1934", - "rooms": {"leave": {}, "join": {}, "invite": {}}, - "to_device": { - "events": [ - { - "sender": "@alice:example.com", - "type": "m.new_device", - "content": { - "device_id": "XYZABCDE", - "rooms": ["!726s6s6q:example.com"] - } - } - ] - } - } - - -.. |/sendToDevice| replace:: ``/sendToDevice`` -.. _/sendToDevice: #put-matrix-client-%CLIENT_MAJOR_VERSION%-sendtodevice-eventtype-txnid diff --git a/specification/modules/server_acls.rst b/specification/modules/server_acls.rst deleted file mode 100644 index 2b2d8f35..00000000 --- a/specification/modules/server_acls.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Server Access Control Lists (ACLs) for rooms -============================================ - -.. _module:server-acls: - -In some scenarios room operators may wish to prevent a malicious or untrusted -server from participating in their room. Sending an `m.room.server_acl`_ state -event into a room is an effective way to prevent the server from participating -in the room at the federation level. - -Server ACLs can also be used to make rooms only federate with a limited set of -servers, or retroactively make the room no longer federate with any other server, -similar to setting the ``m.federate`` value on the `m.room.create`_ event. - -{{m_room_server_acl_event}} - -.. Note:: - Port numbers are not supported because it is unclear to parsers whether a - port number should be matched or an IP address literal. Additionally, it - is unlikely that one would trust a server running on a particular domain's - port but not a different port, especially considering the server host can - easily change ports. - -.. Note:: - CIDR notation is not supported for IP addresses because Matrix does not - encourage the use of IPs for identifying servers. Instead, a blanket - ``allow_ip_literals`` is provided to cover banning them. - -Client behaviour ----------------- -Clients are not expected to perform any additional duties beyond sending the -event. Clients should describe changes to the server ACLs to the user in the -user interface, such as in the timeline. - -Clients may wish to kick affected users from the room prior to denying a server -access to the room to help prevent those servers from participating and to -provide feedback to the users that they have been excluded from the room. - -Server behaviour ----------------- -Servers MUST prevent blacklisted servers from sending events or participating -in the room when an `m.room.server_acl`_ event is present in the room state. -Which APIs are specifically affected are described in the Server-Server API -specification. - -Servers should still send events to denied servers if they are still residents -of the room. - - -Security considerations ------------------------ -Server ACLs are only effective if every server in the room honours them. Servers -that do not honour the ACLs may still permit events sent by denied servers into -the room, leaking them to other servers in the room. To effectively enforce an -ACL in a room, the servers that do not honour the ACLs should be denied in the -room as well. \ No newline at end of file diff --git a/specification/modules/server_notices.rst b/specification/modules/server_notices.rst deleted file mode 100644 index 63b7bfc5..00000000 --- a/specification/modules/server_notices.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Server Notices -============== - -.. _module:server-notices: - -Homeserver hosts often want to send messages to users in an official capacity, -or have resource limits which affect a user's ability to use the homeserver. -For example, the homeserver may be limited to a certain number of active users -per month and has exceeded that limit. To communicate this failure to users, -the homeserver would use the Server Notices room. - -The aesthetics of the room (name, topic, avatar, etc) are left as an implementation -detail. It is recommended that the homeserver decorate the room such that it looks -like an official room to users. - -Events ------- -Notices are sent to the client as normal ``m.room.message`` events with a -``msgtype`` of ``m.server_notice`` in the server notices room. Events with -a ``m.server_notice`` ``msgtype`` outside of the server notice room must -be ignored by clients. - -The specified values for ``server_notice_type`` are: - -:``m.server_notice.usage_limit_reached``: - The server has exceeded some limit which requires the server administrator - to intervene. The ``limit_type`` describes the kind of limit reached. - The specified values for ``limit_type`` are: - - :``monthly_active_user``: - The server's number of active users in the last 30 days has exceeded the - maximum. New connections are being refused by the server. What defines - "active" is left as an implementation detail, however servers are encouraged - to treat syncing users as "active". - - -{{m_room_message_m_server_notice_event}} - -Client behaviour ----------------- -Clients can identify the server notices room by the ``m.server_notice`` tag -on the room. Active notices are represented by the `pinned events <#m-room-pinned-events>`_ -in the server notices room. Server notice events pinned in that room should -be shown to the user through special UI and not through the normal pinned -events interface in the client. For example, clients may show warning banners -or bring up dialogs to get the user's attention. Events which are not server -notice events and are pinned in the server notices room should be shown just -like any other pinned event in a room. - -The client must not expect to be able to reject an invite to join the server -notices room. Attempting to reject the invite must result in a -``M_CANNOT_LEAVE_SERVER_NOTICE_ROOM`` error. Servers should not prevent the user -leaving the room after joining the server notices room, however the same error -code must be used if the server will prevent leaving the room. - -Server behaviour ----------------- -Servers should manage exactly 1 server notices room per user. Servers must -identify this room to clients with the ``m.server_notice`` tag. Servers should -invite the target user rather than automatically join them to the server notice -room. - -How servers send notices to clients, and which user they use to send the events, -is left as an implementation detail for the server. diff --git a/specification/modules/sso_login.rst b/specification/modules/sso_login.rst deleted file mode 100644 index 986bed94..00000000 --- a/specification/modules/sso_login.rst +++ /dev/null @@ -1,347 +0,0 @@ -.. Copyright 2019-2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -SSO client login/authentication -=============================== - -.. _module:sso_login: - -Single Sign-On (SSO) is a generic term which refers to protocols which allow -users to log into applications via a single web-based authentication portal. -Examples include OpenID Connect, "Central Authentication Service" (CAS) and SAML. - -This module allows a Matrix homeserver to delegate user authentication to an -external authentication server supporting one of these protocols. In this -process, there are three systems involved: - - * A Matrix client, using the APIs defined this specification, which is seeking - to authenticate a user to a Matrix homeserver. - - * A Matrix homeserver, implementing the APIs defined in this specification, but - which is delegating user authentication to the authentication server. - - * An "authentication server", which is responsible for authenticating the - user. - -This specification is concerned only with communication between the Matrix -client and the homeserver, and is independent of the SSO protocol used to -communicate with the authentication server. Different Matrix homeserver -implementations might support different SSO protocols. - -Clients and homeservers implementing the SSO flow will need to consider both login_ -and `user-interactive authentication`_. The flow is -similar in both cases, but there are slight differences. - -Typically, SSO systems require a single "callback" URI to be configured at the -authentication server. Once the user is authenticated, their browser is -redirected to that URI. It is up to the Matrix homeserver implementation to -implement a suitable endpoint. For example, for CAS authentication the -homeserver should provide a means for the administrator to configure where the -CAS server is and the REST endpoints which consume the ticket. - -Client login via SSO ---------------------- - -An overview of the process is as follows: - -0. The Matrix client calls |GET /login|_ to find the supported login - types, and the homeserver includes a flow with ``"type": "m.login.sso"`` in the - response. - -1. To initiate the ``m.login.sso`` login type, the Matrix client instructs the - user's browser to navigate to the |/login/sso/redirect|_ endpoint on the - user's homeserver. - -2. The homeserver responds with an HTTP redirect to the SSO user interface, - which the browser follows. - -3. The authentication server and the homeserver interact to verify the user's - identity and other authentication information, potentially using a number of - redirects. - -4. The browser is directed to the ``redirectUrl`` provided by the client with - a ``loginToken`` query parameter for the client to log in with. - -5. The client exchanges the login token for an access token by calling the - |/login|_ endpoint with a ``type`` of ``m.login.token``. - -For native applications, typically steps 1 to 4 are carried out by opening an -embedded web view. - -These steps are illustrated as follows:: - - Matrix Client Matrix Homeserver Auth Server - | | | - |-------------(0) GET /login----------->| | - |<-------------login types--------------| | - | | | - | Webview | | - | | | | - |----->| | | - | |--(1) GET /login/sso/redirect-->| | - | |<---------(2) 302---------------| | - | | | | - | |<========(3) Authentication process================>| - | | | | - | |<--(4) redirect to redirectUrl--| | - |<-----| | | - | | | - |---(5) POST /login with login token--->| | - |<-------------access token-------------| | - - -.. Note:: - In the older `r0.4.0 version `_ - of this specification it was possible to authenticate via CAS when the homeserver - provides a ``m.login.cas`` login flow. This specification deprecates the use - of ``m.login.cas`` to instead prefer ``m.login.sso``, which is the same process - with the only change being which redirect endpoint to use: for ``m.login.cas``, use - ``/cas/redirect`` and for ``m.login.sso`` use ``/sso/redirect`` (described below). - The endpoints are otherwise the same. - -Client behaviour -~~~~~~~~~~~~~~~~ - -The client starts the process by instructing the browser to navigate to -|/login/sso/redirect|_ with an appropriate ``redirectUrl``. Once authentication -is successful, the browser will be redirected to that ``redirectUrl``. - -{{sso_login_redirect_cs_http_api}} - -Security considerations -+++++++++++++++++++++++ - -1. CSRF attacks via manipulation of parameters on the ``redirectUrl`` - - Clients should validate any requests to the ``redirectUrl``. In particular, it - may be possible for attackers to falsify any query parameters, leading to - cross-site request forgery (CSRF) attacks. - - For example, consider a web-based client at ``https://client.example.com``, - which wants to initiate SSO login on the homeserver at ``server.example.org``. - It does this by storing the homeserver name in a query parameter for the - ``redirectUrl``: it redirects to - ``https://server.example.org/login/sso/redirect?redirectUrl=https://client.example.com?hs=server.example.org``. - - An attacker could trick a victim into following a link to - ``https://server.example.org/login/sso/redirect?redirectUrl=https://client.example.com?hs=evil.com``, - which would result in the client sending a login token for the victim's - account to the attacker-controlled site ``evil.com``. - - To guard against this, clients MUST NOT store state (such as the address of - the homeserver being logged into) anywhere it can be modified by external - processes. - - Instead, the state could be stored in `localStorage - `_ or - in a cookie. - -2. For added security, clients SHOULD include a unique identifier in the - ``redirectUrl`` and reject any callbacks that do not contain a recognised - identifier, to guard against unsolicited login attempts and replay attacks. - -Server behaviour -~~~~~~~~~~~~~~~~ - -Redirecting to the Authentication server -++++++++++++++++++++++++++++++++++++++++ - -The server should handle -``/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect`` as follows: - -#. It should build a suitable request for the SSO system. - -#. It should store enough state that the flow can be securely resumed after the - SSO process completes. One way to do this is by storing a cookie which is - stored in the user's browser, by adding a ``Set-Cookie`` header to the response. - -#. It should redirect the user's browser to the SSO login page with the - appropriate parameters. - -See also the "Security considerations" below. - -.. TODO-spec: - - It might be nice if the server did some validation of the ``redirectUrl`` - parameter, so that we could check that aren't going to redirect to a non-TLS - endpoint, and to give more meaningful errors in the case of - faulty/poorly-configured clients. - -Handling the callback from the Authentication server -++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Note that there will normally be a single callback URI which is used for both login -and user-interactive authentication: it is up to the homeserver implementation -to distinguish which is taking place. - -The homeserver should validate the response from the SSO system: this may -require additional calls to the authentication server, and/or may require -checking a signature on the response. - -The homeserver then proceeds as follows: - -#. The homeserver MUST map the user details received from the authentication - server to a valid `Matrix user identifier <../appendices.html#user-identifiers>`_. - The guidance in `Mapping from other character sets - <../appendices.html#mapping-from-other-character-sets>`_ may be useful. - -#. If the generated user identifier represents a new user, it should be - registered as a new user. - -#. The homeserver should generate a short-term login token. This is an opaque - token, suitable for use with the ``m.login.token`` type of the |/login|_ - API. The lifetime of this token SHOULD be limited to around five - seconds. - -#. The homeserver adds a query parameter of ``loginToken``, with the value of - the generated login token, to the ``redirectUrl`` given in the - ``/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect`` - request. (Note: ``redirectURL`` may or may not include existing query - parameters. If it already includes one or more ``loginToken`` parameters, - they should be removed before adding the new one.) - -#. The homeserver redirects the user's browser to the URI thus built. - -Security considerations -~~~~~~~~~~~~~~~~~~~~~~~ - -1. Homeservers should ensure that login tokens are not sent to malicious - clients. - - For example, consider a homeserver at ``server.example.org``. An attacker tricks - a victim into following a link to - ``https://server.example.org/login/sso/redirect?redirectUrl=https://evil.com``, - resulting in a login token being sent to the attacker-controlled site - ``evil.com``. This is a form of cross-site request forgery (CSRF). - - To mitigate this, Homeservers SHOULD confirm with the user that they are - happy to grant access to their matrix account to the site named in the - ``redirectUrl``. This can be done either *before* redirecting to the SSO - login page when handling the - ``/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect`` endpoint, or - *after* login when handling the callback from the authentication server. (If - the check is performed before redirecting, it is particularly important that - the homeserver guards against unsolicited authentication attempts as below). - - It may be appropriate to whitelist a set of known-trusted client URLs in - this process. In particular, the homeserver's own `login fallback`_ - implementation could be excluded. - -2. For added security, homeservers SHOULD guard against unsolicited - authentication attempts by tracking pending requests. One way to do this is - to set a cookie when handling - ``/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect``, which is - checked and cleared when handling the callback from the authentication - server. - -SSO during User-Interactive Authentication ------------------------------------------- - -`User-interactive authentication`_ is used by client-server -endpoints which require additional confirmation of the user's identity (beyond -holding an access token). Typically this means that the user must re-enter -their password, but for homeservers which delegate to an SSO server, this means -redirecting to the authentication server during user-interactive auth. - -The implemementation of this is based on the `Fallback`_ mechanism for -user-interactive auth. - -Client behaviour ----------------- - -Clients do not need to take any particular additional steps beyond ensuring -that the fallback mechanism has been implemented, and treating the -``m.login.sso`` authentication type the same as any other unknown type -(i.e. they should open a browser window for -``/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web?session=``. -Once the flow has completed, the client retries the request with the session -only.) - -Server behaviour ----------------- - -Redirecting to the Authentication server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The server should handle -``/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web`` in -much the same way as -``/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect``, which is to say: - -#. It should build a suitable request for the SSO system. - -#. It should store enough state that the flow can be securely resumed after the - SSO process completes. One way to do this is by storing a cookie which is - stored in the user's browser, by adding a ``Set-Cookie`` header to the response. - -#. It should redirect the user's browser to the SSO login page with the - appropriate parameters. - -See also the "Security considerations" below. - -Handling the callback from the Authentication server -++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Note that there will normally be a single callback URI which is used for both login -and user-interactive authentication: it is up to the homeserver implementation -to distinguish which is taking place. - -The homeserver should validate the response from the SSO system: this may -require additional calls to the authentication server, and/or may require -checking a signature on the response. - -The homeserver then returns the `user-interactive authentication fallback -completion`_ page to the user's browser. - -Security considerations -+++++++++++++++++++++++ - -1. Confirming the operation - - The homeserver SHOULD confirm that the user is happy for the operation to go - ahead. The goal of the user-interactive authentication operation is to guard - against a compromised ``access_token`` being used to take over the user's - account. Simply redirecting the user to the SSO system is insufficient, - since they may not realise what is being asked of them, or the SSO system - may even confirm the authentication automatically. - - For example, the homeserver might serve a page with words to the effect of: - - A client is trying to remove a device from your account. To confirm this - action, re-authenticate with single sign-on. If you did not expect this, your - account may be compromised! - - This confirmation could take place before redirecting to the SSO - authentication page (when handling the - ``/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web`` - endpoint), or *after* authentication when handling the callback from the - authentication server. (If the check is performed before redirecting, it is - particularly important that the homeserver guards against unsolicited - authentication attempts as below). - -2. For added security, homeservers SHOULD guard against unsolicited - authentication attempts by tracking pending requests. One way to do this is - to set a cookie when handling - ``/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web``, - which is checked and cleared when handling the callback from the - authentication server. - - - -.. |GET /login| replace:: ``GET /login`` -.. _GET /login: #get-matrix-client-%CLIENT_MAJOR_VERSION%-login -.. |/login| replace:: ``/login`` -.. _/login: #post-matrix-client-%CLIENT_MAJOR_VERSION%-login -.. |/login/sso/redirect| replace:: ``/login/sso/redirect`` -.. _/login/sso/redirect: #get-matrix-client-%CLIENT_MAJOR_VERSION%-login-sso-redirect diff --git a/specification/modules/stickers.rst b/specification/modules/stickers.rst deleted file mode 100644 index 346b0d84..00000000 --- a/specification/modules/stickers.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. Copyright 2018 New Vector Ltd. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Sticker Messages -================ - -.. _module:stickers: - -This module allows users to send sticker messages in to rooms or direct -messaging sessions. - -Sticker messages are specialised image messages that are displayed without -controls (e.g. no "download" link, or light-box view on click, as would be -displayed for for `m.image`_ events). - -Sticker messages are intended to provide simple "reaction" events in the message -timeline. The matrix client should provide some mechanism to display the sticker -"body" e.g. as a tooltip on hover, or in a modal when the sticker image is -clicked. - -Events ------- -Sticker events are received as a single ``m.sticker`` event in the -``timeline`` section of a room, in a ``/sync``. - -{{m_sticker_event}} - -Client behaviour ----------------- - -Clients supporting this message type should display the image content from the -event URL directly in the timeline. - -A thumbnail image should be provided in the ``info`` object. This is -largely intended as a fallback for clients that do not fully support the -``m.sticker`` event type. In most cases it is fine to set the thumbnail URL to the -same URL as the main event content. - -It is recommended that sticker image content should be 512x512 pixels in size -or smaller. The dimensions of the image file should be twice the intended -display size specified in the ``info`` object in order to assist -rendering sharp images on higher DPI screens. diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst deleted file mode 100644 index a5e23770..00000000 --- a/specification/modules/tags.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Tagging -============ - -.. _module:tagging: - -Users can add tags to rooms. Tags are namespaced strings used to label rooms. -A room may have multiple tags. Tags are only visible to the user that set them -but are shared across all their devices. - -Events ------- - -The tags on a room are received as single ``m.tag`` event in the -``account_data`` section of a room. The content of the ``m.tag`` event is a -``tags`` key whose value is an object mapping the name of each tag to another -object. - -The JSON object associated with each tag gives information about the tag, e.g how -to order the rooms with a given tag. - -Ordering information is given under the ``order`` key as a number between 0 and -1. The numbers are compared such that 0 is displayed first. Therefore a room -with an ``order`` of ``0.2`` would be displayed before a room with an ``order`` -of ``0.7``. If a room has a tag without an ``order`` key then it should appear -after the rooms with that tag that have an ``order`` key. - -The name of a tag MUST NOT exceed 255 bytes. - -The tag namespace is defined as follows: - -* The namespace ``m.*`` is reserved for tags defined in the Matrix specification. Clients must ignore - any tags in this namespace they don't understand. -* The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` - is defined to be the display name of this tag. No other semantics should be inferred from tags in - this namespace. -* A client or app willing to use special tags for advanced functionality should namespace them similarly to state keys: ``tld.name.*`` -* Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored -* Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if - the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the - display name directly). These non-namespaced tags are supported for historical reasons. New tags should use - one of the defined namespaces above. - -Several special names are listed in the specification: -The following tags are defined in the ``m.*`` namespace: - -* ``m.favourite``: The user's favourite rooms. These should be shown with higher precedence than other rooms. -* ``m.lowpriority``: These should be shown with lower precedence than others. -* ``m.server_notice``: Used to identify `Server Notice Rooms <#module-server-notices>`_. - -{{m_tag_event}} - -Client Behaviour ----------------- - -{{tags_cs_http_api}} diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst deleted file mode 100644 index 04c3b180..00000000 --- a/specification/modules/third_party_invites.rst +++ /dev/null @@ -1,258 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Third party invites -=================== - -.. _module:third-party-invites: - -This module adds in support for inviting new members to a room where their -Matrix user ID is not known, instead addressing them by a third party identifier -such as an email address. -There are two flows here; one if a Matrix user ID is known for the third party -identifier, and one if not. Either way, the client calls ``/invite`` with the -details of the third party identifier. - -The homeserver asks the identity server whether a Matrix user ID is known for -that identifier: - -- If it is, an invite is simply issued for that user. - -- If it is not, the homeserver asks the identity server to record the details of - the invitation, and to notify the invitee's homeserver of this pending invitation if it gets - a binding for this identifier in the future. The identity server returns a token - and public key to the inviting homeserver. - -When the invitee's homeserver receives the notification of the binding, it -should insert an ``m.room.member`` event into the room's graph for that user, -with ``content.membership`` = ``invite``, as well as a -``content.third_party_invite`` property which contains proof that the invitee -does indeed own that third party identifier. See the `m.room.member <#m-room-member>`_ -schema for more information. - - -Events ------- - -{{m_room_third_party_invite_event}} - -Client behaviour ----------------- - -A client asks a server to invite a user by their third party identifier. - -{{third_party_membership_cs_http_api}} - -Server behaviour ----------------- - -Upon receipt of an ``/invite``, the server is expected to look up the third party -identifier with the provided identity server. If the lookup yields a result for -a Matrix User ID then the normal invite process can be initiated. This process -ends up looking like this: - -:: - - +---------+ +-------------+ +-----------------+ - | Client | | Homeserver | | IdentityServer | - +---------+ +-------------+ +-----------------+ - | | | - | POST /invite | | - |------------------------------------>| | - | | | - | | GET /lookup | - | |--------------------------------------------------->| - | | | - | | User ID result | - | |<---------------------------------------------------| - | | | - | | Invite process for the discovered User ID | - | |------------------------------------------ | - | | | | - | |<----------------------------------------- | - | | | - | Complete the /invite request | | - |<------------------------------------| | - | | | - - -However, if the lookup does not yield a bound User ID, the homeserver must store -the invite on the identity server and emit a valid ``m.room.third_party_invite`` -event to the room. This process ends up looking like this: - -:: - - +---------+ +-------------+ +-----------------+ - | Client | | Homeserver | | IdentityServer | - +---------+ +-------------+ +-----------------+ - | | | - | POST /invite | | - |------------------------------------>| | - | | | - | | GET /lookup | - | |-------------------------------------------------------------->| - | | | - | | "no users" result | - | |<--------------------------------------------------------------| - | | | - | | POST /store-invite | - | |-------------------------------------------------------------->| - | | | - | | Information needed for the m.room.third_party_invite | - | |<--------------------------------------------------------------| - | | | - | | Emit m.room.third_party_invite to the room | - | |------------------------------------------- | - | | | | - | |<------------------------------------------ | - | | | - | Complete the /invite request | | - |<------------------------------------| | - | | | - - -All homeservers MUST verify the signature in the event's -``content.third_party_invite.signed`` object. - -The third party user will then need to verify their identity, which results in -a call from the identity server to the homeserver that bound the third party -identifier to a user. The homeserver then exchanges the ``m.room.third_party_invite`` -event in the room for a complete ``m.room.member`` event for ``membership: invite`` -for the user that has bound the third party identifier. - -If a homeserver is joining a room for the first time because of an -``m.room.third_party_invite``, the server which is already participating in the -room (which is chosen as per the standard server-server specification) MUST -validate that the public key used for signing is still valid, by checking -``key_validity_url`` in the above described way. - -No other homeservers may reject the joining of the room on the basis of -``key_validity_url``, this is so that all homeservers have a consistent view of -the room. They may, however, indicate to their clients that a member's -membership is questionable. - -For example, given H1, H2, and H3 as homeservers, UserA as a user of H1, and an -identity server IS, the full sequence for a third party invite would look like -the following. This diagram assumes H1 and H2 are residents of the room while -H3 is attempting to join. - -:: - - +-------+ +-----------------+ +-----+ +-----+ +-----+ +-----+ - | UserA | | ThirdPartyUser | | H1 | | H2 | | H3 | | IS | - +-------+ +-----------------+ +-----+ +-----+ +-----+ +-----+ - | | | | | | - | POST /invite for ThirdPartyUser | | | | - |----------------------------------->| | | | - | | | | | | - | | | GET /lookup | | | - | | |---------------------------------------------------------------------------------------------->| - | | | | | | - | | | | Lookup results (empty object) | - | | |<----------------------------------------------------------------------------------------------| - | | | | | | - | | | POST /store-invite | | | - | | |---------------------------------------------------------------------------------------------->| - | | | | | | - | | | | Token, keys, etc for third party invite | - | | |<----------------------------------------------------------------------------------------------| - | | | | | | - | | | (Federation) Emit m.room.third_party_invite | | | - | | |----------------------------------------------->| | | - | | | | | | - | Complete /invite request | | | | - |<-----------------------------------| | | | - | | | | | | - | | Verify identity | | | | - | |-------------------------------------------------------------------------------------------------------------------->| - | | | | | | - | | | | | POST /3pid/onbind | - | | | | |<---------------------------| - | | | | | | - | | | PUT /exchange_third_party_invite/:roomId | | - | | |<-----------------------------------------------------------------| | - | | | | | | - | | | Verify the request | | | - | | |------------------- | | | - | | | | | | | - | | |<------------------ | | | - | | | | | | - | | | (Federation) Emit m.room.member for invite | | | - | | |----------------------------------------------->| | | - | | | | | | - | | | | | | - | | | (Federation) Emit the m.room.member event sent to H2 | | - | | |----------------------------------------------------------------->| | - | | | | | | - | | | Complete /exchange_third_party_invite/:roomId request | | - | | |----------------------------------------------------------------->| | - | | | | | | - | | | | | Participate in the room | - | | | | |------------------------ | - | | | | | | | - | | | | |<----------------------- | - | | | | | | - - -Note that when H1 sends the ``m.room.member`` event to H2 and H3 it does not -have to block on either server's receipt of the event. Likewise, H1 may complete -the ``/exchange_third_party_invite/:roomId`` request at the same time as sending -the ``m.room.member`` event to H2 and H3. Additionally, H3 may complete the -``/3pid/onbind`` request it got from IS at any time - the completion is not shown -in the diagram. - -H1 MUST verify the request from H3 to ensure the ``signed`` property is correct -as well as the ``key_validity_url`` as still being valid. This is done by making -a request to the `identity server /isvalid`_ endpoint, using the provided URL -rather than constructing a new one. The query string and response for the provided -URL must match the Identity Service Specification. - -The reason that no other homeserver may reject the event based on checking -``key_validity_url`` is that we must ensure event acceptance is deterministic. -If some other participating server doesn't have a network path to the keyserver, -or if the keyserver were to go offline, or revoke its keys, that other server -would reject the event and cause the participating servers' graphs to diverge. -This relies on participating servers trusting each other, but that trust is -already implied by the server-server protocol. Also, the public key signature -verification must still be performed, so the attack surface here is minimized. - -Security considerations ------------------------ - -There are a number of privacy and trust implications to this module. - -It is important for user privacy that leaking the mapping between a matrix user -ID and a third party identifier is hard. In particular, being able to look up -all third party identifiers from a matrix user ID (and accordingly, being able -to link each third party identifier) should be avoided wherever possible. -To this end, the third party identifier is not put in any event, rather an -opaque display name provided by the identity server is put into the events. -Clients should not remember or display third party identifiers from invites, -other than for the use of the inviter themself. - -Homeservers are not required to trust any particular identity server(s). It is -generally a client's responsibility to decide which identity servers it trusts, -not a homeserver's. Accordingly, this API takes identity servers as input from -end users, and doesn't have any specific trusted set. It is possible some -homeservers may want to supply defaults, or reject some identity servers for -*its* users, but no homeserver is allowed to dictate which identity servers -*other* homeservers' users trust. - -There is some risk of denial of service attacks by flooding homeservers or -identity servers with many requests, or much state to store. Defending against -these is left to the implementer's discretion. - - - -.. _`identity server /isvalid`: ../identity_service/%IDENTITY_RELEASE_LABEL%.html#get-matrix-identity-v2-pubkey-isvalid diff --git a/specification/modules/third_party_networks.rst b/specification/modules/third_party_networks.rst deleted file mode 100644 index cd4ce414..00000000 --- a/specification/modules/third_party_networks.rst +++ /dev/null @@ -1,20 +0,0 @@ -Third Party Networks -==================== - -.. _module:third-party-networks: - -Application services can provide access to third party networks via bridging. -This allows Matrix users to communicate with users on other communication -platforms, with messages ferried back and forth by the application service. A -single application service may bridge multiple third party networks, and many -individual locations within those networks. A single third party network -location may be bridged to multiple Matrix rooms. - -Third Party Lookups -------------------- - -A client may wish to provide a rich interface for joining third party -locations and connecting with third party users. Information necessary for -such an interface is provided by third party lookups. - -{{third_party_lookup_cs_http_api}} \ No newline at end of file diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst deleted file mode 100644 index 964b804a..00000000 --- a/specification/modules/typing_notifications.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Typing Notifications -==================== - -.. _module:typing: - -Users may wish to be informed when another user is typing in a room. This can be -achieved using typing notifications. These are ephemeral events scoped to a -``room_id``. This means they do not form part of the -`Event Graph `_ but still have a ``room_id`` key. - -Events ------- - -{{m_typing_event}} - -Client behaviour ----------------- - -When a client receives an ``m.typing`` event, it MUST use the user ID list to -**REPLACE** its knowledge of every user who is currently typing. The reason for -this is that the server *does not remember* users who are not currently typing -as that list gets big quickly. The client should mark as not typing any user ID -who is not in that list. - -It is recommended that clients store a ``boolean`` indicating whether the user -is typing or not. Whilst this value is ``true`` a timer should fire periodically -every N seconds to send a typing HTTP request. The value of N is recommended to -be no more than 20-30 seconds. This request should be re-sent by the client to -continue informing the server the user is still typing. As subsequent -requests will replace older requests, a safety margin of 5 seconds before the -expected timeout runs out is recommended. When the user stops typing, the -state change of the ``boolean`` to ``false`` should trigger another HTTP request -to inform the server that the user has stopped typing. - -{{typing_cs_http_api}} - -Security considerations ------------------------ - -Clients may not wish to inform everyone in a room that they are typing and -instead only specific users in the room. - diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst deleted file mode 100644 index fab56517..00000000 --- a/specification/modules/voip_events.rst +++ /dev/null @@ -1,116 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Voice over IP -============= - -.. _module:voip: - -This module outlines how two users in a room can set up a Voice over IP (VoIP) -call to each other. Voice and video calls are built upon the WebRTC 1.0 standard. -Call signalling is achieved by sending `message events`_ to the room. In this -version of the spec, only two-party communication is supported (e.g. between two -peers, or between a peer and a multi-point conferencing unit). -This means that clients MUST only send call events to rooms with exactly two -participants. - -.. _message events: `sect:events`_ - -Events ------- - -{{voip_events}} - -Client behaviour ----------------- - -A call is set up with message events exchanged as follows: - -:: - - Caller Callee - [Place Call] - m.call.invite -----------> - m.call.candidate --------> - [..candidates..] --------> - [Answers call] - <--------------- m.call.answer - [Call is active and ongoing] - <--------------- m.call.hangup - -Or a rejected call: - -:: - - Caller Callee - m.call.invite ------------> - m.call.candidate ---------> - [..candidates..] ---------> - [Rejects call] - <-------------- m.call.hangup - -Calls are negotiated according to the WebRTC specification. - -Glare -~~~~~ - -"Glare" is a problem which occurs when two users call each other at roughly the -same time. This results in the call failing to set up as there already is an -incoming/outgoing call. A glare resolution algorithm can be used to determine -which call to hangup and which call to answer. If both clients implement the -same algorithm then they will both select the same call and the call will be -successfully connected. - - -As calls are "placed" to rooms rather than users, the glare resolution algorithm -outlined below is only considered for calls which are to the same room. The -algorithm is as follows: - -- If an ``m.call.invite`` to a room is received whilst the client is - **preparing to send** an ``m.call.invite`` to the same room: - - * the client should cancel its outgoing call and instead - automatically accept the incoming call on behalf of the user. - -- If an ``m.call.invite`` to a room is received **after the client has sent** - an ``m.call.invite`` to the same room and is waiting for a response: - - * the client should perform a lexicographical comparison of the call IDs of - the two calls and use the *lesser* of the two calls, aborting the - greater. If the incoming call is the lesser, the client should accept - this call on behalf of the user. - - -The call setup should appear seamless to the user as if they had simply placed -a call and the other party had accepted. This means any media stream that had been -setup for use on a call should be transferred and used for the call that -replaces it. - -Server behaviour ----------------- - -The homeserver MAY provide a TURN server which clients can use to contact the -remote party. The following HTTP API endpoints will be used by clients in order -to get information about the TURN server. - -{{voip_cs_http_api}} - - -Security considerations ------------------------ - -Calls should only be placed to rooms with one other user in them. If they are -placed to group chat rooms it is possible that another user will intercept and -answer the call. - diff --git a/specification/proposals.rst b/specification/proposals.rst deleted file mode 100644 index 94878f80..00000000 --- a/specification/proposals.rst +++ /dev/null @@ -1,6 +0,0 @@ -Tables of Tracked Proposals ---------------------------- - -This file is generated by an automated process on our build server. - -View the current live version `at https://matrix.org/docs/spec/proposals `_. diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst deleted file mode 100644 index 4749457b..00000000 --- a/specification/proposals_intro.rst +++ /dev/null @@ -1,503 +0,0 @@ -.. raw:: html - - %proposalscssinjection% - -.. title:: Proposals for Spec Changes to Matrix - -.. contents:: Table of Contents -.. sectnum:: - -Proposals for Spec Changes to Matrix ------------------------------------- - -If you are interested in submitting a change to the Matrix Specification, -please take note of the following guidelines. - -Most changes to the Specification require a formal proposal. Bug fixes, typos, -and clarifications to existing behaviour do not need proposals - see the -`contributing guide `_ -for more information on what does and does not need a proposal. - -The proposal process involves some technical writing, having it reviewed by -everyone, having the proposal being accepted, then actually having your ideas -implemented as committed changes to the `Specification repository -`_. - -Meet the `members of the Core Team -`_, a group of -individuals tasked with ensuring the spec process is as smooth and painless as -possible. Members of the Spec Core Team will do their best to participate in -discussion, summarise when things become long-winded, and generally try to act -towards the benefit of everyone. As a majority, team members have the ability -to change the state of a proposal, and individually have the final say in -proposal discussion. - -Guiding Principles ------------------- - -Proposals **must** act to the greater benefit of the entire Matrix ecosystem, -rather than benefiting or privileging any single player or subset of players - -and must not contain any patent encumbered intellectual property. Members of -the Core Team pledge to act as a neutral custodian for Matrix on behalf of the -whole ecosystem. - -For clarity: the Matrix ecosystem is anyone who uses the Matrix protocol. That -includes client users, server admins, client developers, bot developers, -bridge and application service developers, users and admins who are indirectly -using Matrix via 3rd party networks which happen to be bridged, server developers, -room moderators and admins, companies/projects building products or services on -Matrix, spec contributors, translators, and those who created it in -the first place. - -"Greater benefit" could include maximising: - -* the number of end-users reachable on the open Matrix network -* the number of regular users on the Matrix network (e.g. 30-day retained - federated users) -* the number of online servers in the open federation -* the number of developers building on Matrix -* the number of independent implementations which use Matrix -* the number of bridged end-users reachable on the open Matrix network -* the signal-to-noise ratio of the content on the open Matrix network (i.e. minimising spam) -* the ability for users to discover content on their terms (empowering them to select what to see and what not to see) -* the quality and utility of the Matrix spec (as defined by ease and ability - with which a developer can implement spec-compliant clients, servers, bots, - bridges, and other integrations without needing to refer to any other - external material) - -In addition, proposal authors are expected to uphold the following values in -their proposed changes to the Matrix protocol: - -* Supporting the whole long-term ecosystem rather than individual stakeholder gain -* Openness rather than proprietary lock-in -* Interoperability rather than fragmentation -* Cross-platform rather than platform-specific -* Collaboration rather than competition -* Accessibility rather than elitism -* Transparency rather than stealth -* Empathy rather than contrariness -* Pragmatism rather than perfection -* Proof rather than conjecture - -Please `see MSC1779 `_ -for full details of the project's Guiding Principles. - -Technical notes ---------------- - -Proposals **must** develop Matrix as a layered protocol: with new features -building on layers of shared abstractions rather than introducing tight vertical -coupling within the stack. This ensures that new features can evolve rapidly by -building on existing layers and swapping out old features without impacting the -rest of the stack or requiring substantial upgrades to the whole ecosystem. -This is critical for Matrix to rapidly evolve and compete effectively with -centralised systems, despite being a federated protocol. - -For instance, new features should be implemented using the highest layer -abstractions possible (e.g. new event types, which layer on top of the existing -room semantics, and so don't even require any API changes). Failing that, the -next recourse would be backwards-compatible changes to the next layer down (e.g. -room APIs); failing that, considering changes to the format of events or the -DAG; etc. It would be a very unusual feature which doesn't build on the -existing infrastructure provided by the spec and instead created new primitives -or low level APIs. - -Backwards compatibility is very important for Matrix, but not at the expense of -hindering the protocol's evolution. Backwards incompatible changes to endpoints -are allowed when no other alternative exists, and must be versioned under a new -major release of the API. Backwards incompatible changes to the room algorithm -are also allowed when no other alternative exists, and must be versioned under a -new version of the room algorithm. - -There is sometimes a dilemma over where to include higher level features: for -instance, should video conferencing be formalised in the spec, or should it be -implemented via widgets? Should reputation systems be specified? Should search -engine behaviour be specified? - -There is no universal answer to this, but the following guidelines should be -applied: - -1. If the feature would benefit the whole Matrix ecosystem and is aligned with - the guiding principles above, then it should be supported by the spec. -2. If the spec already makes the feature possible without changing any of the - implementations and spec, then it may not need to be added to the spec. -3. However, if the best user experience for a feature does require custom - implementation behaviour then the behaviour should be defined in the spec - such that all implementations may implement it. -4. However, the spec must never add dependencies on unspecified/nonstandardised - 3rd party behaviour. - -As a worked example: - -1. Video conferencing is clearly a feature which would benefit - the whole ecosystem, and so the spec should find a way to make it happen. -2. Video conferencing can be achieved by widgets without requiring any - compulsory changes to clients nor servers to work, and so could be - omitted from the spec. -3. A better experience could be achieved by embedding Jitsi natively into clients - rather than using a widget... -4. ...except that would add a dependency on unspecified/nonstandardised 3rd party - behaviour, so must not be added to the spec. - -Therefore, our two options in the specific case of video conferencing are -either to spec SFU conferencing semantics for WebRTC (or refer to an existing spec -for doing so), or to keep it as a widget-based approach (optionally with widget -extensions specific for more deeply integrating video conferencing use cases). - -As an alternative example: it's very unlikely that "how to visualise Magnetic -Resonance Imaging data over Matrix" would ever be added to the Matrix spec -(other than perhaps a custom event type in a wider standardised Matrix event -registry) given that the spec's existing primitives of file transfer and -extensible events (MSC1767) give excellent tools for transferring and -visualising arbitrary rich data. - -Supporting public search engines are likely to not require custom spec features -(other than possibly better bulk access APIs), given they can be implemented as -clients using the existing CS API. An exception could be API features required -by decentralised search infrastructure (avoiding centralisation of power by -a centralised search engine). - -Features such as reactions, threaded messages, editable messages, -spam/abuse/content filtering (and reputation systems), are all features which -would clearly benefit the whole Matrix ecosystem, and cannot be implemented in an -interoperable way using the current spec; so they necessitate a spec change. - -Process -------- - -The process for submitting a Matrix Spec Change (MSC) Proposal in detail is as -follows: - -- Create a first draft of your proposal using `GitHub-flavored Markdown - `_ - - - In the document, clearly state the problem being solved, and the possible - solutions being proposed for solving it and their respective trade-offs. - - Proposal documents are intended to be as lightweight and flexible as the - author desires; there is no formal template; the intention is to iterate - as quickly as possible to get to a good design. - - However, a `template with suggested headers - `_ - is available to get you started if necessary. - - Take care in creating your proposal. Specify your intended changes, and - give reasoning to back them up. Changes without justification will likely - be poorly received by the community. - -- Fork and make a PR to the `matrix-doc - `_ repository. The ID of your PR - will become the MSC ID for the lifetime of your proposal. - - - The proposal must live in the ``proposals/`` directory with a filename that - follows the format ``1234-my-new-proposal.md`` where ``1234`` is the MSC - ID. - - Your PR description must include a link to the rendered Markdown document - and a summary of the proposal. - - It is often very helpful to link any related MSCs or `matrix-doc issues - `_ to give context - for the proposal. - - Additionally, please be sure to sign off your proposal PR as per the - guidelines listed on `CONTRIBUTING.rst - `_. - -- Gather feedback as widely as possible. - - - The aim is to get maximum consensus towards an optimal solution. Sometimes - trade-offs are required to meet this goal. Decisions should be made to the - benefit of all major use cases. - - A good place to ask for feedback on a specific proposal is - `#matrix-spec:matrix.org `_. - If preferred, an alternative room can be created and advertised in - #matrix-spec:matrix.org. Please also link to the room in your PR - description. - - For additional discussion areas, know that #matrix-dev:matrix.org is - for developers using existing Matrix APIs, #matrix:matrix.org is for users - trying to run Matrix apps (clients & servers) and - #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's - architectural design. - - The point of the spec proposal process is to be collaborative rather than - competitive, and to try to solve the problem in question with the optimal - set of trade-offs. The author should neutrally gather the various - viewpoints and get consensus, but this can sometimes be time-consuming (or - the author may be biased), in which case an impartial 'shepherd' can be - assigned to help guide the proposal through this process instead. A shepherd is - typically a neutral party from the Spec Core Team or an experienced member of - the community. There is no formal process for assignment. Simply ask for a - shepherd to help get your proposal through and one will be assigned based - on availability. Having a shepherd is not a requirement for proposal - acceptance. - -- Members of the Spec Core Team and community will review and discuss the PR in the - comments and in relevant rooms on Matrix. Discussion outside of GitHub should - be summarised in a comment on the PR. -- When a member of the Spec Core Team believes that no new discussion points are - being made, and the proposal has suitable evidence of working (see `implementing a - proposal`_ below), they will propose a motion for a final comment period (FCP), - along with a *disposition* of either merge, close or postpone. This FCP is - provided to allow a short period of time for any invested party to provide a - final objection before a major decision is made. If sufficient reasoning is - given, an FCP can be cancelled. It is often preceded by a comment summarising - the current state of the discussion, along with reasoning for its occurrence. -- A concern can be raised by a Spec Core Team member at any time, which will block - an FCP from beginning. An FCP will only begin when 75% of the members of the - Spec Core Team agree on its outcome, and all existing concerns have been - resolved. -- The FCP will then begin and last for 5 days, giving anyone else some time to - speak up before it concludes. On its conclusion, the disposition of the FCP - will be carried out. If sufficient reasoning against the disposition is - raised, the FCP can be cancelled and the MSC will continue to evolve - accordingly. -- Once the proposal has been accepted and merged, it is time to submit the - actual change to the Specification that your proposal reasoned about. This is - known as a spec PR. However in order for the spec PR to be accepted, an - implementation **must** be shown to prove that it works well in practice. A - link to the implementation should be included in the PR description. In - addition, any significant unforeseen changes to the original idea found - during this process will warrant another MSC. Any minor, non-fundamental - changes are allowed but **must** be documented in the original proposal - document. This ensures that someone reading a proposal in the future doesn't - assume old information wasn't merged into the spec. - - - Similar to the proposal PR, please sign off the spec PR as per the - guidelines on `CONTRIBUTING.rst - `_. - -- Your PR will then be reviewed and hopefully merged on the grounds it is - implemented sufficiently. If so, then give yourself a pat on the back knowing - you've contributed to the Matrix protocol for the benefit of users and - developers alike :) - -The process for handling proposals is shown visually in the following diagram. -Note that the lifetime of a proposal is tracked through the corresponding -labels for each stage on the `matrix-doc -`_ issue and pull request trackers. - -:: - - + + - Proposals | Spec PRs | Additional States - +-------+ | +------+ | +---------------+ - | | - +----------------------+ | +---------+ | +-----------+ - | | | | | | | | - | Proposal | | +------= Spec PR | | | Postponed | - | Drafting and Initial | | | | Missing | | | | - | Feedback Gathering | | | | | | +-----------+ - | | | | +----+----+ | - +----------+-----------+ | | | | +----------+ - | | | v | | | - v | | +-----------------+ | | Closed | - +-------------------+ | | | | | | | - | | | | | Spec PR Created | | +----------+ - | Proposal PR | | | | and In Review | | - | In Review | | | | | | - | | | | +--------+--------+ | - +---------+---------+ | | | | - | | | v | - v | | +-----------+ | - +----------------------+ | | | | | - | | | | | Spec PR | | - | Proposed Final | | | | Merged! | | - | Comment Period | | | | | | - | | | | +-----------+ | - +----------+-----------+ | | | - | | | | - v | | | - +----------------------+ | | | - | | | | | - | Final Comment Period | | | | - | | | | | - +----------+-----------+ | | | - | | | | - v | | | - +----------------------+ | | | - | | | | | - | Final Comment Period | | | | - | Complete | | | | - | | | | | - +----------+-----------+ | | | - | | | | - +-----------------+ | - | | - + + - -Lifetime States ---------------- - -**Note:** All labels are to be placed on the proposal PR. - -=============================== ============================= ==================================== -Name GitHub Label Description -=============================== ============================= ==================================== -Proposal Drafting and Feedback N/A A proposal document which is still work-in-progress but is being shared to incorporate feedback. Please prefix your proposal's title with ``[WIP]`` to make it easier for reviewers to skim their notifications list. -Proposal In Review proposal-in-review A proposal document which is now ready and waiting for review by the Spec Core Team and community -Proposed Final Comment Period proposed-final-comment-period Currently awaiting signoff of a 75% majority of team members in order to enter the final comment period -Final Comment Period final-comment-period A proposal document which has reached final comment period either for merge, closure or postponement -Final Comment Period Complete finished-final-comment-period The final comment period has been completed. Waiting for a demonstration implementation -Spec PR Missing spec-pr-missing The proposal has been agreed, and proven with a demonstration implementation. Waiting for a PR against the Spec -Spec PR In Review spec-pr-in-review The spec PR has been written, and is currently under review -Spec PR Merged merged A proposal with a sufficient working implementation and whose Spec PR has been merged! -Postponed proposal-postponed A proposal that is temporarily blocked or a feature that may not be useful currently but perhaps - sometime in the future -Closed proposal-closed A proposal which has been reviewed and deemed unsuitable for acceptance -Obsolete obsolete A proposal which has been made obsolete by another proposal or decision elsewhere. -=============================== ============================= ==================================== - -Categories ----------- - -We use category labels on MSCs to place them into a track of work. The Spec Core Team -decides which of the tracks they are focusing on for the next while and generally makes -an effort to pull MSCs out of that category when possible. - -The current categories are: - -============ ================= ====================================== -Name GitHub Label Description -============ ================= ====================================== -Core kind:core Important for the protocol's success. -Feature kind:feature Nice to have additions to the spec. -Maintenance kind:maintenance Fixes or clarifies existing spec. -============ ================= ====================================== - -Some examples of core MSCs would be aggregations, cross-signing, and groups/communities. -These are the sorts of things that if not implemented could cause the protocol to -fail or become second-class. Features would be areas like enhanced media APIs, -new transports, and bookmarks in comparison. Finally, maintenance MSCs would include -improving error codes, clarifying what is required of an API, and adding properties -to an API which makes it easier to use. - -The Spec Core Team assigns a category to each MSC based on the descriptions above. -This can mean that new MSCs get categorized into an area the team isn't focused on, -though that can always change as priorities evolve. We still encourage that MSCs be -opened, even if not the focus for the time being, as they can still make progress and -even be merged without the Spec Core Team focusing on them specifically. - -Implementing a proposal ------------------------ - -As part of the proposal process the spec core team will require evidence of the MSC -working in order for it to move into FCP. This can usually be a branch/pull request -to whichever implementation of choice that proves the MSC works in practice, though -in some cases the MSC itself will be small enough to be considered proven. Where it's -unclear if an MSC will require an implementation proof, ask in `#matrix-spec:matrix.org -`_. - -Early release of an MSC/idea -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To help facilitate early releases of software dependent on a spec release, implementations -are required to use the following process to ensure that the official Matrix namespace -is not cluttered with development or testing data. - -.. Note:: - Unreleased implementations (including proofs-of-concept demonstrating that a - particular MSC works) do not have to follow this process. - -1. Have an idea for a feature. -2. Implement the feature using unstable endpoints, vendor prefixes, and unstable - feature flags as appropriate. - - * When using unstable endpoints, they MUST include a vendor prefix. For example: - ``/_matrix/client/unstable/com.example/login``. Vendor prefixes throughout Matrix - always use the Java package naming convention. The MSC for the feature should - identify which preferred vendor prefix is to be used by early adopters. - * Note that unstable namespaces do not automatically inherit endpoints from stable - namespaces: for example, the fact that ``/_matrix/client/r0/sync`` exists does - not imply that ``/_matrix/client/unstable/com.example/sync`` exists. - * If the client needs to be sure the server supports the feature, an unstable - feature flag that MUST be vendor prefixed is to be used. This kind of flag shows - up in the ``unstable_features`` section of ``/versions`` as, for example, - ``com.example.new_login``. The MSC for the feature should identify which preferred - feature flag is to be used by early adopters. - * When using this approach correctly, the implementation can ship/release the - feature at any time, so long as the implementation is able to accept the technical - debt that results from needing to provide adequate backwards and forwards - compatibility. The implementation MUST support the flag (and server-side implementation) disappearing and be - generally safe for users. Note that implementations early in the MSC review - process may also be required to provide backwards compatibility with earlier - editions of the proposal. - * If the implementation cannot support the technical debt (or if it's impossible - to provide forwards/backwards compatibility - e.g. a user authentication change - which can't be safely rolled back), the implementation should not attempt to - implement the feature and should instead wait for a spec release. - * If at any point after early release, the idea changes in a backwards-incompatible way, the feature flag should also change so that - implementations can adapt as needed. - -3. In parallel, or ahead of implementation, open an MSC and solicit review per above. -4. Before FCP can be called, the Spec Core Team will require evidence of the MSC - working as proposed. A typical example of this is an implementation of the MSC, - though the implementation does not need to be shipped anywhere and can therefore - avoid the forwards/backwards compatibility concerns mentioned here. -5. The FCP process is completed, and assuming nothing is flagged the MSC lands. -6. A spec PR is written to incorporate the changes into Matrix. -7. A spec release happens. -8. Implementations switch to using stable prefixes (e.g.: ``/r0``) if the server - supports the specification version released. If the server doesn't advertise the - specification version, but does have the feature flag, unstable prefixes should - still be used. -9. A transition period of about 2 months starts immediately after the spec release, - before implementations start to encourage other implementations to switch - to stable endpoints. For example, a server implementation should start asking - client implementations to support the stable endpoints 2 months after the spec - release, if they haven't already. The same applies in the reverse: if clients - cannot switch to stable prefixes because server implementations haven't started - supporting the new spec release, some noise should be raised in the general direction - of the implementation. - -.. Note:: - MSCs MUST still describe what the stable endpoints/feature looks like with a note - towards the bottom for what the unstable feature flag/prefixes are. For example, - an MSC would propose `/_matrix/client/r0/new/endpoint`, not `/_matrix/client/unstable/ - com.example/new/endpoint`. - -In summary: - -* Implementations MUST NOT use stable endpoints before the MSC is in the spec. This - includes NOT using stable endpoints in the period between completion of FCP and release of the spec. - passed. -* Implementations are able to ship features that are exposed to users by default before - an MSC has been merged to the spec, provided they follow the process above. -* Implementations SHOULD be wary of the technical debt they are incurring by moving faster - than the spec. -* The vendor prefix is chosen by the developer of the feature, using the Java package - naming convention. The foundation's preferred vendor prefix is `org.matrix`. -* The vendor prefixes, unstable feature flags, and unstable endpoints should be included - in the MSC, though the MSC MUST be written in a way that proposes new stable endpoints. - Typically this is solved by a small table at the bottom mapping the various values - from stable to unstable. - -Proposal Tracking ------------------ - -This is a living document generated from the list of proposals on the issue and -pull request trackers of the `matrix-doc -`_ repo. - -We use labels and some metadata in MSC PR descriptions to generate this page. -Labels are assigned by the Spec Core Team whilst triaging the proposals based on those -which exist in the `matrix-doc `_ -repo already. - -It is worth mentioning that a previous version of the MSC process used a -mixture of GitHub issues and PRs, leading to some MSC numbers deriving from -GitHub issue IDs instead. A useful feature of GitHub is that it does -automatically resolve to an issue, if an issue ID is placed in a pull URL. This -means that https://github.com/matrix-org/matrix-doc/pull/$MSCID will correctly -resolve to the desired MSC, whether it started as an issue or a PR. - -Other metadata: - -- The MSC number is taken from the GitHub Pull Request ID. This is carried for - the lifetime of the proposal. These IDs do not necessarily represent a - chronological order. -- The GitHub PR title will act as the MSC's title. -- Please link to the spec PR (if any) by adding a "PRs: #1234" line in the - issue description. -- The creation date is taken from the GitHub PR, but can be overridden by - adding a "Date: yyyy-mm-dd" line in the PR description. -- Updated Date is taken from GitHub. -- Author is the creator of the MSC PR, but can be overridden by adding an - "Author: @username" line in the body of the issue description. Please make - sure @username is a GitHub user (include the @!) -- A shepherd can be assigned by adding a "Shepherd: @username" line in the - issue description. Again, make sure this is a real GitHub user. diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst deleted file mode 100644 index 46c0000d..00000000 --- a/specification/push_gateway.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2018 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Push Gateway API -================ - -{{unstable_warning_block_PUSH_GATEWAY_RELEASE_LABEL}} - -Clients may want to receive push notifications when events are received at -the homeserver. This is managed by a distinct entity called the Push Gateway. - -.. contents:: Table of Contents -.. sectnum:: - -Changelog ---------- - -.. topic:: Version: %PUSH_GATEWAY_RELEASE_LABEL% -{{push_gateway_changelog}} - -This version of the specification is generated from -`matrix-doc `_ as of Git commit -`{{git_version}} `_. - -For the full historical changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/changelogs/push_gateway.rst - -Other versions of this specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following other versions are also available, in reverse chronological order: - -- `HEAD `_: Includes all changes since the latest versioned release. -- `r0.1.0 `_ - -Overview --------- - -A client's homeserver forwards information about received events to the push -gateway. The gateway then submits a push notification to the push notification -provider (e.g. APNS, GCM). - - -:: - - +--------------------+ +-------------------+ - Matrix HTTP | | | | - Notification Protocol | App Developer | | Device Vendor | - | | | | - +-------------------+ | +----------------+ | | +---------------+ | - | | | | | | | | | | - | Matrix homeserver +-----> Push Gateway +------> Push Provider | | - | | | | | | | | | | - +-^-----------------+ | +----------------+ | | +----+----------+ | - | | | | | | - Matrix | | | | | | - Client/Server API + | | | | | - | | +--------------------+ +-------------------+ - | +--+-+ | - | | <-------------------------------------------+ - +---+ | - | | Provider Push Protocol - +----+ - - Mobile Device or Client - - -Homeserver behaviour --------------------- - -This describes the format used by "HTTP" pushers to send notifications of -events to Push Gateways. If the endpoint returns an HTTP error code, the -homeserver SHOULD retry for a reasonable amount of time using exponential backoff. - -When pushing notifications for events, the homeserver is expected to include all of -the event-related fields in the ``/notify`` request. When the homeserver is performing -a push where the ``format`` is ``"event_id_only"``, only the ``event_id``, ``room_id``, -``counts``, and ``devices`` are required to be populated. - -Note that most of the values and behaviour of this endpoint is described by the Client-Server -API's `Push Module <../client_server/%CLIENT_RELEASE_LABEL%.html#module-push>`_. - -{{push_notifier_push_http_api}} diff --git a/specification/rooms/v1.rst b/specification/rooms/v1.rst deleted file mode 100644 index a7d5383d..00000000 --- a/specification/rooms/v1.rst +++ /dev/null @@ -1,368 +0,0 @@ -.. Copyright 2017,2019 New Vector Ltd -.. Copyright 2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Version 1 -============== - -This room version is the first ever version for rooms, and contains the building -blocks for other room versions. - -.. contents:: Table of Contents -.. sectnum:: - -Client considerations ---------------------- - -Clients may need to consider some algorithms performed by the server for their own -implementation. - -Redactions -~~~~~~~~~~ - -Upon receipt of a redaction event, the server must strip off any keys not in -the following list: - -- ``event_id`` -- ``type`` -- ``room_id`` -- ``sender`` -- ``state_key`` -- ``content`` -- ``hashes`` -- ``signatures`` -- ``depth`` -- ``prev_events`` -- ``prev_state`` -- ``auth_events`` -- ``origin`` -- ``origin_server_ts`` -- ``membership`` - -.. Note: - Some of the keys, such as ``hashes``, will appear on the federation-formatted - event and therefore the client may not be aware of them. - -The content object must also be stripped of all keys, unless it is one of -one of the following event types: - -- ``m.room.member`` allows key ``membership``. -- ``m.room.create`` allows key ``creator``. -- ``m.room.join_rules`` allows key ``join_rule``. -- ``m.room.power_levels`` allows keys ``ban``, ``events``, ``events_default``, - ``kick``, ``redact``, ``state_default``, ``users``, ``users_default``. -- ``m.room.aliases`` allows key ``aliases``. -- ``m.room.history_visibility`` allows key ``history_visibility``. - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - intricacies contained here. The section above regarding client considerations - is the resource that Client-Server API use cases should reference. - - -The algorithms defined here should only apply to version 1 rooms. Other algorithms -may be used by other room versions, and as such servers should be aware of which -version room they are dealing with prior to executing a given algorithm. - -.. WARNING:: - Although there are many rooms using room version 1, it is known to have - undesirable effects. Servers implementing support for room version 1 should be - aware that restrictions should be generally relaxed and that inconsistencies - may occur. - -State resolution -~~~~~~~~~~~~~~~~ - -.. WARNING:: - Room version 1 is known to have bugs that can cause the state of rooms to reset - to older versions of the room's state. For example this could mean that users - who had joined the room may be removed from the room, admins and moderators - could lose their power level, and users who have been banned from the room may - be able to rejoin. Other state events such as the the room's name or topic could - also reset to a previous version. - - This is fixed in the state resolution algorithm introduced in room version 2. - -The room state :math:`S'(E)` after an event :math:`E` is defined in terms of -the room state :math:`S(E)` before :math:`E`, and depends on whether -:math:`E` is a state event or a message event: - -* If :math:`E` is a message event, then :math:`S'(E) = S(E)`. - -* If :math:`E` is a state event, then :math:`S'(E)` is :math:`S(E)`, except - that its entry corresponding to :math:`E`'s ``event_type`` and ``state_key`` - is replaced by :math:`E`'s ``event_id``. - -The room state :math:`S(E)` before :math:`E` is the *resolution* of the set of -states :math:`\{ S'(E'), S'(E''), … \}` consisting of the states after each of -:math:`E`'s ``prev_event``\s :math:`\{ E', E'', … \}`. - -The *resolution* of a set of states is defined as follows. The resolved state -is built up in a number of passes; here we use :math:`R` to refer to the -results of the resolution so far. - -* Start by setting :math:`R` to the union of the states to be resolved, - excluding any *conflicting* events. - -* First we resolve conflicts between ``m.room.power_levels`` events. If there - is no conflict, this step is skipped, otherwise: - - * Assemble all the ``m.room.power_levels`` events from the states to - be resolved into a list. - - * Sort the list by ascending ``depth`` then descending ``sha1(event_id)``. - - * Add the first event in the list to :math:`R`. - - * For each subsequent event in the list, check that the event would be - allowed by the authorization rules for a room in state :math:`R`. If the - event would be allowed, then update :math:`R` with the event and continue - with the next event in the list. If it would not be allowed, stop and - continue below with ``m.room.join_rules`` events. - -* Repeat the above process for conflicts between ``m.room.join_rules`` events. - -* Repeat the above process for conflicts between ``m.room.member`` events. - -* No other events affect the authorization rules, so for all other conflicts, - just pick the event with the highest depth and lowest ``sha1(event_id)`` that - passes authentication in :math:`R` and add it to :math:`R`. - -A *conflict* occurs between states where those states have different -``event_ids`` for the same ``(event_type, state_key)``. The events thus -affected are said to be *conflicting* events. - - -Authorization rules -~~~~~~~~~~~~~~~~~~~ - -The types of state events that affect authorization are: - -- ``m.room.create`` -- ``m.room.member`` -- ``m.room.join_rules`` -- ``m.room.power_levels`` -- ``m.room.third_party_invite`` - -.. NOTE:: - - Power levels are inferred from defaults when not explicitly supplied. - For example, mentions of the ``sender``'s power level can also refer - to the default power level for users in the room. - -The rules are as follows: - -1. If type is ``m.room.create``: - - a. If it has any previous events, reject. - b. If the domain of the ``room_id`` does not match the domain of the - ``sender``, reject. - c. If ``content.room_version`` is present and is not a recognised version, - reject. - d. If ``content`` has no ``creator`` field, reject. - e. Otherwise, allow. - -#. Reject if event has ``auth_events`` that: - - a. have duplicate entries for a given ``type`` and ``state_key`` pair - #. have entries whose ``type`` and ``state_key`` don't match those - specified by the `auth events selection`_ algorithm described in the - server specification. - -#. If event does not have a ``m.room.create`` in its ``auth_events``, reject. - -#. If type is ``m.room.aliases``: - - a. If event has no ``state_key``, reject. - b. If sender's domain doesn't matches ``state_key``, reject. - c. Otherwise, allow. - -#. If type is ``m.room.member``: - - a. If no ``state_key`` key or ``membership`` key in ``content``, reject. - - #. If ``membership`` is ``join``: - - i. If the only previous event is an ``m.room.create`` - and the ``state_key`` is the creator, allow. - - #. If the ``sender`` does not match ``state_key``, reject. - - #. If the ``sender`` is banned, reject. - - #. If the ``join_rule`` is ``invite`` then allow if membership state - is ``invite`` or ``join``. - - #. If the ``join_rule`` is ``public``, allow. - - #. Otherwise, reject. - - #. If ``membership`` is ``invite``: - - i. If ``content`` has ``third_party_invite`` key: - - #. If *target user* is banned, reject. - - #. If ``content.third_party_invite`` does not have a - ``signed`` key, reject. - - #. If ``signed`` does not have ``mxid`` and ``token`` keys, reject. - - #. If ``mxid`` does not match ``state_key``, reject. - - #. If there is no ``m.room.third_party_invite`` event in the - current room state with ``state_key`` matching ``token``, reject. - - #. If ``sender`` does not match ``sender`` of the - ``m.room.third_party_invite``, reject. - - #. If any signature in ``signed`` matches any public key in the - ``m.room.third_party_invite`` event, allow. The public keys are - in ``content`` of ``m.room.third_party_invite`` as: - - #. A single public key in the ``public_key`` field. - #. A list of public keys in the ``public_keys`` field. - - #. Otherwise, reject. - - #. If the ``sender``'s current membership state is not ``join``, reject. - - #. If *target user*'s current membership state is ``join`` or ``ban``, - reject. - - #. If the ``sender``'s power level is greater than or equal to the *invite - level*, allow. - - #. Otherwise, reject. - - #. If ``membership`` is ``leave``: - - i. If the ``sender`` matches ``state_key``, allow if and only if that user's - current membership state is ``invite`` or ``join``. - - #. If the ``sender``'s current membership state is not ``join``, reject. - - #. If the *target user*'s current membership state is ``ban``, and the - ``sender``'s power level is less than the *ban level*, reject. - - #. If the ``sender``'s power level is greater than or equal to the *kick - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. - - #. Otherwise, reject. - - #. If ``membership`` is ``ban``: - - i. If the ``sender``'s current membership state is not ``join``, reject. - - #. If the ``sender``'s power level is greater than or equal to the *ban - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. - - #. Otherwise, reject. - - #. Otherwise, the membership is unknown. Reject. - -#. If the ``sender``'s current membership state is not ``join``, reject. - -#. If type is ``m.room.third_party_invite``: - - a. Allow if and only if ``sender``'s current power level is greater than - or equal to the *invite level*. - -#. If the event type's *required power level* is greater than the ``sender``'s power - level, reject. - -#. If the event has a ``state_key`` that starts with an ``@`` and does not match - the ``sender``, reject. - -#. If type is ``m.room.power_levels``: - - a. If ``users`` key in ``content`` is not a dictionary with keys that are - valid user IDs with values that are integers (or a string that is an - integer), reject. - - #. If there is no previous ``m.room.power_levels`` event in the room, allow. - - #. For the keys ``users_default``, ``events_default``, - ``state_default``, ``ban``, ``redact``, ``kick``, ``invite`` check if they - were added, changed or removed. For each found alteration: - - i. If the current value is higher than the ``sender``'s current power level, - reject. - - #. If the new value is higher than the ``sender``'s current power level, - reject. - - #. For each entry being added, changed or removed in both the ``events`` and - ``users`` keys: - - i. If the current value is higher than the ``sender``'s current power level, - reject. - - #. If the new value is higher than the ``sender``'s current power level, - reject. - - #. For each entry being changed under the ``users`` key, other than the - ``sender``'s own entry: - - i. If the current value is equal to the ``sender``'s current power level, - reject. - - #. Otherwise, allow. - -#. If type is ``m.room.redaction``: - - a. If the ``sender``'s power level is greater than or equal to the *redact - level*, allow. - - #. If the domain of the ``event_id`` of the event being redacted is the same - as the domain of the ``event_id`` of the ``m.room.redaction``, allow. - - #. Otherwise, reject. - -#. Otherwise, allow. - -.. NOTE:: - - Some consequences of these rules: - - * Unless you are a member of the room, the only permitted operations (apart - from the initial create/join) are: joining a public room; accepting or - rejecting an invitation to a room. - - * To unban somebody, you must have power level greater than or equal to both - the kick *and* ban levels, *and* greater than the target user's power - level. - -Event format -~~~~~~~~~~~~ - -Events in version 1 rooms have the following structure: - -{{definition_ss_pdu}} - -Canonical JSON -~~~~~~~~~~~~~~ - -Servers MUST NOT strictly enforce the JSON format specified in the -`appendices <../appendices.html#canonical-json>`_ for the reasons described there. - - -.. _`auth events selection`: ../server_server/%SERVER_RELEASE_LABEL%.html#auth-events-selection -.. _`Signing Events`: ../server_server/%SERVER_RELEASE_LABEL%.html#signing-events diff --git a/specification/rooms/v2.rst b/specification/rooms/v2.rst deleted file mode 100644 index afc114f8..00000000 --- a/specification/rooms/v2.rst +++ /dev/null @@ -1,204 +0,0 @@ -.. Copyright 2018-2019 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Version 2 -============== - -This room version builds off of `version 1 `_ with an improved state -resolution algorithm. - -.. contents:: Table of Contents -.. sectnum:: - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - details contained here, and can safely ignore their presence. - - -Room version 2 uses the base components of `room version 1 `_, changing -only the state resolution algorithm. - - -State resolution -~~~~~~~~~~~~~~~~ - -The room state :math:`S'(E)` after an event :math:`E` is defined in terms of -the room state :math:`S(E)` before :math:`E`, and depends on whether -:math:`E` is a state event or a message event: - -* If :math:`E` is a message event, then :math:`S'(E) = S(E)`. - -* If :math:`E` is a state event, then :math:`S'(E)` is :math:`S(E)`, except - that its entry corresponding to :math:`E`'s ``event_type`` and ``state_key`` - is replaced by :math:`E`'s ``event_id``. - -The room state :math:`S(E)` before :math:`E` is the *resolution* of the set of -states :math:`\{ S'(E_1), S'(E_2), … \}` consisting of the states after each of -:math:`E`'s ``prev_event``\s :math:`\{ E_1, E_2, … \}`, where the resolution of -a set of states is given in the algorithm below. - -Definitions -+++++++++++ - -The state resolution algorithm for version 2 rooms uses the following -definitions, given the set of room states :math:`\{ S_1, S_2, \ldots \}`: - -Power events - A *power event* is a state event with type ``m.room.power_levels`` or - ``m.room.join_rules``, or a state event with type ``m.room.member`` where the - ``membership`` is ``leave`` or ``ban`` and the ``sender`` does not match the - ``state_key``. The idea behind this is that power events are events that might - remove someone's ability to do something in the room. - -Unconflicted state map and conflicted state set - The *unconflicted state map* is the state where the value of each key exists - and is the same in each state :math:`S_i`. The *conflicted state set* is the - set of all other state events. Note that the unconflicted state map only has - one event per ``(event_type, state_key)``, whereas the conflicted state set - may have multiple events. - -Auth difference - The *auth difference* is calculated by first calculating the full auth chain - for each state :math:`S_i`, that is the union of the auth chains for each - event in :math:`S_i`, and then taking every event that doesn't appear in - every auth chain. If :math:`C_i` is the full auth chain of :math:`S_i`, then - the auth difference is :math:`\cup C_i - \cap C_i`. - -Full conflicted set - The *full conflicted set* is the union of the conflicted state set and the - auth difference. - -Reverse topological power ordering - The *reverse topological power ordering* of a set of events is the - lexicographically smallest topological ordering based on the DAG formed by - auth events. The reverse topological power ordering is ordered from earliest - event to latest. For comparing two topological orderings to determine which - is the lexicographically smallest, the following comparison relation on - events is used: for events :math:`x` and :math:`y`, :math:`x`_ with an improved event format. - -.. note: - All requirements listed in this room version specification are scoped to rooms - which actually use this room version. For example, a requirement of "all APIs must - accept the new event format" does in fact apply to all APIs, but only so much as - where the contextual room of the request is using this room version. Rooms using - other room versions should not be affected by these sweeping requirements. - -.. contents:: Table of Contents -.. sectnum:: - - -Client considerations ---------------------- - -This room version changes the format for event IDs sent to clients. Clients should be -aware that these event IDs may contain slashes and other potentially problematic -characters. Clients should be treating event IDs as opaque identifiers and should not -be attempting to parse them into a usable form, just like with other room versions. - -Clients should expect to see event IDs changed from the format of ``$randomstring:example.org`` -to something like ``$acR1l0raoZnm60CBwAVgqbZqoO/mYU81xysh1u7XcJk`` (note the lack of -domain and the potentially problematic slash). - - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - intricacies contained here. The section above regarding client considerations - is the resource that Client-Server API use cases should reference. - - -Room version 3 uses the state resolution algorithm defined in `room version 2 `_, -and the event format defined here. - -Event IDs -~~~~~~~~~ - -.. admonition:: Rationale - - In other room versions (namely version 1 and 2) the event ID is a distinct field - from the remainder of the event, which must be tracked as such. This leads to - complications where servers receive multiple events with the same ID in either the - same or different rooms where the server cannot easily keep track of which event it - should be using. By removing the use of a dedicated event ID, servers are required - to track the hashes on an event to determine its ID. - -The event ID is the `reference hash`_ of the event encoded using `Unpadded Base64`_, -prefixed with ``$``. A resulting event ID using this approach should look similar to -``$CD66HAED5npg6074c6pDtLKalHjVfYb2q4Q3LZgrW6o``. - -Event IDs should not be sent over federation to servers when the room uses -this room version. On the receiving end of an event, the server should compute -the relevant event ID for itself. - -Additionally, the ``auth_events`` and ``prev_events`` have had a format change -compared to other room versions to make it easier to handle. Instead of a tuple -of values, they are now plain lists of events. - -{{definition_ss_pdu_v3}} - -Changes to APIs -~~~~~~~~~~~~~~~ - -Due to the event ID being removed from the event, some APIs need to change. All -APIs which currently accept an event ID must do so with the new format. Servers -must append the calculated event ID to all events sent to clients where an event -ID would normally be expected. - -Because the format of events has changed, servers must be aware of the room version -where the event resides so that the server may parse and handle the event. The -federation API has taken this concern into consideration by ensuring that servers -are aware of (or can find) the room version during a request. - -Authorization rules for events -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The authorization rules for a given event have changed in this room version due -to the change in event format: - -* The event no longer needs to be signed by the domain of the event ID (as there - is no domain in the event ID), but still needs to be signed by the sender's - domain. - -* In past room versions, redactions were only permitted to enter the DAG if the - sender's domain matched the domain in the event ID being redacted, or the sender - had appropriate permissions per the power levels. Due to servers now not being - able to determine where an event came from during event authorization, redaction - events are always accepted (provided the event is allowed by ``events`` and - ``events_default`` in the power levels). However, servers should not apply or send - redactions to clients until both the redaction event and original event have been - seen, and are valid. Servers should only apply redactions to events where the - sender's domains match, or the sender of the redaction has the appropriate - permissions per the power levels. - - -The remaining rules are the same as `room version 1 `_. - - -.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 -.. _`Canonical JSON`: ../appendices.html#canonical-json -.. _`Signing Events`: ../server_server/%SERVER_RELEASE_LABEL%.html#signing-events -.. _`reference hash`: ../server_server/%SERVER_RELEASE_LABEL%.html#reference-hashes diff --git a/specification/rooms/v4.rst b/specification/rooms/v4.rst deleted file mode 100644 index bcd821cb..00000000 --- a/specification/rooms/v4.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Version 4 -============== - -This room version builds on `version 3 `_ using a different encoding for -event IDs. - -.. contents:: Table of Contents -.. sectnum:: - - -Client considerations ---------------------- - -This room version changes the format form event IDs sent to clients. Clients should -already be treating event IDs as opaque identifiers, and should not be concerned with -the format of them. Clients should still encode the event ID when including it in a -request path. - -Clients should expect to see event IDs changed from the format of ``$randomstring:example.org`` -to something like ``$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg`` (note the lack of domain). - - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - intricacies contained here. The section above regarding client considerations - is the resource that Client-Server API use cases should reference. - - -Room version 4 uses the same algorithms defined in `room version 3 `_, however -using URL-safe base64 to generate the event ID. - -Event IDs -~~~~~~~~~ - -.. admonition:: Rationale - - Room version 3 generated event IDs that were difficult for client implementations - which were not encoding the event ID to function in those rooms. It additionally - raised concern due to the ``/`` character being interpretted differently by some - reverse proxy software, and generally made administration harder. - -The event ID is the `reference hash`_ of the event encoded using a variation of -`Unpadded Base64`_ which replaces the 62nd and 63rd characters with ``-`` and ``_`` -instead of using ``+`` and ``/``. This matches `RFC4648's definition of URL-safe base64 -`_. Event IDs are still prefixed -with ``$`` and may result in looking like ``$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg``. - -Just like in room version 3, event IDs should not be sent over federation to servers -when the room uses this room version. On the receiving end of an event, the server -should compute the relevant event ID for itself. Room version 3 also changes the format -of ``auth_events`` and ``prev_events`` in a PDU. - -{{definition_ss_pdu_v4}} - -.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 -.. _`Canonical JSON`: ../appendices.html#canonical-json -.. _`Signing Events`: ../server_server/%SERVER_RELEASE_LABEL%.html#signing-events -.. _`reference hash`: ../server_server/%SERVER_RELEASE_LABEL%.html#reference-hashes diff --git a/specification/rooms/v5.rst b/specification/rooms/v5.rst deleted file mode 100644 index 6d34ec93..00000000 --- a/specification/rooms/v5.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. Copyright 2019 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Version 5 -============== - -This room version builds on `version 4 `_ while enforcing signing -key validity periods for events. - -.. contents:: Table of Contents -.. sectnum:: - - -Client considerations ---------------------- - -There are no specific requirements for clients in this room version. Clients should -be aware of event ID changes in `room version 4 `_, however. - - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - intricacies contained here. The section above regarding client considerations - is the resource that Client-Server API use cases should reference. - - -Room version 5 uses the same algorithms defined in `room version 4 `_, ensuring -that signing key validity is respected. - -Signing key validity period -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When validating event signatures, servers MUST enforce the ``valid_until_ts`` property -from a key request is at least as large as the ``origin_server_ts`` for the event being -validated. Servers missing a copy of the signing key MUST try to obtain one via the -`GET /_matrix/key/v2/server <../server_server/%SERVER_RELEASE_LABEL%.html#get-matrix-key-v2-server-keyid>`_ -or `POST /_matrix/key/v2/query <../server_server/%SERVER_RELEASE_LABEL%.html#post-matrix-key-v2-query>`_ -APIs. When using the ``/query`` endpoint, servers MUST set the ``minimum_valid_until_ts`` -property to prompt the notary server to attempt to refresh the key if appropriate. - -Servers MUST use the lesser of ``valid_until_ts`` and 7 days into the future when -determining if a key is valid. This is to avoid a situation where an attacker -publishes a key which is valid for a significant amount of time without a way for -the homeserver owner to revoke it. diff --git a/specification/rooms/v6.rst b/specification/rooms/v6.rst deleted file mode 100644 index e5378d0e..00000000 --- a/specification/rooms/v6.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. Copyright 2020 The Matrix.org Foundation C.I.C. -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Room Version 6 -============== - -This room version builds on `version 5 `_ while changing various -authorization rules performed on events. - -.. contents:: Table of Contents -.. sectnum:: - - -Client considerations ---------------------- - -The redaction algorithm has changed from `room version 1 `_ to remove -all rules against events of type ``m.room.aliases``. Room versions 2, 3, 4, and -5 all use v1's redaction algorithm. The algorithm is otherwise unchanged. - - -Server implementation components --------------------------------- - -.. WARNING:: - The information contained in this section is strictly for server implementors. - Applications which use the Client-Server API are generally unaffected by the - intricacies contained here. The section above regarding client considerations - is the resource that Client-Server API use cases should reference. - - -Room version 6 makes the following alterations to algorithms described in `room version 5 `_. - -Redactions -~~~~~~~~~~ - -As mentioned in the client considerations portion of this specification, all -special meaning has been removed for events of type ``m.room.aliases``. The -algorithm is otherwise unchanged. - -Authorization rules for events -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Like redactions, all rules relating specifically to events of type ``m.room.aliases`` -are removed. They must still pass authorization checks relating to state events. - -Additionally, the authorization rules for events of type ``m.room.power_levels`` -now include the content key ``notifications``. This new rule takes the place of the -rule which checks the ``events`` and ``users`` keys. - -For completeness, the changes to the auth rules can be represented as follows: - -.. code:: diff - - ... - - -If type is `m.room.aliases`: - - - - a. If event has no `state_key`, reject. - - b. If sender's domain doesn't matches `state_key`, reject. - - c. Otherwise, allow. - - ... - - If type is `m.room.power_levels`: - - ... - - - * For each entry being added, changed or removed in both the `events` and `users` keys: - + * For each entry being added, changed or removed in the `events`, `users`, and `notifications` keys: - - i. If the current value is higher than the `sender`'s current power level, reject. - - ii. If the new value is higher than the `sender`'s current power level, reject. - - ... - - -The remaining rules are the same as in `room version 3 `_ -(the last inherited room version to specify the authorization rules). - -Canonical JSON -~~~~~~~~~~~~~~ - -Servers MUST strictly enforce the JSON format specified in the -`appendices <../appendices.html#canonical-json>`_. This translates to a 400 ``M_BAD_JSON`` error -on most endpoints, or discarding of events over federation. For example, the Federation API's -``/send`` endpoint would discard the event whereas the Client Server API's ``/send/{eventType}`` -endpoint would return a ``M_BAD_JSON`` error. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst deleted file mode 100644 index b4d5f22e..00000000 --- a/specification/server_server_api.rst +++ /dev/null @@ -1,1269 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. Copyright 2017-2019 New Vector Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -Federation API -============== - -{{unstable_warning_block_SERVER_RELEASE_LABEL}} - -Matrix homeservers use the Federation APIs (also known as server-server APIs) -to communicate with each other. Homeservers use these APIs to push messages to -each other in real-time, to retrieve historic messages from each other, and to -query profile and presence information about users on each other's servers. - -The APIs are implemented using HTTPS requests between each of the servers. -These HTTPS requests are strongly authenticated using public key signatures -at the TLS transport layer and using public key signatures in HTTP -Authorization headers at the HTTP layer. - -There are three main kinds of communication that occur between homeservers: - -Persisted Data Units (PDUs): - These events are broadcast from one homeserver to any others that have - joined the same room (identified by Room ID). They are persisted in - long-term storage and record the history of messages and state for a - room. - - Like email, it is the responsibility of the originating server of a PDU - to deliver that event to its recipient servers. However PDUs are signed - using the originating server's private key so that it is possible to - deliver them through third-party servers. - -Ephemeral Data Units (EDUs): - These events are pushed between pairs of homeservers. They are not - persisted and are not part of the history of a room, nor does the - receiving homeserver have to reply to them. - -Queries: - These are single request/response interactions between a given pair of - servers, initiated by one side sending an HTTPS GET request to obtain some - information, and responded by the other. They are not persisted and contain - no long-term significant history. They simply request a snapshot state at - the instant the query is made. - - -EDUs and PDUs are further wrapped in an envelope called a Transaction, which is -transferred from the origin to the destination homeserver using an HTTPS PUT -request. - -.. contents:: Table of Contents -.. sectnum:: - -Changelog ---------- - -.. topic:: Version: %SERVER_RELEASE_LABEL% -{{server_server_changelog}} - -This version of the specification is generated from -`matrix-doc `_ as of Git commit -`{{git_version}} `_. - -For the full historical changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/changelogs/server_server.rst - -Other versions of this specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following other versions are also available, in reverse chronological order: - -- `HEAD `_: Includes all changes since the latest versioned release. -- `r0.1.4 `_ -- `r0.1.3 `_ -- `r0.1.2 `_ -- `r0.1.1 `_ -- `r0.1.0 `_ - - -API standards -------------- - -The mandatory baseline for client-server communication in Matrix is exchanging -JSON objects over HTTP APIs. More efficient optional transports will in future -be supported as optional extensions - e.g. a packed binary encoding over -stream-cipher encrypted TCP socket for low-bandwidth/low-roundtrip mobile usage. -For the default HTTP transport, all API calls use a Content-Type of -``application/json``. In addition, all strings MUST be encoded as UTF-8. - -Server discovery ----------------- - -Resolving server names -~~~~~~~~~~~~~~~~~~~~~~ - -Each Matrix homeserver is identified by a server name consisting of a hostname -and an optional port, as described by the `grammar -<../appendices.html#server-name>`_. Where applicable, a delegated server name -uses the same grammar. - -Server names are resolved to an IP address and port to connect to, and have -various conditions affecting which certificates and ``Host`` headers to send. -The process overall is as follows: - -.. Note from the author: The repetitive "use this Host header and this cert" - comments are intentional. The process is overall quite complicated, and - explaining explicitly what requests look like at each step helps ease the - understanding and ensure everyone is on the same page. Implementations - are of course welcome to realize where the process can be optimized, and - do so - just ensure that the result is the same! - -1. If the hostname is an IP literal, then that IP address should be used, - together with the given port number, or 8448 if no port is given. The - target server must present a valid certificate for the IP address. - The ``Host`` header in the request should be set to the server name, - including the port if the server name included one. - -2. If the hostname is not an IP literal, and the server name includes an - explicit port, resolve the IP address using AAAA or A records. Requests - are made to the resolved IP address and given port with a ``Host`` header - of the original server name (with port). The target server must present a - valid certificate for the hostname. - -3. If the hostname is not an IP literal, a regular HTTPS request is made - to ``https:///.well-known/matrix/server``, expecting the - schema defined later in this section. 30x redirects should be followed, - however redirection loops should be avoided. Responses (successful or - otherwise) to the ``/.well-known`` endpoint should be cached by the - requesting server. Servers should respect the cache control headers - present on the response, or use a sensible default when headers are not - present. The recommended sensible default is 24 hours. Servers should - additionally impose a maximum cache time for responses: 48 hours is - recommended. Errors are recommended to be cached for up to an hour, - and servers are encouraged to exponentially back off for repeated - failures. The schema of the ``/.well-known`` request is later in this - section. If the response is invalid (bad JSON, missing properties, non-200 - response, etc), skip to step 4. If the response is valid, the ``m.server`` - property is parsed as ``[:]`` and - processed as follows: - - * If ```` is an IP literal, then that IP address - should be used together with the ```` or 8448 if no - port is provided. The target server must present a valid TLS certificate - for the IP address. Requests must be made with a ``Host`` header containing - the IP address, including the port if one was provided. - - * If ```` is not an IP literal, and ```` - is present, an IP address is discovered by looking up an AAAA or A - record for ````. The resulting IP address is - used, alongside the ````. Requests must be made with a - ``Host`` header of ``:``. The - target server must present a valid certificate for ````. - - * If ```` is not an IP literal and no - ```` is present, an SRV record is looked up for - ``_matrix._tcp.``. This may result in another - hostname (to be resolved using AAAA or A records) and port. Requests - should be made to the resolved IP address and port with a ``Host`` - header containing the ````. The target server - must present a valid certificate for ````. - - * If no SRV record is found, an IP address is resolved using AAAA - or A records. Requests are then made to the resolve IP address - and a port of 8448, using a ``Host`` header of ````. - The target server must present a valid certificate for ````. - -4. If the ``/.well-known`` request resulted in an error response, a server - is found by resolving an SRV record for ``_matrix._tcp.``. This - may result in a hostname (to be resolved using AAAA or A records) and - port. Requests are made to the resolved IP address and port, using 8448 - as a default port, with a ``Host`` header of ````. The target - server must present a valid certificate for ````. - -5. If the ``/.well-known`` request returned an error response, and the SRV - record was not found, an IP address is resolved using AAAA and A records. - Requests are made to the resolved IP address using port 8448 and a ``Host`` - header containing the ````. The target server must present a - valid certificate for ````. - - -The TLS certificate provided by the target server must be signed by a known -Certificate Authority. Servers are ultimately responsible for determining -the trusted Certificate Authorities, however are strongly encouraged to -rely on the operating system's judgement. Servers can offer administrators -a means to override the trusted authorities list. Servers can additionally -skip the certificate validation for a given whitelist of domains or netmasks -for the purposes of testing or in networks where verification is done -elsewhere, such as with ``.onion`` addresses. Servers should respect SNI -when making requests where possible: a SNI should be sent for the certificate -which is expected, unless that certificate is expected to be an IP address in -which case SNI is not supported and should not be sent. - -Servers are encouraged to make use of the -`Certificate Transparency `_ project. - -{{wellknown_ss_http_api}} - -Server implementation -~~~~~~~~~~~~~~~~~~~~~~ - -{{version_ss_http_api}} - -Retrieving server keys -~~~~~~~~~~~~~~~~~~~~~~ - -.. NOTE:: - There was once a "version 1" of the key exchange. It has been removed from the - specification due to lack of significance. It may be reviewed `from the historical draft - `_. - -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}``. -Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` -directly or by querying an intermediate notary server using a -``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers -query the ``/_matrix/key/v2/server/{keyId}`` API on behalf of another server and -sign the response with their own key. A server may query multiple notary servers to -ensure that they all report the same public keys. - -This approach is borrowed from the `Perspectives Project`_, but modified to -include the NACL keys and to use JSON instead of XML. It has the advantage of -avoiding a single trust-root since each server is free to pick which notary -servers they trust and can corroborate the keys returned by a given notary -server by querying other servers. - -.. _Perspectives Project: https://web.archive.org/web/20170702024706/https://perspectives-project.org/ - -Publishing Keys -+++++++++++++++ - -Homeservers publish their signing keys in a JSON -object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of -``verify_keys`` that are valid for signing federation requests made by the -homeserver and for signing events. It contains a list of ``old_verify_keys`` which -are only valid for signing events. - -{{keys_server_ss_http_api}} - - -Querying Keys Through Another Server -++++++++++++++++++++++++++++++++++++ - -Servers may query another server's keys through a notary server. The notary -server may be another homeserver. The notary server will retrieve keys from -the queried servers through use of the ``/_matrix/key/v2/server/{keyId}`` -API. The notary server will additionally sign the response from the queried -server before returning the results. - -Notary servers can return keys for servers that are offline or having issues -serving their own keys by using cached responses. Keys can be queried from -multiple servers to mitigate against DNS spoofing. - -{{keys_query_ss_http_api}} - -Authentication --------------- - -Request Authentication -~~~~~~~~~~~~~~~~~~~~~~ - -Every HTTP request made by a homeserver is authenticated using public key -digital signatures. The request method, target and body are signed by wrapping -them in a JSON object and signing it using the JSON signing algorithm. The -resulting signatures are added as an Authorization header with an auth scheme -of ``X-Matrix``. Note that the target field should include the full path -starting with ``/_matrix/...``, including the ``?`` and any query parameters if -present, but should not include the leading ``https:``, nor the destination -server's hostname. - -Step 1 sign JSON: - -.. code:: - - { - "method": "GET", - "uri": "/target", - "origin": "origin.hs.example.com", - "destination": "destination.hs.example.com", - "content": , - "signatures": { - "origin.hs.example.com": { - "ed25519:key1": "ABCDEF..." - } - } - } - -The server names in the JSON above are the server names for each homeserver involved. Delegation from -the `server name resolution section <#resolving-server-names>`_ above do not affect -these - the server names from before delegation would take place are used. This -same condition applies throughout the request signing process. - -Step 2 add Authorization header: - -.. code:: - - GET /target HTTP/1.1 - Authorization: X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..." - Content-Type: application/json - - - - -Example python code: - -.. code:: python - - def authorization_headers(origin_name, origin_signing_key, - destination_name, request_method, request_target, - content=None): - request_json = { - "method": request_method, - "uri": request_target, - "origin": origin_name, - "destination": destination_name, - } - - if content is not None: - request_json["content"] = content - - signed_json = sign_json(request_json, origin_name, origin_signing_key) - - authorization_headers = [] - - for key, sig in signed_json["signatures"][origin_name].items(): - authorization_headers.append(bytes( - "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( - origin_name, key, sig, - ) - )) - - return ("Authorization", authorization_headers) - -Response Authentication -~~~~~~~~~~~~~~~~~~~~~~~ - -Responses are authenticated by the TLS server certificate. A homeserver should -not send a request until it has authenticated the connected server to avoid -leaking messages to eavesdroppers. - -Client TLS Certificates -~~~~~~~~~~~~~~~~~~~~~~~ - -Requests are authenticated at the HTTP layer rather than at the TLS layer -because HTTP services like Matrix are often deployed behind load balancers that -handle the TLS and these load balancers make it difficult to check TLS client -certificates. - -A homeserver may provide a TLS client certificate and the receiving homeserver -may check that the client certificate matches the certificate of the origin -homeserver. - -Transactions ------------- - -The transfer of EDUs and PDUs between homeservers is performed by an exchange -of Transaction messages, which are encoded as JSON objects, passed over an HTTP -PUT request. A Transaction is meaningful only to the pair of homeservers that -exchanged it; they are not globally-meaningful. - -Transactions are limited in size; they can have at most 50 PDUs and 100 EDUs. - -{{transactions_ss_http_api}} - -.. _`Persistent Data Unit schema`: - -PDUs ----- - -Each PDU contains a single Room Event which the origin server wants to send to -the destination. - -The ``prev_events`` field of a PDU identifies the "parents" of the event, and -thus establishes a partial ordering on events within the room by linking them -into a Directed Acyclic Graph (DAG). The sending server should populate this -field with all of the events in the room for which it has not yet seen a -child - thus demonstrating that the event comes after all other known events. - -For example, consider a room whose events form the DAG shown below. A server -creating a new event in this room should populate the new event's -``prev_events`` field with ``E4`` and ``E5``, since neither event yet has a child:: - - E1 - ^ - | - +-> E2 <-+ - | | - E3 E5 - ^ - | - E4 - -For a full schema of what a PDU looks like, see the `room version specification`_. - - -Checks performed on receipt of a PDU -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Whenever a server receives an event from a remote server, the receiving server -must ensure that the event: - -1. Is a valid event, otherwise it is dropped. -2. Passes signature checks, otherwise it is dropped. -3. Passes hash checks, otherwise it is redacted before being processed - further. -4. Passes authorization rules based on the event's auth events, otherwise it - is rejected. -5. Passes authorization rules based on the state at the event, otherwise it - is rejected. -6. Passes authorization rules based on the current state of the room, otherwise it - is "soft failed". - -Further details of these checks, and how to handle failures, are described -below. - -The `Signing Events <#signing-events>`_ section has more information on which hashes -and signatures are expected on events, and how to calculate them. - - -Definitions -+++++++++++ - -Required Power Level - A given event type has an associated *required power level*. This is given by - the current ``m.room.power_levels`` event. The event type is either listed - explicitly in the ``events`` section or given by either ``state_default`` or - ``events_default`` depending on if the event is a state event or not. - -Invite Level, Kick Level, Ban Level, Redact Level - The levels given by the ``invite``, ``kick``, ``ban``, and ``redact`` - properties in the current ``m.room.power_levels`` state. Each defaults to 50 - if unspecified. - -Target User - For an ``m.room.member`` state event, the user given by the ``state_key`` of - the event. - -.. _`authorization rules`: - -Authorization rules -+++++++++++++++++++ - -The rules governing whether an event is authorized depends on a set of state. A -given event is checked multiple times against different sets of state, as -specified above. Each room version can have a different algorithm for how the -rules work, and which rules are applied. For more detailed information, please -see the `room version specification`_. - - -Auth events selection -^^^^^^^^^^^^^^^^^^^^^ - -The ``auth_events`` field of a PDU identifies the set of events which give the -sender permission to send the event. The ``auth_events`` for the -``m.room.create`` event in a room is empty; for other events, it should be the -following subset of the room state: - -- The ``m.room.create`` event. -- The current ``m.room.power_levels`` event, if any. -- The sender's current ``m.room.member`` event, if any. -- If type is ``m.room.member``: - - - The target's current ``m.room.member`` event, if any. - - If ``membership`` is ``join`` or ``invite``, the current - ``m.room.join_rules`` event, if any. - - If membership is ``invite`` and ``content`` contains a - ``third_party_invite`` property, the current - ``m.room.third_party_invite`` event with ``state_key`` matching - ``content.third_party_invite.signed.token``, if any. - - -Rejection -+++++++++ - -If an event is rejected it should neither be relayed to clients nor be included -as a prev event in any new events generated by the server. Subsequent events -from other servers that reference rejected events should be allowed if they -still pass the auth rules. The state used in the checks should be calculated as -normal, except not updating with the rejected event where it is a state event. - -If an event in an incoming transaction is rejected, this should not cause the -transaction request to be responded to with an error response. - -.. NOTE:: - - This means that events may be included in the room DAG even though they - should be rejected. - -.. NOTE:: - - This is in contrast to redacted events which can still affect the - state of the room. For example, a redacted ``join`` event will still - result in the user being considered joined. - - -Soft failure -++++++++++++ - -.. admonition:: Rationale - - It is important that we prevent users from evading bans (or other power - restrictions) by creating events which reference old parts of the DAG. For - example, a banned user could continue to send messages to a room by having - their server send events which reference the event before they were banned. - Note that such events are entirely valid, and we cannot simply reject them, as - it is impossible to distinguish such an event from a legitimate one which has - been delayed. We must therefore accept such events and let them participate in - state resolution and the federation protocol as normal. However, servers may - choose not to send such events on to their clients, so that end users won't - actually see the events. - - When this happens it is often fairly obvious to servers, as they can see that - the new event doesn't actually pass auth based on the "current state" (i.e. - the resolved state across all forward extremities). While the event is - technically valid, the server can choose to not notify clients about the new - event. - - This discourages servers from sending events that evade bans etc. in this way, - as end users won't actually see the events. - - -When the homeserver receives a new event over federation it should also check -whether the event passes auth checks based on the current state of the room (as -well as based on the state at the event). If the event does not pass the auth -checks based on the *current state* of the room (but does pass the auth checks -based on the state at that event) it should be "soft failed". - -When an event is "soft failed" it should not be relayed to the client nor be -referenced by new events created by the homeserver (i.e. they should not be -added to the server's list of forward extremities of the room). Soft failed -events are otherwise handled as usual. - - -.. NOTE:: - - Soft failed events participate in state resolution as normal if further events - are received which reference it. It is the job of the state resolution - algorithm to ensure that malicious events cannot be injected into the room - state via this mechanism. - - -.. NOTE:: - - Because soft failed state events participate in state resolution as normal, it - is possible for such events to appear in the current state of the room. In - that case the client should be told about the soft failed event in the usual - way (e.g. by sending it down in the ``state`` section of a sync response). - - -.. NOTE:: - - A soft failed event should be returned in response to federation requests - where appropriate (e.g. in ``/event/``). Note that soft failed - events are returned in ``/backfill`` and ``/get_missing_events`` responses - only if the requests include events referencing the soft failed events. - - -.. admonition:: Example - - As an example consider the event graph:: - - A - / - B - - where ``B`` is a ban of a user ``X``. If the user ``X`` tries to set the topic - by sending an event ``C`` while evading the ban:: - - A - / \ - B C - - servers that receive ``C`` after ``B`` should soft fail event ``C``, and so - will neither relay ``C`` to its clients nor send any events referencing ``C``. - - If later another server sends an event ``D`` that references both ``B`` and - ``C`` (this can happen if it received ``C`` before ``B``):: - - A - / \ - B C - \ / - D - - then servers will handle ``D`` as normal. ``D`` is sent to the servers' - clients (assuming ``D`` passes auth checks). The state at ``D`` may resolve to - a state that includes ``C``, in which case clients should also to be told that - the state has changed to include ``C``. (*Note*: This depends on the exact - state resolution algorithm used. In the original version of the algorithm - ``C`` would be in the resolved state, whereas in latter versions the algorithm - tries to prioritise the ban over the topic change.) - - Note that this is essentially equivalent to the situation where one server - doesn't receive ``C`` at all, and so asks another server for the state of the - ``C`` branch. - - Let's go back to the graph before ``D`` was sent:: - - A - / \ - B C - - If all the servers in the room saw ``B`` before ``C`` and so soft fail ``C``, - then any new event ``D'`` will not reference ``C``:: - - A - / \ - B C - | - D - - -Retrieving event authorization information -++++++++++++++++++++++++++++++++++++++++++ - -The homeserver may be missing event authorization information, or wish to check -with other servers to ensure it is receiving the correct auth chain. These APIs -give the homeserver an avenue for getting the information it needs. - -{{event_auth_ss_http_api}} - -EDUs ----- - -EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of -"previous" IDs. They are intended to be non-persistent data such as user -presence, typing notifications, etc. - -{{definition_ss_edu}} - -Room State Resolution ---------------------- - -The *state* of a room is a map of ``(event_type, state_key)`` to -``event_id``. Each room starts with an empty state, and each state event which -is accepted into the room updates the state of that room. - -Where each event has a single ``prev_event``, it is clear what the state of the -room after each event should be. However, when two branches in the event graph -merge, the state of those branches might differ, so a *state resolution* -algorithm must be used to determine the resultant state. - -For example, consider the following event graph (where the oldest event, E0, -is at the top):: - - E0 - | - E1 - / \ - E2 E4 - | | - E3 | - \ / - E5 - - -Suppose E3 and E4 are both ``m.room.name`` events which set the name of the -room. What should the name of the room be at E5? - -The algorithm to be used for state resolution depends on the room version. For -a description of each room version's algorithm, please see the `room version specification`_. - - -Backfilling and retrieving missing events ------------------------------------------ - -Once a homeserver has joined a room, it receives all the events emitted by -other homeservers in that room, and is thus aware of the entire history of the -room from that moment onwards. Since users in that room are able to request the -history by the ``/messages`` client API endpoint, it's possible that they might -step backwards far enough into history before the homeserver itself was a -member of that room. - -To cover this case, the federation API provides a server-to-server analog of -the ``/messages`` client API, allowing one homeserver to fetch history from -another. This is the ``/backfill`` API. - -To request more history, the requesting homeserver picks another homeserver -that it thinks may have more (most likely this should be a homeserver for -some of the existing users in the room at the earliest point in history it -has currently), and makes a ``/backfill`` request. - -Similar to backfilling a room's history, a server may not have all the events -in the graph. That server may use the ``/get_missing_events`` API to acquire -the events it is missing. - -.. TODO-spec - Specify (or remark that it is unspecified) how the server handles divergent - history. DFS? BFS? Anything weirder? - -{{backfill_ss_http_api}} - -Retrieving events ------------------ - -In some circumstances, a homeserver may be missing a particular event or information -about the room which cannot be easily determined from backfilling. These APIs provide -homeservers with the option of getting events and the state of the room at a given -point in the timeline. - -{{events_ss_http_api}} - - -Joining Rooms -------------- - -When a new user wishes to join a room that the user's homeserver already knows -about, the homeserver can immediately determine if this is allowable by -inspecting the state of the room. If it is acceptable, it can generate, sign, -and emit a new ``m.room.member`` state event adding the user into that room. -When the homeserver does not yet know about the room it cannot do this -directly. Instead, it must take a longer multi-stage handshaking process by -which it first selects a remote homeserver which is already participating in -that room, and use it to assist in the joining process. This is the remote -join handshake. - -This handshake involves the homeserver of the new member wishing to join -(referred to here as the "joining" server), the directory server hosting the -room alias the user is requesting to join with, and a homeserver where existing -room members are already present (referred to as the "resident" server). - -In summary, the remote join handshake consists of the joining server querying -the directory server for information about the room alias; receiving a room ID -and a list of join candidates. The joining server then requests information -about the room from one of the residents. It uses this information to construct -an ``m.room.member`` event which it finally sends to a resident server. - -Conceptually these are three different roles of homeserver. In practice the -directory server is likely to be resident in the room, and so may be selected -by the joining server to be the assisting resident. Likewise, it is likely that -the joining server picks the same candidate resident for both phases of event -construction, though in principle any valid candidate may be used at each time. -Thus, any join handshake can potentially involve anywhere from two to four -homeservers, though most in practice will use just two. - -:: - - Client Joining Directory Resident - Server Server Server - - join request --> - | - directory request -------> - <---------- directory response - | - make_join request -----------------------> - <------------------------------- make_join response - | - send_join request -----------------------> - <------------------------------- send_join response - | - <---------- join response - -The first part of the handshake usually involves using the directory server to -request the room ID and join candidates through the |/query/directory|_ -API endpoint. In the case of a new user joining a room as a result of a received -invite, the joining user's homeserver could optimise this step away by picking -the origin server of that invite message as the join candidate. However, the -joining server should be aware that the origin server of the invite might since -have left the room, so should be prepared to fall back on the regular join flow -if this optimisation fails. - -Once the joining server has the room ID and the join candidates, it then needs -to obtain enough information about the room to fill in the required fields of -the ``m.room.member`` event. It obtains this by selecting a resident from the -candidate list, and using the ``GET /make_join`` endpoint. The resident server -will then reply with enough information for the joining server to fill in the -event. - -The joining server is expected to add or replace the ``origin``, ``origin_server_ts``, -and ``event_id`` on the templated event received by the resident server. This -event is then signed by the joining server. - -To complete the join handshake, the joining server must now submit this new -event to a resident homeserver, by using the ``PUT /send_join`` endpoint. - -The resident homeserver then accepts this event into the room's event graph, -and responds to the joining server with the full set of state for the -newly-joined room. The resident server must also send the event to other servers -participating in the room. - -{{joins_v1_ss_http_api}} - -{{joins_v2_ss_http_api}} - -.. TODO-spec - - (paul) I don't really understand why the full auth_chain events are given - here. What purpose does it serve expanding them out in full, when surely - they'll appear in the state anyway? - -Inviting to a room ------------------- - -When a user on a given homeserver invites another user on the same homeserver, -the homeserver may sign the membership event itself and skip the process defined -here. However, when a user invites another user on a different homeserver, a request -to that homeserver to have the event signed and verified must be made. - -{{invites_v1_ss_http_api}} - -{{invites_v2_ss_http_api}} - -Leaving Rooms (Rejecting Invites) ---------------------------------- - -Normally homeservers can send appropriate ``m.room.member`` events to have users -leave the room, or to reject local invites. Remote invites from other homeservers -do not involve the server in the graph and therefore need another approach to -reject the invite. Joining the room and promptly leaving is not recommended as -clients and servers will interpret that as accepting the invite, then leaving the -room rather than rejecting the invite. - -Similar to the `Joining Rooms`_ handshake, the server which wishes to leave the -room starts with sending a ``/make_leave`` request to a resident server. In the -case of rejecting invites, the resident server may be the server which sent the -invite. After receiving a template event from ``/make_leave``, the leaving server -signs the event and replaces the ``event_id`` with its own. This is then sent to -the resident server via ``/send_leave``. The resident server will then send the -event to other servers in the room. - -{{leaving_v1_ss_http_api}} - -{{leaving_v2_ss_http_api}} - -Third-party invites -------------------- - -.. NOTE:: - More information about third party invites is available in the `Client-Server API`_ - under the Third Party Invites module. - -When a user wants to invite another user in a room but doesn't know the Matrix -ID to invite, they can do so using a third-party identifier (e.g. an e-mail or a -phone number). - -This identifier and its bindings to Matrix IDs are verified by an identity server -implementing the `Identity Service API`_. - -Cases where an association exists for a third-party identifier -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If the third-party identifier is already bound to a Matrix ID, a lookup request -on the identity server will return it. The invite is then processed by the inviting -homeserver as a standard ``m.room.member`` invite event. This is the simplest case. - -Cases where an association doesn't exist for a third-party identifier -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If the third-party identifier isn't bound to any Matrix ID, the inviting -homeserver will request the identity server to store an invite for this identifier -and to deliver it to whoever binds it to its Matrix ID. It will also send an -``m.room.third_party_invite`` event in the room to specify a display name, a token -and public keys the identity server provided as a response to the invite storage -request. - -When a third-party identifier with pending invites gets bound to a Matrix ID, -the identity server will send a POST request to the ID's homeserver as described -in the `Invitation Storage`_ section of the Identity Service API. - -The following process applies for each invite sent by the identity server: - -The invited homeserver will create an ``m.room.member`` invite event containing -a special ``third_party_invite`` section containing the token and a signed object, -both provided by the identity server. - -If the invited homeserver is in the room the invite came from, it can auth the -event and send it. - -However, if the invited homeserver isn't in the room the invite came from, it -will need to request the room's homeserver to auth the event. - -{{third_party_invite_ss_http_api}} - -Verifying the invite -++++++++++++++++++++ - -When a homeserver receives an ``m.room.member`` invite event for a room it's in -with a ``third_party_invite`` object, it must verify that the association between -the third-party identifier initially invited to the room and the Matrix ID that -claims to be bound to it has been verified without having to rely on a third-party -server. - -To do so, it will fetch from the room's state events the ``m.room.third_party_invite`` -event for which the state key matches with the value for the ``token`` key in the -``third_party_invite`` object from the ``m.room.member`` event's content to fetch the -public keys initially delivered by the identity server that stored the invite. - -It will then use these keys to verify that the ``signed`` object (in the -``third_party_invite`` object from the ``m.room.member`` event's content) was -signed by the same identity server. - -Since this ``signed`` object can only be delivered once in the POST request -emitted by the identity server upon binding between the third-party identifier -and the Matrix ID, and contains the invited user's Matrix ID and the token -delivered when the invite was stored, this verification will prove that the -``m.room.member`` invite event comes from the user owning the invited third-party -identifier. - -Public Room Directory ---------------------- - -To complement the `Client-Server API`_'s room directory, homeservers need a -way to query the public rooms for another server. This can be done by making -a request to the ``/publicRooms`` endpoint for the server the room directory -should be retrieved for. - -{{public_rooms_ss_http_api}} - - -Typing Notifications --------------------- - -When a server's users send typing notifications, those notifications need to -be sent to other servers in the room so their users are aware of the same -state. Receiving servers should verify that the user is in the room, and is -a user belonging to the sending server. - -{{definition_ss_event_schemas_m_typing}} - -Presence --------- -The server API for presence is based entirely on exchange of the following -EDUs. There are no PDUs or Federation Queries involved. - -Servers should only send presence updates for users that the receiving server -would be interested in. Such as the receiving server sharing a room -with a given user. - -.. TODO-doc - - Explain the timing-based round-trip reduction mechanism for presence - messages - - Explain the zero-byte presence inference logic - See also: docs/client-server/model/presence - -{{definition_ss_event_schemas_m_presence}} - -Receipts --------- - -Receipts are EDUs used to communicate a marker for a given event. Currently the -only kind of receipt supported is a "read receipt", or where in the event graph -the user has read up to. - -Read receipts for events that a user sent do not need to be sent. It is -implied that by sending the event the user has read up to the event. - -{{definition_ss_event_schemas_m_receipt}} - -Querying for information ------------------------- - -Queries are a way to retrieve information from a homeserver about a resource, -such as a user or room. The endpoints here are often called in conjunction with -a request from a client on the client-server API in order to complete the call. - -There are several types of queries that can be made. The generic endpoint to -represent all queries is described first, followed by the more specific queries -that can be made. - -{{query_ss_http_api}} - -OpenID ------- - -Third party services can exchange an access token previously generated by the -`Client-Server API` for information about a user. This can help verify that a -user is who they say they are without granting full access to the user's account. - -Access tokens generated by the OpenID API are only good for the OpenID API and -nothing else. - -{{openid_ss_http_api}} - -Device Management ------------------ - -Details of a user's devices must be efficiently published to other users and kept -up-to-date. This is critical for reliable end-to-end encryption, in order for users -to know which devices are participating in a room. It's also required for to-device -messaging to work. This section is intended to complement the `Device Management module`_ -of the Client-Server API. - -Matrix currently uses a custom pubsub system for synchronising information -about the list of devices for a given user over federation. When a server -wishes to determine a remote user's device list for the first time, -it should populate a local cache from the result of a ``/user/keys/query`` API -on the remote server. However, subsequent updates to the cache should be applied -by consuming ``m.device_list_update`` EDUs. Each new ``m.device_list_update`` EDU -describes an incremental change to one device for a given user which should replace -any existing entry in the local server's cache of that device list. Servers must send -``m.device_list_update`` EDUs to all the servers who share a room with a given -local user, and must be sent whenever that user's device list changes (i.e. for new or -deleted devices, when that user joins a room which contains servers which are not -already receiving updates for that user's device list, or changes in device information -such as the device's human-readable name). - -Servers send ``m.device_list_update`` EDUs in a sequence per origin user, each with -a unique ``stream_id``. They also include a pointer to the most recent previous EDU(s) -that this update is relative to in the ``prev_id`` field. To simplify implementation -for clustered servers which could send multiple EDUs at the same time, the ``prev_id`` -field should include all ``m.device_list_update`` EDUs which have not been yet been -referenced in a EDU. If EDUs are emitted in series by a server, there should only ever -be one ``prev_id`` in the EDU. - -This forms a simple directed acyclic graph of ``m.device_list_update`` EDUs, showing -which EDUs a server needs to have received in order to apply an update to its local -copy of the remote user's device list. If a server receives an EDU which refers to -a ``prev_id`` it does not recognise, it must resynchronise its list by calling the -``/user/keys/query API`` and resume the process. The response contains a ``stream_id`` -which should be used to correlate with subsequent ``m.device_list_update`` EDUs. - -.. TODO: this whole thing desperately feels like it should just be state in a room, - rather than inventing a whole different DAG. The same room could be used for - profiles etc. - -{{user_devices_ss_http_api}} - -{{definition_ss_event_schemas_m_device_list_update}} - - -End-to-End Encryption ---------------------- - -This section complements the `End-to-End Encryption module`_ of the Client-Server -API. For detailed information about end-to-end encryption, please see that module. - -The APIs defined here are designed to be able to proxy much of the client's request -through to federation, and have the response also be proxied through to the client. - -{{user_keys_ss_http_api}} - -{{definition_ss_event_schemas_m_signing_key_update}} - - -Send-to-device messaging ------------------------- - -.. TODO: add modules to the federation spec and make this a module - -The server API for send-to-device messaging is based on the -``m.direct_to_device`` EDU. There are no PDUs or Federation Queries involved. - -Each send-to-device message should be sent to the destination server using -the following EDU: - -{{definition_ss_event_schemas_m_direct_to_device}} - - -Content Repository ------------------- - -Attachments to events (images, files, etc) are uploaded to a homeserver via the -Content Repository described in the `Client-Server API`_. When a server wishes -to serve content originating from a remote server, it needs to ask the remote -server for the media. - -Servers should use the server described in the Matrix Content URI, which has the -format ``mxc://{ServerName}/{MediaID}``. Servers should use the download endpoint -described in the `Client-Server API`_, being sure to use the ``allow_remote`` -parameter (set to ``false``). - - -Server Access Control Lists (ACLs) ----------------------------------- - -Server ACLs and their purpose are described in the `Server ACLs`_ section of the -Client-Server API. - -When a remote server makes a request, it MUST be verified to be allowed by the -server ACLs. If the server is denied access to a room, the receiving server -MUST reply with a 403 HTTP status code and an ``errcode`` of ``M_FORBIDDEN``. - -The following endpoint prefixes MUST be protected: - -* ``/_matrix/federation/v1/send`` (on a per-PDU basis) -* ``/_matrix/federation/v1/make_join`` -* ``/_matrix/federation/v1/make_leave`` -* ``/_matrix/federation/v1/send_join`` -* ``/_matrix/federation/v2/send_join`` -* ``/_matrix/federation/v1/send_leave`` -* ``/_matrix/federation/v2/send_leave`` -* ``/_matrix/federation/v1/invite`` -* ``/_matrix/federation/v2/invite`` -* ``/_matrix/federation/v1/state`` -* ``/_matrix/federation/v1/state_ids`` -* ``/_matrix/federation/v1/backfill`` -* ``/_matrix/federation/v1/event_auth`` -* ``/_matrix/federation/v1/get_missing_events`` - - -Signing Events --------------- - -Signing events is complicated by the fact that servers can choose to redact -non-essential parts of an event. - -Adding hashes and signatures to outgoing events -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Before signing the event, the *content hash* of the event is calculated as -described below. The hash is encoded using `Unpadded Base64`_ and stored in the -event object, in a ``hashes`` object, under a ``sha256`` key. - -The event object is then *redacted*, following the `redaction -algorithm`_. Finally it is signed as described in `Signing JSON`_, using the -server's signing key (see also `Retrieving server keys`_). - -The signature is then copied back to the original event object. - -See `Persistent Data Unit schema`_ for an example of a signed event. - - -Validating hashes and signatures on received events -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When a server receives an event over federation from another server, the -receiving server should check the hashes and signatures on that event. - -First the signature is checked. The event is redacted following the `redaction -algorithm`_, and the resultant object is checked for a signature from the -originating server, following the algorithm described in `Checking for a signature`_. -Note that this step should succeed whether we have been sent the full event or -a redacted copy. - -The signatures expected on an event are: - -* The ``sender``'s server, unless the invite was created as a result of 3rd party invite. - The sender must already match the 3rd party invite, and the server which actually - sends the event may be a different server. -* For room versions 1 and 2, the server which created the ``event_id``. Other room - versions do not track the ``event_id`` over federation and therefore do not need - a signature from those servers. - -If the signature is found to be valid, the expected content hash is calculated -as described below. The content hash in the ``hashes`` property of the received -event is base64-decoded, and the two are compared for equality. - -If the hash check fails, then it is assumed that this is because we have only -been given a redacted version of the event. To enforce this, the receiving -server should use the redacted copy it calculated rather than the full copy it -received. - -.. _`reference hashes`: - -Calculating the reference hash for an event -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The *reference hash* of an event covers the essential fields of an event, -including content hashes. It is used for event identifiers in some room versions. -See the `room version specification`_ for more information. It is calculated as -follows. - -1. The event is put through the redaction algorithm. - -2. The ``signatures``, ``age_ts``, and ``unsigned`` properties are removed - from the event, if present. - -3. The event is converted into `Canonical JSON`_. - -4. A sha256 hash is calculated on the resulting JSON object. - - -Calculating the content hash for an event -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The *content hash* of an event covers the complete event including the -*unredacted* contents. It is calculated as follows. - -First, any existing ``unsigned``, ``signature``, and ``hashes`` members are -removed. The resulting object is then encoded as `Canonical JSON`_, and the -JSON is hashed using SHA-256. - - -Example code -~~~~~~~~~~~~ - -.. code:: python - - def hash_and_sign_event(event_object, signing_key, signing_name): - # First we need to hash the event object. - content_hash = compute_content_hash(event_object) - event_object["hashes"] = {"sha256": encode_unpadded_base64(content_hash)} - - # Strip all the keys that would be removed if the event was redacted. - # The hashes are not stripped and cover all the keys in the event. - # This means that we can tell if any of the non-essential keys are - # modified or removed. - stripped_object = strip_non_essential_keys(event_object) - - # Sign the stripped JSON object. The signature only covers the - # essential keys and the hashes. This means that we can check the - # signature even if the event is redacted. - signed_object = sign_json(stripped_object, signing_key, signing_name) - - # Copy the signatures from the stripped event to the original event. - event_object["signatures"] = signed_object["signatures"] - - def compute_content_hash(event_object): - # take a copy of the event before we remove any keys. - event_object = dict(event_object) - - # Keys under "unsigned" can be modified by other servers. - # They are useful for conveying information like the age of an - # event that will change in transit. - # Since they can be modified we need to exclude them from the hash. - event_object.pop("unsigned", None) - - # Signatures will depend on the current value of the "hashes" key. - # We cannot add new hashes without invalidating existing signatures. - event_object.pop("signatures", None) - - # The "hashes" key might contain multiple algorithms if we decide to - # migrate away from SHA-2. We don't want to include an existing hash - # output in our hash so we exclude the "hashes" dict from the hash. - event_object.pop("hashes", None) - - # Encode the JSON using a canonical encoding so that we get the same - # bytes on every server for the same JSON object. - event_json_bytes = encode_canonical_json(event_object) - - return hashlib.sha256(event_json_bytes) - -.. TODO - - [[TODO(markjh): Since the ``hash`` object cannot be redacted a server - shouldn't allow too many hashes to be listed, otherwise a server might embed - illicit data within the ``hash`` object. - - We might want to specify a maximum number of keys for the - ``hash`` and we might want to specify the maximum output size of a hash]] - - [[TODO(markjh) We might want to allow the server to omit the output of well - known hash functions like SHA-256 when none of the keys have been redacted]] - - -Security considerations ------------------------ - -When a domain's ownership changes, the new controller of the domain can masquerade -as the previous owner, receiving messages (similarly to email) and request past -messages from other servers. In the future, proposals like -`MSC1228 `_ will address this -issue. - - -.. |/query/directory| replace:: ``/query/directory`` -.. _/query/directory: #get-matrix-federation-v1-query-directory - -.. _`Invitation storage`: ../identity_service/%IDENTITY_RELEASE_LABEL%.html#invitation-storage -.. _`Identity Service API`: ../identity_service/%IDENTITY_RELEASE_LABEL%.html -.. _`Client-Server API`: ../client_server/%CLIENT_RELEASE_LABEL%.html -.. _`Inviting to a room`: #inviting-to-a-room -.. _`Canonical JSON`: ../appendices.html#canonical-json -.. _`Unpadded Base64`: ../appendices.html#unpadded-base64 -.. _`Server ACLs`: ../client_server/%CLIENT_RELEASE_LABEL%.html#module-server-acls -.. _`redaction algorithm`: ../client_server/%CLIENT_RELEASE_LABEL%.html#redactions -.. _`Signing JSON`: ../appendices.html#signing-json -.. _`Checking for a signature`: ../appendices.html#checking-for-a-signature -.. _`Device Management module`: ../client_server/%CLIENT_RELEASE_LABEL%.html#device-management -.. _`End-to-End Encryption module`: ../client_server/%CLIENT_RELEASE_LABEL%.html#end-to-end-encryption -.. _`room version specification`: ../index.html#room-versions -.. _`Client-Server Key Algorithms`: ../client_server/%CLIENT_RELEASE_LABEL%.html#key-algorithms diff --git a/specification/targets.yaml b/specification/targets.yaml deleted file mode 100644 index df66218f..00000000 --- a/specification/targets.yaml +++ /dev/null @@ -1,113 +0,0 @@ -targets: - index: - files: - - index.rst - client_server: - files: - - client_server_api.rst - - { 1: modules.rst } - - { 2: feature_profiles.rst } - - { 2: "group:modules" } # reference a group of files - version_label: "%CLIENT_RELEASE_LABEL%" - application_service: - files: - - application_service_api.rst - version_label: "%APPSERVICE_RELEASE_LABEL%" - server_server: - files: - - server_server_api.rst - version_label: "%SERVER_RELEASE_LABEL%" - identity_service: - files: - - identity_service_api.rst - version_label: "%IDENTITY_RELEASE_LABEL%" - push_gateway: - files: - - push_gateway.rst - version_label: "%PUSH_GATEWAY_RELEASE_LABEL%" - rooms@v1: # this is translated to be rooms/v1.html - files: - - rooms/v1.rst - version_label: v1 - rooms@v2: # this is translated to be rooms/v2.html - files: - - rooms/v2.rst - version_label: v2 - rooms@v3: # this is translated to be rooms/v3.html - files: - - rooms/v3.rst - version_label: v3 - rooms@v4: # this is translated to be rooms/v4.html - files: - - rooms/v4.rst - version_label: v4 - rooms@v5: # this is translated to be rooms/v5.html - files: - - rooms/v5.rst - version_label: v5 - rooms@v6: # this is translated to be rooms/v6.html - files: - - rooms/v6.rst - version_label: v6 - appendices: - files: - - appendices.rst - - appendices/base64.rst - - appendices/signing_json.rst - - appendices/identifier_grammar.rst - - appendices/threepids.rst - - appendices/threat_model.rst - - appendices/test_vectors.rst - proposals: - files: - - proposals_intro.rst - - proposals.rst -groups: # reusable blobs of files when prefixed with 'group:' - modules: - - modules/instant_messaging.rst - - modules/voip_events.rst - - modules/typing_notifications.rst - - modules/receipts.rst - - modules/read_markers.rst - - modules/presence.rst - - modules/content_repo.rst - - modules/send_to_device.rst - - modules/device_management.rst - - modules/end_to_end_encryption.rst - - modules/secrets.rst - - modules/history_visibility.rst - - modules/push.rst - - modules/third_party_invites.rst - - modules/search.rst - - modules/guest_access.rst - - modules/room_previews.rst - - modules/tags.rst - - modules/account_data.rst - - modules/admin.rst - - modules/event_context.rst - - modules/sso_login.rst - - modules/dm.rst - - modules/ignore_users.rst - - modules/stickers.rst - - modules/report_content.rst - - modules/third_party_networks.rst - - modules/openid.rst - - modules/server_acls.rst - - modules/mentions.rst - - modules/room_upgrades.rst - - modules/server_notices.rst - - modules/moderation_policies.rst - - -title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] - -# The templating system doesn't know the right title style to use when generating -# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title -# at the same level (0)", or "make a title one above (+1)". The gendoc script -# will inspect this file and replace these relative styles with actual title -# styles. The templating system will also inspect this file to know which symbols -# to inject. -relative_title_styles: - subtitle: "<" - sametitle: "/" - supertitle: ">" From 62896cdea9ce20ab3946eaa734a6cdcb5c1ffd1e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:02:24 -0600 Subject: [PATCH 06/11] Apply suggestions from code review Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- CONTRIBUTING.rst | 2 +- README.md | 10 +++++----- meta/documentation_style.rst | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b9c66805..e3db28fd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -103,7 +103,7 @@ the ``newsfragments`` directory. The ``type`` can be one of the following: All news fragments must have a brief summary explaining the change in the contents of the file. The summary must end in a full stop to be in line with -the style guide and and formatting must be done using Markdown. +the style guide and formatting must be done using Markdown. Changes that do not change the spec, such as changes to the build script, formatting, CSS, etc should not get a news fragment. diff --git a/README.md b/README.md index 009b94e6..838a5365 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ The Matrix spec is compiled with [Hugo](https://gohugo.io/) (a static site gener * `/themes`: you can use just Hugo or use it with a theme. Themes primarily provide additional templates, which are supplied in a `/themes/$theme_name/layouts` directory. You can use a theme but customise it by providing your own - versions of any of the them layouts in the base `/layouts` directory. That is, if a theme provides - `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of this + versions of any of the theme layouts in the base `/layouts` directory. That is, if a theme provides + `/themes/$theme_name/layouts/sidebar.html` and you provide `/layouts/sidebar.html`, then your version of the template will be used. It also has the following top-level file: @@ -51,7 +51,7 @@ Additionally, the following directories may be of interest: * `/event-schemas`: [JSON Schema](http://json-schema.org/) definitions for the spec. * `/data-definitions`: Bits of structured data consumable by Matrix implementations. * `/meta`: Documentation relating to the spec's processes that are otherwise untracked (release instructions, etc). -* `/scripts`: Various scripts for generating the spec. +* `/scripts`: Various scripts for generating the spec and validating its contents. * `/proposals`: Matrix Spec Change (MSC) proposals. See . ## Authoring changes to the spec @@ -62,7 +62,7 @@ place after an MSC has been accepted, not as part of a proposal itself. 1. Install the extended version (often the OS default) of Hugo: 2. Run `git submodule update --init --recursive` for good measure. 3. Run `npm i` to install the dependencies. Note that this will require NodeJS to be installed. -4. Run `npm run get-proposals` to seed the proposals data. This is not required. +4. Run `npm run get-proposals` to seed proposal data. This is merely for populating the content of the "Spec Change Proposals" page and is not required. 5. Run `hugo serve` to run a local webserver which builds whenever a file change is detected. If watching doesn't appear to be working for you, try `hugo serve --disableFastRender` instead. 6. Edit the specification 🙂 @@ -73,7 +73,7 @@ Awesome. If you're looking at making design-related changes to the spec site, pl ## Building the specification -If for some reason you're not a CI/CD system and want to render the spec yourself, follow the above steps for authoring +If for some reason you're not a CI/CD system and want to render a static version of the spec for yourself, follow the above steps for authoring changes to the specification and instead of `hugo serve` run `hugo -d "spec"` - this will generate the spec to `/spec`. If you'd like to serve the spec off a path instead of a domain root (eg: `/unstable`), add `--baseURL "/unstable"` to the `hugo -d "spec"` command. diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index 5528f73e..cfec5e3b 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -8,7 +8,7 @@ in. Format ------ -Documentation is written either in github-flavored markdown. +Documentation is written in github-flavored markdown. Sections -------- From b455fc73f1ca67e8a5da498672589580a36ae7a2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:42:33 -0600 Subject: [PATCH 07/11] Reference drafts properly --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 12ec2aec..e09846a8 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -337,7 +337,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { log.Printf("Serving pr %s (%s)\n", branchName, sha) } else if strings.ToLower(branchName) == "head" || branchName == "master" || - strings.HasPrefix(branchName, "drafts/") { + strings.HasPrefix(branchName, "attic/drafts/") { branchSHA, err := s.lookupBranch(branchName) if err != nil { writeError(w, 400, err) From b518e4ade0a0c0149f9c3116ce075863a0466141 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:43:05 -0600 Subject: [PATCH 08/11] Finish sentence --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 838a5365..ad98da40 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The Matrix spec is compiled with [Hugo](https://gohugo.io/) (a static site gener * `/data`: this can contain TOML, YAML, or JSON files. Files kept here are directly available to template code as [data objects](https://gohugo.io/templates/data-templates/), so templates don't need to load them from a file and - parse them. This is also where our + parse them. This is also where our Swagger/OpenAPI definitions are. * `/layouts`: this contains [Hugo templates](https://gohugo.io/templates/). Some templates define the overall layout of a page: for example, whether it has header, footer, sidebar, and so on. From a2280c7815ad0d4a7322ee83e4cf609fa7af2ddc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:43:38 -0600 Subject: [PATCH 09/11] s --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ad98da40..3f85920d 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ This repository contains the Matrix Specification, rendered at [spec.matrix.org](http://spec.matrix.org/). -Developers looking to use Matrix should join [#matrix-dev:matrix.org](http://matrix.to/#/#matrix-dev:matrix.org) +Developers looking to use Matrix should join [#matrix-dev:matrix.org](https://matrix.to/#/#matrix-dev:matrix.org) on Matrix for help. -Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](http://matrix.to/#/#matrix-spec:matrix.org). We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details. +Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details. ## Structure From 7e6ab07f96b2ca9924069f116fad117db220abf0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:44:56 -0600 Subject: [PATCH 10/11] Line length and words --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3f85920d..bfcfd87d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ This repository contains the Matrix Specification, rendered at [spec.matrix.org] Developers looking to use Matrix should join [#matrix-dev:matrix.org](https://matrix.to/#/#matrix-dev:matrix.org) on Matrix for help. -Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details. +Spec authors and proposal writers are welcome to join [#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). +We welcome contributions! See [CONTRIBUTING.rst](./CONTRIBUTING.rst) for details. ## Structure @@ -62,7 +63,8 @@ place after an MSC has been accepted, not as part of a proposal itself. 1. Install the extended version (often the OS default) of Hugo: 2. Run `git submodule update --init --recursive` for good measure. 3. Run `npm i` to install the dependencies. Note that this will require NodeJS to be installed. -4. Run `npm run get-proposals` to seed proposal data. This is merely for populating the content of the "Spec Change Proposals" page and is not required. +4. Run `npm run get-proposals` to seed proposal data. This is merely for populating the content of the "Spec Change Proposals" + page and is not required. 5. Run `hugo serve` to run a local webserver which builds whenever a file change is detected. If watching doesn't appear to be working for you, try `hugo serve --disableFastRender` instead. 6. Edit the specification 🙂 @@ -73,18 +75,18 @@ Awesome. If you're looking at making design-related changes to the spec site, pl ## Building the specification -If for some reason you're not a CI/CD system and want to render a static version of the spec for yourself, follow the above steps for authoring -changes to the specification and instead of `hugo serve` run `hugo -d "spec"` - this will generate the spec to `/spec`. -If you'd like to serve the spec off a path instead of a domain root (eg: `/unstable`), add `--baseURL "/unstable"` to -the `hugo -d "spec"` command. +If for some reason you're not a CI/CD system and want to render a static version of the spec for yourself, follow the above +steps for authoring changes to the specification and instead of `hugo serve` run `hugo -d "spec"` - this will generate the +spec to `/spec`. If you'd like to serve the spec off a path instead of a domain root (eg: `/unstable`), add `--baseURL "/unstable"` +to the `hugo -d "spec"` command. -For building the swagger definitions, create a python3 virtualenv and activate it. Then run `pip install -r ./scripts/requirements.txt` and finally `python ./scripts/dump-swagger.py` to generate it to `./scripts/swagger/api-docs.json`. -To make use of the generated file, there are a number of options: +For building the swagger definitions, create a python3 virtualenv and activate it. Then run `pip install -r ./scripts/requirements.txt` +and finally `python ./scripts/dump-swagger.py` to generate it to `./scripts/swagger/api-docs.json`. To make use of the generated file, +there are a number of options: -* It can be uploaded from your filesystem to an online editor/viewer such as - http://editor.swagger.io/ +* It can be uploaded from your filesystem to an online editor/viewer such as [on the swagger website](http://editor.swagger.io/). * You can run a local HTTP server by running `./scripts/swagger-http-server.py`, and then view the documentation via an - online viewer; for example, at + online viewer; for example, at . * You can host the swagger UI yourself. See for advice on how to do so. From 2603ea6339f5041cfddfe58b611e43a1b6175e14 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Apr 2021 09:45:23 -0600 Subject: [PATCH 11/11] goodbye legacy config --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 17a9e62d..19a6ee8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,5 @@ [ tool.gilesbot ] - [ tool.gilesbot.circleci_artifacts.legacydocs ] - url = "gen/index.html" - message = "Click details to preview the legacy HTML documentation." - [ tool.gilesbot.circleci_artifacts.docs ] url = "public/index.html" message = "Click details to preview the HTML documentation."