diff --git a/assets-hugo/scss/custom.scss b/assets-hugo/scss/custom.scss index 13676acb..7b80d434 100644 --- a/assets-hugo/scss/custom.scss +++ b/assets-hugo/scss/custom.scss @@ -209,6 +209,39 @@ footer { } +/* Styles for alert boxes */ +.alert { + &.note { + &:not(.omit-title):before { + content: "INFO: "; + font-weight: $font-weight-bold; + } + border: 2px solid $note; + border-left-width: 5px; + background: $note-background; + } + + &.rationale { + &:not(.omit-title):before { + content: "RATIONALE: "; + font-weight: $font-weight-bold; + } + border: 2px solid $note; + border-left-width: 5px; + background: $note-background; + } + + &.warning { + &:not(.omit-title):before { + content: "WARNING: "; + font-weight: $font-weight-bold; + } + border: 2px solid $warning; + border-left-width: 5px; + background: $warning-background; + } +} + /* Miscellaneous custom bits */ /* Update link colours for MAtrix style */ @@ -226,8 +259,18 @@ a, a:hover { max-width: 100%; } +/* The default CSS applies a style for blockquotes but only to immediate children +of .td-content. This applies the same style to any blockquotes that descend from +.td-content. */ +.td-content blockquote { + padding: 0 0 0 1rem; + margin-bottom: $spacer; + color: $gray-600; + border-left: 6px solid $secondary; +} + /* -Make padding symmetrical (this selector is used to apply padding-left: 3rem) +Make padding symmetrical (this selector is used in the default styles to apply padding-left: 3rem) */ .pl-md-5, .px-md-5 { padding-right: 3rem; diff --git a/content/_index.md b/content/_index.md index 7740f2d1..0b82c74a 100644 --- a/content/_index.md +++ b/content/_index.md @@ -13,20 +13,24 @@ together existing communication silos - providing the basis of a new open real-time communication ecosystem. To propose a change to the Matrix Spec, see the explanations at -[Proposals for Spec Changes to Matrix](proposals). +[Proposals for Spec Changes to Matrix](/proposals). ## Matrix APIs The specification consists of the following parts: -{{apis}} +* [Client-Server API](client-server-api) +* [Server-Server API](server-server-api) +* [Application Service API](application-service-api) +* [Identity Service API](identity-service-api) +* [Push Gateway API](push-gateway-api) Additionally, this introduction page contains the key baseline information required to understand the specific APIs, including the sections on [room versions](#room-versions) and [overall architecture](#architecture). -The [Appendices](appendices.html) contain supplemental information not +The [Appendices](/appendices) contain supplemental information not specific to one of the above APIs. The [Matrix Client-Server API Swagger @@ -35,13 +39,13 @@ browsing the Client-Server API. ### Matrix versions -Note - +{{% boxes/note %}} As of June 10th 2019, the Matrix specification is considered out of beta -indicating that all currently released APIs are considered stable and secure to the best of our knowledge, and the spec should contain the complete information necessary to develop production-grade implementations of Matrix without the need for external reference. +{{% /boxes/note %}} Matrix 1.0 (released June 10th, 2019) consists of the following minimum API versions: @@ -123,7 +127,7 @@ an interoperable and federated manner. ### Spec Change Proposals To propose a change to the Matrix Spec, see the explanations at -[Proposals for Spec Changes to Matrix](proposals). +[Proposals for Spec Changes to Matrix](/proposals). ## Architecture @@ -159,20 +163,21 @@ contents and then adds it to its copy of the room's event graph. Client B then receives the message from his homeserver via a long-lived GET request. - How data flows between clients - ============================== +How data flows between clients: +``` { Matrix client A } { Matrix client B } - ^ | ^ | - | events | Client-Server API | events | - | V | V + ^ | ^ | + | events | Client-Server API | events | + | V | V +------------------+ +------------------+ | |---------( HTTPS )--------->| | | homeserver | | homeserver | | |<--------( HTTPS )----------| | +------------------+ Server-Server API +------------------+ - History Synchronisation - (Federation) + History Synchronisation + (Federation) +``` ### Users @@ -183,7 +188,7 @@ which allocated the account and has the form: @localpart:domain See ['Identifier Grammar' in the -appendices](appendices.html#identifier-grammar) for full details of the +appendices](/appendices#identifier-grammar) for full details of the structure of user IDs. ### Devices @@ -265,7 +270,7 @@ contain a domain, it is simply for globally namespacing room IDs. The room does NOT reside on the domain specified. See ['Identifier Grammar' in the -appendices](appendices.html#identifier-grammar) for full details of the +appendices](/appendices#identifier-grammar) for full details of the structure of a room ID. The following conceptual diagram shows an `m.room.message` event being @@ -334,13 +339,13 @@ participating servers in a room, currently using full mesh topology. Servers may also request backfill of events over federation from the other servers participating in a room. -Note - +{{% boxes/note %}} Events are not limited to the types defined in this specification. New or custom event types can be created on a whim using the Java package naming convention. For example, a `com.example.game.score` event can be sent by clients and other clients would receive it through Matrix, assuming the client has access to the `com.example` namespace. +{{% /boxes/note %}} #### Room Aliases @@ -349,7 +354,7 @@ Each room can also have multiple "Room Aliases", which look like: #room_alias:domain See ['Identifier Grammar' in the -appendices](appendices.html#identifier-grammar) for full details of the +appendices](/appendices#identifier-grammar) for full details of the structure of a room alias. A room alias "points" to a room ID and is the human-readable label by @@ -493,17 +498,17 @@ the default room version when creating new rooms. The available room versions are: -- [Version 1](rooms/v1.html) - **Stable**. The current version of most +- [Version 1](/rooms/v1) - **Stable**. The current version of most rooms. -- [Version 2](rooms/v2.html) - **Stable**. Implements State Resolution +- [Version 2](/rooms/v2) - **Stable**. Implements State Resolution Version 2. -- [Version 3](rooms/v3.html) - **Stable**. Introduces events whose IDs +- [Version 3](/rooms/v3) - **Stable**. Introduces events whose IDs are the event's hash. -- [Version 4](rooms/v4.html) - **Stable**. Builds on v3 by using +- [Version 4](/rooms/v4) - **Stable**. Builds on v3 by using URL-safe base64 for event IDs. -- [Version 5](rooms/v5.html) - **Stable**. Introduces enforcement of +- [Version 5](/rooms/v5) - **Stable**. Introduces enforcement of signing key validity periods. -- [Version 6](rooms/v6.html) - **Stable**. Alters several +- [Version 6](/rooms/v6) - **Stable**. Alters several authorization rules for events. ## Specification Versions diff --git a/content/appendices.md b/content/appendices.md new file mode 100644 index 00000000..63caebf6 --- /dev/null +++ b/content/appendices.md @@ -0,0 +1,1051 @@ +--- +title: "Appendices" +weight: 70 +type: docs +--- + +## Unpadded Base64 + +*Unpadded* Base64 refers to 'standard' Base64 encoding as defined in +[RFC 4648](https://tools.ietf.org/html/rfc4648), without "=" padding. +Specifically, where RFC 4648 requires that encoded data be padded to a +multiple of four characters using `=` characters, unpadded Base64 omits +this padding. + +For reference, RFC 4648 uses the following alphabet for Base 64: + + Value Encoding Value Encoding Value Encoding Value Encoding + 0 A 17 R 34 i 51 z + 1 B 18 S 35 j 52 0 + 2 C 19 T 36 k 53 1 + 3 D 20 U 37 l 54 2 + 4 E 21 V 38 m 55 3 + 5 F 22 W 39 n 56 4 + 6 G 23 X 40 o 57 5 + 7 H 24 Y 41 p 58 6 + 8 I 25 Z 42 q 59 7 + 9 J 26 a 43 r 60 8 + 10 K 27 b 44 s 61 9 + 11 L 28 c 45 t 62 + + 12 M 29 d 46 u 63 / + 13 N 30 e 47 v + 14 O 31 f 48 w + 15 P 32 g 49 x + 16 Q 33 h 50 y + +Examples of strings encoded using unpadded Base64: + + UNPADDED_BASE64("") = "" + UNPADDED_BASE64("f") = "Zg" + UNPADDED_BASE64("fo") = "Zm8" + UNPADDED_BASE64("foo") = "Zm9v" + UNPADDED_BASE64("foob") = "Zm9vYg" + UNPADDED_BASE64("fooba") = "Zm9vYmE" + UNPADDED_BASE64("foobar") = "Zm9vYmFy" + +When decoding Base64, implementations SHOULD accept input with or +without padding characters wherever possible, to ensure maximum +interoperability. + +## Signing JSON + +Various points in the Matrix specification require JSON objects to be +cryptographically signed. This requires us to encode the JSON as a +binary string. Unfortunately the same JSON can be encoded in different +ways by changing how much white space is used or by changing the order +of keys within objects. + +Signing an object therefore requires it to be encoded as a sequence of +bytes using [Canonical JSON](#canonical-json), computing the signature +for that sequence and then adding the signature to the original JSON +object. + +### Canonical JSON + +We define the canonical JSON encoding for a value to be the shortest +UTF-8 JSON encoding with dictionary keys lexicographically sorted by +Unicode codepoint. Numbers in the JSON must be integers in the range +`[-(2**53)+1, (2**53)-1]`. + +We pick UTF-8 as the encoding as it should be available to all platforms +and JSON received from the network is likely to be already encoded using +UTF-8. We sort the keys to give a consistent ordering. We force integers +to be in the range where they can be accurately represented using IEEE +double precision floating point numbers since a number of JSON libraries +represent all numbers using this representation. + +{{% boxes/warning %}} +Events in room versions 1, 2, 3, 4, and 5 might not be fully compliant +with these restrictions. Servers SHOULD be capable of handling JSON +which is considered invalid by these restrictions where possible. + +The most notable consideration is that integers might not be in the +range specified above. +{{% /boxes/warning %}} + +{{% boxes/note %}} +Float values are not permitted by this encoding. +{{% /boxes/note %}} + +```py +import json + +def canonical_json(value): + return json.dumps( + value, + # Encode code-points outside of ASCII as UTF-8 rather than \u escapes + ensure_ascii=False, + # Remove unnecessary white space. + separators=(',',':'), + # Sort the keys of dictionaries. + sort_keys=True, + # Encode the resulting Unicode as UTF-8 bytes. + ).encode("UTF-8") +``` + +#### Grammar + +Adapted from the grammar in +removing insignificant whitespace, fractions, exponents and redundant +character escapes. + + value = false / null / true / object / array / number / string + false = %x66.61.6c.73.65 + null = %x6e.75.6c.6c + true = %x74.72.75.65 + object = %x7B [ member *( %x2C member ) ] %7D + member = string %x3A value + array = %x5B [ value *( %x2C value ) ] %5B + number = [ %x2D ] int + int = %x30 / ( %x31-39 *digit ) + digit = %x30-39 + string = %x22 *char %x22 + char = unescaped / %x5C escaped + unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + escaped = %x22 ; " quotation mark U+0022 + / %x5C ; \ reverse solidus U+005C + / %x62 ; b backspace U+0008 + / %x66 ; f form feed U+000C + / %x6E ; n line feed U+000A + / %x72 ; r carriage return U+000D + / %x74 ; t tab U+0009 + / %x75.30.30.30 (%x30-37 / %x62 / %x65-66) ; u000X + / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X + +#### Examples + +To assist in the development of compatible implementations, the +following test values may be useful for verifying the canonical +transformation code. + +Given the following JSON object: + +```json +{} +``` + +The following canonical JSON should be produced: + +```json +{} +``` + +Given the following JSON object: + +```json +{ + "one": 1, + "two": "Two" +} +``` + +The following canonical JSON should be produced: + +```json +{"one":1,"two":"Two"} +``` + +Given the following JSON object: + +```json +{ + "b": "2", + "a": "1" +} +``` + +The following canonical JSON should be produced: + +```json +{"a":"1","b":"2"} +``` + +Given the following JSON object: + +```json +{"b":"2","a":"1"} +``` + +The following canonical JSON should be produced: + +```json +{"a":"1","b":"2"} +``` + +Given the following JSON object: + +```json +{ + "auth": { + "success": true, + "mxid": "@john.doe:example.com", + "profile": { + "display_name": "John Doe", + "three_pids": [ + { + "medium": "email", + "address": "john.doe@example.org" + }, + { + "medium": "msisdn", + "address": "123456789" + } + ] + } + } +} +``` + +The following canonical JSON should be produced: + +```json +{"auth":{"mxid":"@john.doe:example.com","profile":{"display_name":"John Doe","three_pids":[{"address":"john.doe@example.org","medium":"email"},{"address":"123456789","medium":"msisdn"}]},"success":true}} +``` + +Given the following JSON object: + +```json +{ + "a": "日本語" +} +``` + +The following canonical JSON should be produced: + +```json +{"a":"日本語"} +``` + +Given the following JSON object: + +```json +{ + "本": 2, + "日": 1 +} +``` + +The following canonical JSON should be produced: + +```json +{"日":1,"本":2} +``` + +Given the following JSON object: + +```json +{ + "a": "\u65E5" +} +``` + +The following canonical JSON should be produced: + +```json +{"a":"日"} +``` + +Given the following JSON object: + +```json +{ + "a": null +} +``` + +The following canonical JSON should be produced: + +```json +{"a":null} +``` + +### Signing Details + +JSON is signed by encoding the JSON object without `signatures` or keys +grouped as `unsigned`, using the canonical encoding described above. The +JSON bytes are then signed using the signature algorithm and the +signature is encoded using [unpadded Base64](#unpadded-base64). The resulting base64 +signature is added to an object under the *signing key identifier* which +is added to the `signatures` object under the name of the entity signing +it which is added back to the original JSON object along with the +`unsigned` object. + +The *signing key identifier* is the concatenation of the *signing +algorithm* and a *key identifier*. The *signing algorithm* identifies +the algorithm used to sign the JSON. The currently supported value for +*signing algorithm* is `ed25519` as implemented by NACL +(). The *key identifier* is used to distinguish +between different signing keys used by the same entity. + +The `unsigned` object and the `signatures` object are not covered by the +signature. Therefore intermediate entities can add unsigned data such as +timestamps and additional signatures. + +```json +{ + "name": "example.org", + "signing_keys": { + "ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ" + }, + "unsigned": { + "age_ts": 922834800000 + }, + "signatures": { + "example.org": { + "ed25519:1": "s76RUgajp8w172am0zQb/iPTHsRnb4SkrzGoeCOSFfcBY2V/1c8QfrmdXHpvnc2jK5BD1WiJIxiMW95fMjK7Bw" + } + } +} +``` + +```py +def sign_json(json_object, signing_key, signing_name): + signatures = json_object.pop("signatures", {}) + unsigned = json_object.pop("unsigned", None) + + signed = signing_key.sign(encode_canonical_json(json_object)) + signature_base64 = encode_base64(signed.signature) + + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + signatures.setdefault(signing_name, {})[key_id] = signature_base64 + + json_object["signatures"] = signatures + if unsigned is not None: + json_object["unsigned"] = unsigned + + return json_object +``` + +### Checking for a Signature + +To check if an entity has signed a JSON object an implementation does +the following: + +1. Checks if the `signatures` member of the object contains an entry + with the name of the entity. If the entry is missing then the check + fails. +2. Removes any *signing key identifiers* from the entry with algorithms + it doesn't understand. If there are no *signing key identifiers* + left then the check fails. +3. Looks up *verification keys* for the remaining *signing key + identifiers* either from a local cache or by consulting a trusted + key server. If it cannot find a *verification key* then the check + fails. +4. Decodes the base64 encoded signature bytes. If base64 decoding fails + then the check fails. +5. Removes the `signatures` and `unsigned` members of the object. +6. Encodes the remainder of the JSON object using the [Canonical + JSON](#canonical-json) encoding. +7. Checks the signature bytes against the encoded object using the + *verification key*. If this fails then the check fails. Otherwise + the check succeeds. + +## Identifier Grammar + +Some identifiers are specific to given room versions, please refer to +the [room versions specification](/#room-versions) for more +information. + +### Server Name + +A homeserver is uniquely identified by its server name. This value is +used in a number of identifiers, as described below. + +The server name represents the address at which the homeserver in +question can be reached by other homeservers. All valid server names are +included by the following grammar: + + server_name = hostname [ ":" port ] + + port = 1*5DIGIT + + hostname = IPv4address / "[" IPv6address "]" / dns-name + + IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + + IPv6address = 2*45IPv6char + + IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." + ; 0-9, A-F, a-f, :, . + + dns-name = 1*255dns-char + + dns-char = DIGIT / ALPHA / "-" / "." + +— in other words, the server name is the hostname, followed by an +optional numeric port specifier. The hostname may be a dotted-quad IPv4 +address literal, an IPv6 address literal surrounded with square +brackets, or a DNS name. + +IPv4 literals must be a sequence of four decimal numbers in the range 0 +to 255, separated by `.`. IPv6 literals must be as specified by +[RFC3513, section 2.2](https://tools.ietf.org/html/rfc3513#section-2.2). + +DNS names for use with Matrix should follow the conventional +restrictions for internet hostnames: they should consist of a series of +labels separated by `.`, where each label consists of the alphanumeric +characters or hyphens. + +Examples of valid server names are: + +- `matrix.org` +- `matrix.org:8888` +- `1.2.3.4` (IPv4 literal) +- `1.2.3.4:1234` (IPv4 literal with explicit port) +- `[1234:5678::abcd]` (IPv6 literal) +- `[1234:5678::abcd]:5678` (IPv6 literal with explicit port) + +{{% boxes/note %}} +This grammar is based on the standard for internet host names, as +specified by [RFC1123, section +2.1](https://tools.ietf.org/html/rfc1123#page-13), with an extension for +IPv6 literals. +{{% /boxes/note %}} + +Server names must be treated case-sensitively: in other words, +`@user:matrix.org` is a different person from `@user:MATRIX.ORG`. + +Some recommendations for a choice of server name follow: + +- The length of the complete server name should not exceed 230 + characters. +- Server names should not use upper-case characters. + +### Common Identifier Format + +The Matrix protocol uses a common format to assign unique identifiers to +a number of entities, including users, events and rooms. Each identifier +takes the form: + + &string + +where `&` represents a 'sigil' character; `string` is the string which +makes up the identifier. + +The sigil characters are as follows: + +- `@`: User ID +- `!`: Room ID +- `$`: Event ID +- `+`: Group ID +- `#`: Room alias + +User IDs, group IDs, room IDs, room aliases, and sometimes event IDs +take the form: + + &localpart:domain + +where `domain` is the [server name](#server-name) of the homeserver +which allocated the identifier, and `localpart` is an identifier +allocated by that homeserver. + +The precise grammar defining the allowable format of an identifier +depends on the type of identifier. For example, event IDs can sometimes +be represented with a `domain` component under some conditions - see the +[Event IDs](#room-ids-and-event-ids) section below for more information. + +#### User Identifiers + +Users within Matrix are uniquely identified by their Matrix user ID. The +user ID is namespaced to the homeserver which allocated the account and +has the form: + + @localpart:domain + +The `localpart` of a user ID is an opaque identifier for that user. It +MUST NOT be empty, and MUST contain only the characters `a-z`, `0-9`, +`.`, `_`, `=`, `-`, and `/`. + +The `domain` of a user ID is the [server name](#server-name) of the +homeserver which allocated the account. + +The length of a user ID, including the `@` sigil and the domain, MUST +NOT exceed 255 characters. + +The complete grammar for a legal user ID is: + + user_id = "@" user_id_localpart ":" server_name + user_id_localpart = 1*user_id_char + user_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + +{{% boxes/rationale %}} +A number of factors were considered when defining the allowable +characters for a user ID. + +Firstly, we chose to exclude characters outside the basic US-ASCII +character set. User IDs are primarily intended for use as an identifier +at the protocol level, and their use as a human-readable handle is of +secondary benefit. Furthermore, they are useful as a last-resort +differentiator between users with similar display names. Allowing the +full Unicode character set would make very difficult for a human to +distinguish two similar user IDs. The limited character set used has the +advantage that even a user unfamiliar with the Latin alphabet should be +able to distinguish similar user IDs manually, if somewhat laboriously. + +We chose to disallow upper-case characters because we do not consider it +valid to have two user IDs which differ only in case: indeed it should +be possible to reach `@user:matrix.org` as `@USER:matrix.org`. However, +user IDs are necessarily used in a number of situations which are +inherently case-sensitive (notably in the `state_key` of `m.room.member` +events). Forbidding upper-case characters (and requiring homeservers to +downcase usernames when creating user IDs for new users) is a relatively +simple way to ensure that `@USER:matrix.org` cannot refer to a different +user to `@user:matrix.org`. + +Finally, we decided to restrict the allowable punctuation to a very +basic set to reduce the possibility of conflicts with special characters +in various situations. For example, "\*" is used as a wildcard in some +APIs (notably the filter API), so it cannot be a legal user ID +character. + +The length restriction is derived from the limit on the length of the +`sender` key on events; since the user ID appears in every event sent by +the user, it is limited to ensure that the user ID does not dominate +over the actual content of the events. +{{% /boxes/rationale %}} + +Matrix user IDs are sometimes informally referred to as MXIDs. + +##### Historical User IDs + +Older versions of this specification were more tolerant of the +characters permitted in user ID localparts. There are currently active +users whose user IDs do not conform to the permitted character set, and +a number of rooms whose history includes events with a `sender` which +does not conform. In order to handle these rooms successfully, clients +and servers MUST accept user IDs with localparts from the expanded +character set: + + extended_user_id_char = %x21-39 / %x3B-7E ; all ASCII printing chars except : + +##### Mapping from other character sets + +In certain circumstances it will be desirable to map from a wider +character set onto the limited character set allowed in a user ID +localpart. Examples include a homeserver creating a user ID for a new +user based on the username passed to `/register`, or a bridge mapping +user ids from another protocol. + +Implementations are free to do this mapping however they choose. Since +the user ID is opaque except to the implementation which created it, the +only requirement is that the implementation can perform the mapping +consistently. However, we suggest the following algorithm: + +1. Encode character strings as UTF-8. +2. Convert the bytes `A-Z` to lower-case. + - In the case where a bridge must be able to distinguish two + different users with ids which differ only by case, escape + upper-case characters by prefixing with `_` before downcasing. + For example, `A` becomes `_a`. Escape a real `_` with a second + `_`. +3. Encode any remaining bytes outside the allowed character set, as + well as `=`, as their hexadecimal value, prefixed with `=`. For + example, `#` becomes `=23`; `á` becomes `=c3=a1`. + +{{% boxes/rationale %}} +The suggested mapping is an attempt to preserve human-readability of +simple ASCII identifiers (unlike, for example, base-32), whilst still +allowing representation of *any* character (unlike punycode, which +provides no way to encode ASCII punctuation). +{{% /boxes/rationale %}} + +#### Room IDs and Event IDs + +A room has exactly one room ID. A room ID has the format: + + !opaque_id:domain + +An event has exactly one event ID. The format of an event ID depends +upon the [room version specification](/#room-versions). + +The `domain` of a room ID is the [server name](#server-name) of the +homeserver which created the room/event. The domain is used only for +namespacing to avoid the risk of clashes of identifiers between +different homeservers. There is no implication that the room or event in +question is still available at the corresponding homeserver. + +Event IDs and Room IDs are case-sensitive. They are not meant to be +human-readable. They are intended to be treated as fully opaque strings +by clients. + +#### Group Identifiers + +Groups within Matrix are uniquely identified by their group ID. The +group ID is namespaced to the group server which hosts this group and +has the form: + + +localpart:domain + +The `localpart` of a group ID is an opaque identifier for that group. It +MUST NOT be empty, and MUST contain only the characters `a-z`, `0-9`, +`.`, `_`, `=`, `-`, and `/`. + +The `domain` of a group ID is the [server name](#server-name) of the +group server which hosts this group. + +The length of a group ID, including the `+` sigil and the domain, MUST +NOT exceed 255 characters. + +The complete grammar for a legal group ID is: + + group_id = "+" group_id_localpart ":" server_name + group_id_localpart = 1*group_id_char + group_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + +#### Room Aliases + +A room may have zero or more aliases. A room alias has the format: + + #room_alias:domain + +The `domain` of a room alias is the [server name](#server-name) of the +homeserver which created the alias. Other servers may contact this +homeserver to look up the alias. + +Room aliases MUST NOT exceed 255 bytes (including the `#` sigil and the +domain). + +#### matrix.to navigation + +{{% boxes/note %}} +This namespacing is in place pending a `matrix://` (or similar) URI +scheme. This is **not** meant to be interpreted as an available web +service - see below for more details. +{{% /boxes/note %}} + +Rooms, users, aliases, and groups may be represented as a "matrix.to" +URI. This URI can be used to reference particular objects in a given +context, such as mentioning a user in a message or linking someone to a +particular point in the room's history (a permalink). + +A matrix.to URI has the following format, based upon the specification +defined in RFC 3986: + +> <identifier>/<extra +> parameter>?<additional arguments> + +The identifier may be a room ID, room alias, user ID, or group ID. The +extra parameter is only used in the case of permalinks where an event ID +is referenced. The matrix.to URI, when referenced, must always start +with `https://matrix.to/#/` followed by the identifier. + +The `` and the preceding question mark are +optional and only apply in certain circumstances, documented below. + +Clients should not rely on matrix.to URIs falling back to a web server +if accessed and instead should perform some sort of action within the +client. For example, if the user were to click on a matrix.to URI for a +room alias, the client may open a view for the user to participate in +the room. + +The components of the matrix.to URI (`` and +``) are to be percent-encoded as per RFC 3986. + +Examples of matrix.to URIs are: + +- Room alias: `https://matrix.to/#/%23somewhere%3Aexample.org` +- Room: `https://matrix.to/#/!somewhere%3Aexample.org` +- Permalink by room: + `https://matrix.to/#/!somewhere%3Aexample.org/%24event%3Aexample.org` +- Permalink by room alias: + `https://matrix.to/#/%23somewhere:example.org/%24event%3Aexample.org` +- User: `https://matrix.to/#/%40alice%3Aexample.org` +- Group: `https://matrix.to/#/%2Bexample%3Aexample.org` + +{{% boxes/note %}} +Historically, clients have not produced URIs which are fully encoded. +Clients should try to interpret these cases to the best of their +ability. For example, an unencoded room alias should still work within +the client if possible. +{{% /boxes/note %}} + +{{% boxes/note %}} +Clients should be aware that decoding a matrix.to URI may result in +extra slashes appearing due to some [room +versions](/#room-versions). These slashes should normally be +encoded when producing matrix.to URIs, however. +{{% /boxes/note %}} + +##### Routing + +Room IDs are not routable on their own as there is no reliable domain to +send requests to. This is partially mitigated with the addition of a +`via` argument on a matrix.to URI, however the problem of routability is +still present. Clients should do their best to route Room IDs to where +they need to go, however they should also be aware of [issue +\#1579](https://github.com/matrix-org/matrix-doc/issues/1579). + +A room (or room permalink) which isn't using a room alias should supply +at least one server using `via` in the ``, like +so: +`https://matrix.to/!somewhere%3Aexample.org?via=example.org&via=alt.example.org`. +The parameter can be supplied multiple times to specify multiple servers +to try. + +The values of `via` are intended to be passed along as the `server_name` +parameters on the Client Server `/join` API. + +When generating room links and permalinks, the application should pick +servers which have a high probability of being in the room in the +distant future. How these servers are picked is left as an +implementation detail, however the current recommendation is to pick 3 +unique servers based on the following criteria: + +- The first server should be the server of the highest power level + user in the room, provided they are at least power level 50. If no + user meets this criterion, pick the most popular server in the room + (most joined users). The rationale for not picking users with power + levels under 50 is that they are unlikely to be around into the + distant future while higher ranking users (and therefore servers) + are less likely to give up their power and move somewhere else. Most + rooms in the public federation have a power level 100 user and have + not deviated from the default structure where power level 50 users + have moderator-style privileges. +- The second server should be the next highest server by population, + or the first highest by population if the first server was based on + a user's power level. The rationale for picking popular servers is + that the server is unlikely to be removed as the room naturally + grows in membership due to that server joining users. The server + could be refused participation in the future due to server ACLs or + similar, however the chance of that happening to a server which is + organically joining the room is unlikely. +- The third server should be the next highest server by population. +- Servers which are blocked due to server ACLs should never be chosen. +- Servers which are IP addresses should never be chosen. Servers which + use a domain name are less likely to be unroutable in the future + whereas IP addresses cannot be pointed to a different location and + therefore higher risk options. +- All 3 servers should be unique from each other. If the room does not + have enough users to supply 3 servers, the application should only + specify the servers it can. For example, a room with only 2 users in + it would result in maximum 2 `via` parameters. + +## 3PID Types + +Third Party Identifiers (3PIDs) represent identifiers on other +namespaces that might be associated with a particular person. They +comprise a tuple of `medium` which is a string that identifies the +namespace in which the identifier exists, and an `address`: a string +representing the identifier in that namespace. This must be a canonical +form of the identifier, *i.e.* if multiple strings could represent the +same identifier, only one of these strings must be used in a 3PID +address, in a well-defined manner. + +For example, for e-mail, the `medium` is 'email' and the `address` would +be the email address, *e.g.* the string `bob@example.com`. Since domain +resolution is case-insensitive, the email address `bob@Example.com` is +also has the 3PID address of `bob@example.com` (without the capital 'e') +rather than `bob@Example.com`. + +The namespaces defined by this specification are listed below. More +namespaces may be defined in future versions of this specification. + +### E-Mail + +Medium: `email` + +Represents E-Mail addresses. The `address` is the raw email address in +`user@domain` form with the domain in lowercase. It must not contain +other text such as real name, angle brackets or a mailto: prefix. + +### PSTN Phone numbers + +Medium: `msisdn` + +Represents telephone numbers on the public switched telephone network. +The `address` is the telephone number represented as a MSISDN (Mobile +Station International Subscriber Directory Number) as defined by the +E.164 numbering plan. Note that MSISDNs do not include a leading '+'. + +## Security Threat Model + +### Denial of Service + +The attacker could attempt to prevent delivery of messages to or from +the victim in order to: + +- Disrupt service or marketing campaign of a commercial competitor. +- Censor a discussion or censor a participant in a discussion. +- Perform general vandalism. + +#### Threat: Resource Exhaustion + +An attacker could cause the victim's server to exhaust a particular +resource (e.g. open TCP connections, CPU, memory, disk storage) + +#### Threat: Unrecoverable Consistency Violations + +An attacker could send messages which created an unrecoverable +"split-brain" state in the cluster such that the victim's servers could +no longer derive a consistent view of the chatroom state. + +#### Threat: Bad History + +An attacker could convince the victim to accept invalid messages which +the victim would then include in their view of the chatroom history. +Other servers in the chatroom would reject the invalid messages and +potentially reject the victims messages as well since they depended on +the invalid messages. + +#### Threat: Block Network Traffic + +An attacker could try to firewall traffic between the victim's server +and some or all of the other servers in the chatroom. + +#### Threat: High Volume of Messages + +An attacker could send large volumes of messages to a chatroom with the +victim making the chatroom unusable. + +#### Threat: Banning users without necessary authorisation + +An attacker could attempt to ban a user from a chatroom without the +necessary authorisation. + +### Spoofing + +An attacker could try to send a message claiming to be from the victim +without the victim having sent the message in order to: + +- Impersonate the victim while performing illicit activity. +- Obtain privileges of the victim. + +#### Threat: Altering Message Contents + +An attacker could try to alter the contents of an existing message from +the victim. + +#### Threat: Fake Message "origin" Field + +An attacker could try to send a new message purporting to be from the +victim with a phony "origin" field. + +### Spamming + +The attacker could try to send a high volume of solicited or unsolicited +messages to the victim in order to: + +- Find victims for scams. +- Market unwanted products. + +#### Threat: Unsolicited Messages + +An attacker could try to send messages to victims who do not wish to +receive them. + +#### Threat: Abusive Messages + +An attacker could send abusive or threatening messages to the victim + +### Spying + +The attacker could try to access message contents or metadata for +messages sent by the victim or to the victim that were not intended to +reach the attacker in order to: + +- Gain sensitive personal or commercial information. +- Impersonate the victim using credentials contained in the messages. + (e.g. password reset messages) +- Discover who the victim was talking to and when. + +#### Threat: Disclosure during Transmission + +An attacker could try to expose the message contents or metadata during +transmission between the servers. + +#### Threat: Disclosure to Servers Outside Chatroom + +An attacker could try to convince servers within a chatroom to send +messages to a server it controls that was not authorised to be within +the chatroom. + +#### Threat: Disclosure to Servers Within Chatroom + +An attacker could take control of a server within a chatroom to expose +message contents or metadata for messages in that room. + +## Cryptographic Test Vectors + +To assist in the development of compatible implementations, the +following test values may be useful for verifying the cryptographic +event signing code. + +### Signing Key + +The following test vectors all use the 32-byte value given by the +following Base64-encoded string as the seed for generating the `ed25519` +signing key: + + SIGNING_KEY_SEED = decode_base64( + "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" + ) + +In each case, the server name and key ID are as follows: + + SERVER_NAME = "domain" + + KEY_ID = "ed25519:1" + +### JSON Signing + +Given an empty JSON object: + +```json +{} +``` + +The JSON signing algorithm should emit the following signed data: + +```json +{ + "signatures": { + "domain": { + "ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ" + } + } +} +``` + +Given the following JSON object with data values in it: + +```json +{ + "one": 1, + "two": "Two" +} +``` + +The JSON signing algorithm should emit the following signed JSON: + +```json +{ + "one": 1, + "signatures": { + "domain": { + "ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw" + } + }, + "two": "Two" +} +``` + +### Event Signing + +Given the following minimally-sized event: + +```json +{ + "room_id": "!x:domain", + "sender": "@a:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": {}, + "hashes": {}, + "type": "X", + "content": {}, + "prev_events": [], + "auth_events": [], + "depth": 3, + "unsigned": { + "age_ts": 1000000 + } +} +``` + +The event signing algorithm should emit the following signed event: + +```json +{ + "auth_events": [], + "content": {}, + "depth": 3, + "hashes": { + "sha256": "5jM4wQpv6lnBo7CLIghJuHdW+s2CMBJPUOGOC89ncos" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "prev_events": [], + "room_id": "!x:domain", + "sender": "@a:domain", + "signatures": { + "domain": { + "ed25519:1": "KxwGjPSDEtvnFgU00fwFz+l6d2pJM6XBIaMEn81SXPTRl16AqLAYqfIReFGZlHi5KLjAWbOoMszkwsQma+lYAg" + } + }, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } +} +``` + +Given the following event containing redactable content: + +```json +{ + "content": { + "body": "Here is the message content" + }, + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": {}, + "unsigned": { + "age_ts": 1000000 + } +} +``` + +The event signing algorithm should emit the following signed event: + +```json +{ + "content": { + "body": "Here is the message content" + }, + "event_id": "$0:domain", + "hashes": { + "sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": { + "domain": { + "ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" + } + }, + "unsigned": { + "age_ts": 1000000 + } +} +``` diff --git a/content/application-service-api.md b/content/application-service-api.md new file mode 100644 index 00000000..13d43883 --- /dev/null +++ b/content/application-service-api.md @@ -0,0 +1,376 @@ +--- +title: "Application Service API" +weight: 30 +type: docs +--- + +The Matrix client-server API and server-server APIs provide the means to +implement a consistent self-contained federated messaging fabric. +However, they provide limited means of implementing custom server-side +behaviour in Matrix (e.g. gateways, filters, extensible hooks etc). The +Application Service API (AS API) defines a standard API to allow such +extensible functionality to be implemented irrespective of the +underlying homeserver implementation. + +## Application Services + +Application services are passive and can only observe events from +homeserver. They can inject events into rooms they are participating in. +They cannot prevent events from being sent, nor can they modify the +content of the event being sent. In order to observe events from a +homeserver, the homeserver needs to be configured to pass certain types +of traffic to the application service. This is achieved by manually +configuring the homeserver with information about the application +service. + +### Registration + +{{% boxes/note %}} +Previously, application services could register with a homeserver via +HTTP APIs. This was removed as it was seen as a security risk. A +compromised application service could re-register for a global `*` regex +and sniff *all* traffic on the homeserver. To protect against this, +application services now have to register via configuration files which +are linked to the homeserver configuration file. The addition of +configuration files allows homeserver admins to sanity check the +registration for suspicious regex strings. +{{% /boxes/note %}} + +Application services register "namespaces" of user IDs, room aliases and +room IDs. These namespaces are represented as regular expressions. An +application service is said to be "interested" in a given event if one +of the IDs in the event match the regular expression provided by the +application service, such as the room having an alias or ID in the +relevant namespaces. Similarly, the application service is said to be +interested in a given event if one of the application service's +namespaced users is the target of the event, or is a joined member of +the room where the event occurred. + +An application service can also state whether they should be the only +ones who can manage a specified namespace. This is referred to as an +"exclusive" namespace. An exclusive namespace prevents humans and other +application services from creating/deleting entities in that namespace. +Typically, exclusive namespaces are used when the rooms represent real +rooms on another service (e.g. IRC). Non-exclusive namespaces are used +when the application service is merely augmenting the room itself (e.g. +providing logging or searching facilities). Namespaces are represented +by POSIX extended regular expressions and look like: + + users: + - exclusive: true + regex: "@_irc_bridge_.*" + +Application services may define the following namespaces (with none +being explicitly required): + +| Name | Description | +|----------|------------------------------------------------------------| +| users | Events which are sent from certain users. | +| aliases | Events which are sent in rooms with certain room aliases. | +| rooms | Events which are sent in rooms with certain room IDs. | + +Each individual namespace MUST declare the following fields: + +| Name | Description | +|------------|------------------------------------------------------------------------------------------------------------------------------------| +| exclusive | **Required** A true or false value stating whether this application service has exclusive access to events within this namespace. | +| regex | **Required** A regular expression defining which values this namespace includes. | + +Exclusive user and alias namespaces should begin with an underscore +after the sigil to avoid collisions with other users on the homeserver. +Application services should additionally attempt to identify the service +they represent in the reserved namespace. For example, `@_irc_.*` would +be a good namespace to register for an application service which deals +with IRC. + +The registration is represented by a series of key-value pairs, which +this specification will present as YAML. See below for the possible +options along with their explanation: + + +| Name | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| id | **Required** A unique, user-defined ID of the application service which will never change. | +| url | **Required** The URL for the application service. May include a path after the domain name. Optionally set to null if no traffic is required. | +| as_token | **Required** A unique token for application services to use to authenticate requests to Homeservers. | +| hs_token | **Required** A unique token for Homeservers to use to authenticate requests to application services. | +| sender_localpart | **Required** The localpart of the user associated with the application service. | +| namespaces | **Required** A list of `users`, `aliases` and `rooms` namespaces that the application service controls. | +| rate_limited | Whether requests from masqueraded users are rate-limited. The sender is excluded. | +| protocols | The external protocols which the application service provides (e.g. IRC). | + +An example registration file for an IRC-bridging application service is +below: + + id: "IRC Bridge" + url: "http://127.0.0.1:1234" + as_token: "30c05ae90a248a4188e620216fa72e349803310ec83e2a77b34fe90be6081f46" + hs_token: "312df522183efd404ec1cd22d2ffa4bbc76a8c1ccf541dd692eef281356bb74e" + sender_localpart: "_irc_bot" # Will result in @_irc_bot:example.org + namespaces: + users: + - exclusive: true + regex: "@_irc_bridge_.*" + aliases: + - exclusive: false + regex: "#_irc_bridge_.*" + rooms: [] + +{{% boxes/warning %}} +If the homeserver in question has multiple application services, each +`as_token` and `id` MUST be unique per application service as these are +used to identify the application service. The homeserver MUST enforce +this. +{{% /boxes/warning %}} + +### Homeserver -> Application Service API + +#### Authorization + +Homeservers MUST include a query parameter named `access_token` +containing the `hs_token` from the application service's registration +when making requests to the application service. Application services +MUST verify the provided `access_token` matches their known `hs_token`, +failing the request with an `M_FORBIDDEN` error if it does not match. + +#### Legacy routes + +Previous drafts of the application service specification had a mix of +endpoints that have been used in the wild for a significant amount of +time. The application service specification now defines a version on all +endpoints to be more compatible with the rest of the Matrix +specification and the future. + +Homeservers should attempt to use the specified endpoints first when +communicating with application services. However, if the application +service receives an HTTP status code that does not indicate success +(i.e.: 404, 500, 501, etc) then the homeserver should fall back to the +older endpoints for the application service. + +The older endpoints have the exact same request body and response +format, they just belong at a different path. The equivalent path for +each is as follows: + +- `/_matrix/app/v1/transactions/{txnId}` should fall back to + `/transactions/{txnId}` +- `/_matrix/app/v1/users/{userId}` should fall back to + `/users/{userId}` +- `/_matrix/app/v1/rooms/{roomAlias}` should fall back to + `/rooms/{roomAlias}` +- `/_matrix/app/v1/thirdparty/protocol/{protocol}` should fall back to + `/_matrix/app/unstable/thirdparty/protocol/{protocol}` +- `/_matrix/app/v1/thirdparty/user/{user}` should fall back to + `/_matrix/app/unstable/thirdparty/user/{user}` +- `/_matrix/app/v1/thirdparty/location/{location}` should fall back to + `/_matrix/app/unstable/thirdparty/location/{location}` +- `/_matrix/app/v1/thirdparty/user` should fall back to + `/_matrix/app/unstable/thirdparty/user` +- `/_matrix/app/v1/thirdparty/location` should fall back to + `/_matrix/app/unstable/thirdparty/location` + +Homeservers should periodically try again for the newer endpoints +because the application service may have been updated. + +#### Pushing events + +The application service API provides a transaction API for sending a +list of events. Each list of events includes a transaction ID, which +works as follows: + +``` + Typical + HS ---> AS : Homeserver sends events with transaction ID T. + <--- : Application Service sends back 200 OK. +``` + +``` + AS ACK Lost + HS ---> AS : Homeserver sends events with transaction ID T. + <-/- : AS 200 OK is lost. + HS ---> AS : Homeserver retries with the same transaction ID of T. + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). +``` + +The events sent to the application service should be linearised, as if +they were from the event stream. The homeserver MUST maintain a queue of +transactions to send to the application service. If the application +service cannot be reached, the homeserver SHOULD backoff exponentially +until the application service is reachable again. As application +services cannot *modify* the events in any way, these requests can be +made without blocking other aspects of the homeserver. Homeservers MUST +NOT alter (e.g. add more) events they were going to send within that +transaction ID on retries, as the application service may have already +processed the events. + +{{transactions\_as\_http\_api}} + +#### Querying + +The application service API includes two querying APIs: for room aliases +and for user IDs. The application service SHOULD create the queried +entity if it desires. During this process, the application service is +blocking the homeserver until the entity is created and configured. If +the homeserver does not receive a response to this request, the +homeserver should retry several times before timing out. This should +result in an HTTP status 408 "Request Timeout" on the client which +initiated this request (e.g. to join a room alias). + +{{% boxes/rationale %}} +Blocking the homeserver and expecting the application service to create +the entity using the client-server API is simpler and more flexible than +alternative methods such as returning an initial sync style JSON blob +and get the HS to provision the room/user. This also meant that there +didn't need to be a "backchannel" to inform the application service +about information about the entity such as room ID to room alias +mappings. +{{% /boxes/rationale %}} + +{{query\_user\_as\_http\_api}} + +{{query\_room\_as\_http\_api}} + +#### Third party networks + +Application services may declare which protocols they support via their +registration configuration for the homeserver. These networks are +generally for third party services such as IRC that the application +service is managing. Application services may populate a Matrix room +directory for their registered protocols, as defined in the +Client-Server API Extensions. + +Each protocol may have several "locations" (also known as "third party +locations" or "3PLs"). A location within a protocol is a place in the +third party network, such as an IRC channel. Users of the third party +network may also be represented by the application service. + +Locations and users can be searched by fields defined by the application +service, such as by display name or other attribute. When clients +request the homeserver to search in a particular "network" (protocol), +the search fields will be passed along to the application service for +filtering. + +{{protocols\_as\_http\_api}} + +### Client-Server API Extensions + +Application services can use a more powerful version of the +client-server API by identifying itself as an application service to the +homeserver. + +Endpoints defined in this section MUST be supported by homeservers in +the client-server API as accessible only by application services. + +#### Identity assertion + +The client-server API infers the user ID from the `access_token` +provided in every request. To avoid the application service from having +to keep track of each user's access token, the application service +should identify itself to the Client-Server API by providing its +`as_token` for the `access_token` alongside the user the application +service would like to masquerade as. + +Inputs: +- Application service token (`as_token`) +- User ID in the AS namespace to act as. + +Notes: +- This applies to all aspects of the Client-Server API, except for + Account Management. +- The `as_token` is inserted into `access_token` which is usually + where the client token is, such as via the query string or + `Authorization` header. This is done on purpose to allow application + services to reuse client SDKs. +- The `access_token` should be supplied through the `Authorization` + header where possible to prevent the token appearing in HTTP request + logs by accident. + +The application service may specify the virtual user to act as through +use of a `user_id` query string parameter on the request. The user +specified in the query string must be covered by one of the application +service's `user` namespaces. If the parameter is missing, the homeserver +is to assume the application service intends to act as the user implied +by the `sender_localpart` property of the registration. + +An example request would be: + + GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org + Authorization: Bearer YourApplicationServiceTokenHere + +#### Timestamp massaging + +Previous drafts of the Application Service API permitted application +services to alter the timestamp of their sent events by providing a `ts` +query parameter when sending an event. This API has been excluded from +the first release due to design concerns, however some servers may still +support the feature. Please visit [issue +\#1585](https://github.com/matrix-org/matrix-doc/issues/1585) for more +information. + +#### Server admin style permissions + +The homeserver needs to give the application service *full control* over +its namespace, both for users and for room aliases. This means that the +AS should be able to create/edit/delete any room alias in its namespace, +as well as create/delete any user in its namespace. No additional API +changes need to be made in order for control of room aliases to be +granted to the AS. Creation of users needs API changes in order to: + +- Work around captchas. +- Have a 'passwordless' user. + +This involves bypassing the registration flows entirely. This is +achieved by including the `as_token` on a `/register` request, along +with a login type of `m.login.application_service` to set the desired +user ID without a password. + + POST /_matrix/client/%CLIENT_MAJOR_VERSION%/register + Authorization: Bearer YourApplicationServiceTokenHere + + Content: + { + type: "m.login.application_service", + username: "_irc_example" + } + +Application services which attempt to create users or aliases *outside* +of their defined namespaces will receive an error code `M_EXCLUSIVE`. +Similarly, normal users who attempt to create users or aliases *inside* +an application service-defined namespace will receive the same +`M_EXCLUSIVE` error code, but only if the application service has +defined the namespace as `exclusive`. + +#### Using `/sync` and `/events` + +Application services wishing to use `/sync` or `/events` from the +Client-Server API MUST do so with a virtual user (provide a `user_id` +via the query string). It is expected that the application service use +the transactions pushed to it to handle events rather than syncing with +the user implied by `sender_localpart`. + +#### Application service room directories + +Application services can maintain their own room directories for their +defined third party protocols. These room directories may be accessed by +clients through additional parameters on the `/publicRooms` +client-server endpoint. + +{{appservice\_room\_directory\_cs\_http\_api}} + +### Referencing messages from a third party network + +Application services should include an `external_url` in the `content` +of events it emits to indicate where the message came from. This +typically applies to application services that bridge other networks +into Matrix, such as IRC, where an HTTP URL may be available to +reference. + +Clients should provide users with a way to access the `external_url` if +it is present. Clients should additionally ensure the URL has a scheme +of `https` or `http` before making use of it. + +The presence of an `external_url` on an event does not necessarily mean +the event was sent from an application service. Clients should be wary +of the URL contained within, as it may not be a legitimate reference to +the event's source. diff --git a/content/client-server-api/_index.md b/content/client-server-api/_index.md new file mode 100644 index 00000000..b6a0b075 --- /dev/null +++ b/content/client-server-api/_index.md @@ -0,0 +1,2051 @@ +--- +title: "Client-Server API" +weight: 10 +type: docs +--- + +The client-server API provides a simple lightweight API to let clients +send messages, control rooms and synchronise conversation history. It is +designed to support both lightweight clients which store no state and +lazy-load data from the server as required - as well as heavyweight +clients which maintain a full local persistent copy of server state. + +## API Standards + +The mandatory baseline for client-server communication in Matrix is +exchanging JSON objects over HTTP APIs. HTTPS is recommended for +communication, although HTTP may be supported as a fallback to support +basic HTTP clients. More efficient optional transports will in future be +supported as optional extensions - e.g. a packed binary encoding over +stream-cipher encrypted TCP socket for low-bandwidth/low-roundtrip +mobile usage. For the default HTTP transport, all API calls use a +Content-Type of `application/json`. In addition, all strings MUST be +encoded as UTF-8. Clients are authenticated using opaque `access_token` +strings (see [Client Authentication](#client-authentication) for +details), passed as a query string parameter on all requests. + +The names of the API endpoints for the HTTP transport follow a +convention of using underscores to separate words (for example +`/delete_devices`). The key names in JSON objects passed over the API +also follow this convention. + +{{% boxes/note %}} +There are a few historical exceptions to this rule, such as +`/createRoom`. A future version of this specification will address the +inconsistency. +{{% /boxes/note %}} + +Any errors which occur at the Matrix API level MUST return a "standard +error response". This is a JSON object which looks like: + +```json +{ + "errcode": "", + "error": "" +} +``` + +The `error` string will be a human-readable error message, usually a +sentence explaining what went wrong. The `errcode` string will be a +unique string which can be used to handle an error message e.g. +`M_FORBIDDEN`. These error codes should have their namespace first in +ALL CAPS, followed by a single \_ to ease separating the namespace from +the error code. For example, if there was a custom namespace +`com.mydomain.here`, and a `FORBIDDEN` code, the error code should look +like `COM.MYDOMAIN.HERE_FORBIDDEN`. There may be additional keys +depending on the error, but the keys `error` and `errcode` MUST always +be present. + +Errors are generally best expressed by their error code rather than the +HTTP status code returned. When encountering the error code `M_UNKNOWN`, +clients should prefer the HTTP status code as a more reliable reference +for what the issue was. For example, if the client receives an error +code of `M_NOT_FOUND` but the request gave a 400 Bad Request status +code, the client should treat the error as if the resource was not +found. However, if the client were to receive an error code of +`M_UNKNOWN` with a 400 Bad Request, the client should assume that the +request being made was invalid. + +The common error codes are: + +`M_FORBIDDEN` +Forbidden access, e.g. joining a room without permission, failed login. + +`M_UNKNOWN_TOKEN` +The access token specified was not recognised. + +An additional response parameter, `soft_logout`, might be present on the +response for 401 HTTP status codes. See [the soft logout +section](#soft-logout) for more information. + +`M_MISSING_TOKEN` +No access token was specified for the request. + +`M_BAD_JSON` +Request contained valid JSON, but it was malformed in some way, e.g. +missing required keys, invalid values for keys. + +`M_NOT_JSON` +Request did not contain valid JSON. + +`M_NOT_FOUND` +No resource was found for this request. + +`M_LIMIT_EXCEEDED` +Too many requests have been sent in a short period of time. Wait a while +then try again. + +`M_UNKNOWN` +An unknown error has occurred. + +Other error codes the client might encounter are: + +`M_UNRECOGNIZED` +The server did not understand the request. + +`M_UNAUTHORIZED` +The request was not correctly authorized. Usually due to login failures. + +`M_USER_DEACTIVATED` +The user ID associated with the request has been deactivated. Typically +for endpoints that prove authentication, such as `/login`. + +`M_USER_IN_USE` +Encountered when trying to register a user ID which has been taken. + +`M_INVALID_USERNAME` +Encountered when trying to register a user ID which is not valid. + +`M_ROOM_IN_USE` +Sent when the room alias given to the `createRoom` API is already in +use. + +`M_INVALID_ROOM_STATE` +Sent when the initial state given to the `createRoom` API is invalid. + +`M_THREEPID_IN_USE` +Sent when a threepid given to an API cannot be used because the same +threepid is already in use. + +`M_THREEPID_NOT_FOUND` +Sent when a threepid given to an API cannot be used because no record +matching the threepid was found. + +`M_THREEPID_AUTH_FAILED` +Authentication could not be performed on the third party identifier. + +`M_THREEPID_DENIED` +The server does not permit this third party identifier. This may happen +if the server only permits, for example, email addresses from a +particular domain. + +`M_SERVER_NOT_TRUSTED` +The client's request used a third party server, e.g. identity server, +that this server does not trust. + +`M_UNSUPPORTED_ROOM_VERSION` +The client's request to create a room used a room version that the +server does not support. + +`M_INCOMPATIBLE_ROOM_VERSION` +The client attempted to join a room that has a version the server does +not support. Inspect the `room_version` property of the error response +for the room's version. + +`M_BAD_STATE` +The state change requested cannot be performed, such as attempting to +unban a user who is not banned. + +`M_GUEST_ACCESS_FORBIDDEN` +The room or resource does not permit guests to access it. + +`M_CAPTCHA_NEEDED` +A Captcha is required to complete the request. + +`M_CAPTCHA_INVALID` +The Captcha provided did not match what was expected. + +`M_MISSING_PARAM` +A required parameter was missing from the request. + +`M_INVALID_PARAM` +A parameter that was specified has the wrong value. For example, the +server expected an integer and instead received a string. + +`M_TOO_LARGE` +The request or entity was too large. + +`M_EXCLUSIVE` +The resource being requested is reserved by an application service, or +the application service making the request has not created the resource. + +`M_RESOURCE_LIMIT_EXCEEDED` +The request cannot be completed because the homeserver has reached a +resource limit imposed on it. For example, a homeserver held in a shared +hosting environment may reach a resource limit if it starts using too +much memory or disk space. The error MUST have an `admin_contact` field +to provide the user receiving the error a place to reach out to. +Typically, this error will appear on routes which attempt to modify +state (e.g.: sending messages, account data, etc) and not routes which +only read state (e.g.: `/sync`, get account data, etc). + +`M_CANNOT_LEAVE_SERVER_NOTICE_ROOM` +The user is unable to reject an invite to join the server notices room. +See the [Server Notices](#server-notices) module for more information. + +The client-server API typically uses `HTTP PUT` to submit requests with +a client-generated transaction identifier. This means that these +requests are idempotent. The scope of a transaction identifier is a +particular access token. It **only** serves to identify new requests +from retransmits. After the request has finished, the `{txnId}` value +should be changed (how is not specified; a monotonically increasing +integer is recommended). + +Some API endpoints may allow or require the use of `POST` requests +without a transaction ID. Where this is optional, the use of a `PUT` +request is strongly recommended. + +{{versions\_cs\_http\_api}} + +## Web Browser Clients + +It is realistic to expect that some clients will be written to be run +within a web browser or similar environment. In these cases, the +homeserver should respond to pre-flight requests and supply Cross-Origin +Resource Sharing (CORS) headers on all requests. + +Servers MUST expect that clients will approach them with `OPTIONS` +requests, allowing clients to discover the CORS headers. All endpoints +in this specification support the `OPTIONS` method, however the server +MUST NOT perform any logic defined for the endpoints when approached +with an `OPTIONS` request. + +When a client approaches the server with a request, the server should +respond with the CORS headers for that route. The recommended CORS +headers to be returned by servers on all requests are: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + +## Server Discovery + +In order to allow users to connect to a Matrix server without needing to +explicitly specify the homeserver's URL or other parameters, clients +SHOULD use an auto-discovery mechanism to determine the server's URL +based on a user's Matrix ID. Auto-discovery should only be done at login +time. + +In this section, the following terms are used with specific meanings: + +`PROMPT` +Retrieve the specific piece of information from the user in a way which +fits within the existing client user experience, if the client is +inclined to do so. Failure can take place instead if no good user +experience for this is possible at this point. + +`IGNORE` +Stop the current auto-discovery mechanism. If no more auto-discovery +mechanisms are available, then the client may use other methods of +determining the required parameters, such as prompting the user, or +using default values. + +`FAIL_PROMPT` +Inform the user that auto-discovery failed due to invalid/empty data and +`PROMPT` for the parameter. + +`FAIL_ERROR` +Inform the user that auto-discovery did not return any usable URLs. Do +not continue further with the current login process. At this point, +valid data was obtained, but no server is available to serve the client. +No further guess should be attempted and the user should make a +conscientious decision what to do next. + +### Well-known URI + +{{% boxes/note %}} +Servers hosting the `.well-known` JSON file SHOULD offer CORS headers, +as per the [CORS](#web-browser-clients) section in this specification. +{{% /boxes/note %}} + +The `.well-known` method uses a JSON file at a predetermined location to +specify parameter values. The flow for this method is as follows: + +1. Extract the server name from the user's Matrix ID by splitting the + Matrix ID at the first colon. +2. Extract the hostname from the server name. +3. Make a GET request to `https://hostname/.well-known/matrix/client`. + 1. If the returned status code is 404, then `IGNORE`. + 2. If the returned status code is not 200, or the response body is + empty, then `FAIL_PROMPT`. + 3. Parse the response body as a JSON object + 1. If the content cannot be parsed, then `FAIL_PROMPT`. + 4. Extract the `base_url` value from the `m.homeserver` property. + This value is to be used as the base URL of the homeserver. + 1. If this value is not provided, then `FAIL_PROMPT`. + 5. Validate the homeserver base URL: + 1. Parse it as a URL. If it is not a URL, then `FAIL_ERROR`. + 2. Clients SHOULD validate that the URL points to a valid + homeserver before accepting it by connecting to the + `/_matrix/client/versions`\_ endpoint, ensuring that it does + not return an error, and parsing and validating that the + data conforms with the expected response format. If any step + in the validation fails, then `FAIL_ERROR`. Validation is + done as a simple check against configuration errors, in + order to ensure that the discovered address points to a + valid homeserver. + 6. If the `m.identity_server` property is present, extract the + `base_url` value for use as the base URL of the identity server. + Validation for this URL is done as in the step above, but using + `/_matrix/identity/api/v1` as the endpoint to connect to. If the + `m.identity_server` property is present, but does not have a + `base_url` value, then `FAIL_ERROR`. + +{{wellknown\_cs\_http\_api}} + +## Client Authentication + +Most API endpoints require the user to identify themselves by presenting +previously obtained credentials in the form of an `access_token` query +parameter or through an Authorization Header of `Bearer $access_token`. +An access token is typically obtained via the [Login](#login) or +[Registration](#account-registration-and-management) processes. + +{{% boxes/note %}} +This specification does not mandate a particular format for the access +token. Clients should treat it as an opaque byte sequence. Servers are +free to choose an appropriate format. Server implementors may like to +investigate [macaroons](http://research.google.com/pubs/pub41892.html). +{{% /boxes/note %}} + +### Using access tokens + +Access tokens may be provided in two ways, both of which the homeserver +MUST support: + +1. Via a query string parameter, `access_token=TheTokenHere`. +2. Via a request header, `Authorization: Bearer TheTokenHere`. + +Clients are encouraged to use the `Authorization` header where possible +to prevent the access token being leaked in access/HTTP logs. The query +string should only be used in cases where the `Authorization` header is +inaccessible for the client. + +When credentials are required but missing or invalid, the HTTP call will +return with a status of 401 and the error code, `M_MISSING_TOKEN` or +`M_UNKNOWN_TOKEN` respectively. + +### Relationship between access tokens and devices + +Client [devices](../index.html#devices) are closely related to access +tokens. Matrix servers should record which device each access token is +assigned to, so that subsequent requests can be handled correctly. + +By default, the [Login](#login) and [Registration](#account-registration-and-management) +processes auto-generate a new `device_id`. A client is also free to +generate its own `device_id` or, provided the user remains the same, +reuse a device: in either case the client should pass the `device_id` in +the request body. If the client sets the `device_id`, the server will +invalidate any access token previously assigned to that device. There is +therefore at most one active access token assigned to each device at any +one time. + +### Soft logout + +When a request fails due to a 401 status code per above, the server can +include an extra response parameter, `soft_logout`, to indicate if the +client's persisted information can be retained. This defaults to +`false`, indicating that the server has destroyed the session. Any +persisted state held by the client, such as encryption keys and device +information, must not be reused and must be discarded. + +When `soft_logout` is true, the client can acquire a new access token by +specifying the device ID it is already using to the login API. In most +cases a `soft_logout: true` response indicates that the user's session +has expired on the server-side and the user simply needs to provide +their credentials again. + +In either case, the client's previously known access token will no +longer function. + +### User-Interactive Authentication API + +#### Overview + +Some API endpoints require authentication that interacts with the user. +The homeserver may provide many different ways of authenticating, such +as user/password auth, login via a single-sign-on server (SSO), etc. +This specification does not define how homeservers should authorise +their users but instead defines the standard interface which +implementations should follow so that ANY client can log in to ANY +homeserver. + +The process takes the form of one or more 'stages'. At each stage the +client submits a set of data for a given authentication type and awaits +a response from the server, which will either be a final success or a +request to perform an additional stage. This exchange continues until +the final success. + +For each endpoint, a server offers one or more 'flows' that the client +can use to authenticate itself. Each flow comprises a series of stages, +as described above. The client is free to choose which flow it follows, +however the flow's stages must be completed in order. Failing to follow +the flows in order must result in an HTTP 401 response, as defined +below. When all stages in a flow are complete, authentication is +complete and the API call succeeds. + +#### User-interactive API in the REST API + +In the REST API described in this specification, authentication works by +the client and server exchanging JSON dictionaries. The server indicates +what authentication data it requires via the body of an HTTP 401 +response, and the client submits that authentication data via the `auth` +request parameter. + +A client should first make a request with no `auth` parameter. +The homeserver returns an HTTP 401 response, with a JSON body, as follows: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + +```json +{ + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" +} +``` + +In addition to the `flows`, this object contains some extra information: + +* `params`: This section contains any information that the client will +need to know in order to use a given type of authentication. For each +authentication type presented, that type may be present as a key in this +dictionary. For example, the public part of an OAuth client ID could be +given here. + +* `session`: This is a session identifier that the client must pass back +to the homeserver, if one is provided, in subsequent attempts to authenticate +in the same API call. + +The client then chooses a flow and attempts to complete the first stage. +It does this by resubmitting the same request with the addition of an +`auth` key in the object that it submits. This dictionary contains a +`type` key whose value is the name of the authentication type that the +client is attempting to complete. It must also contain a `session` key +with the value of the session key given by the homeserver, if one was +given. It also contains other keys dependent on the auth type being +attempted. For example, if the client is attempting to complete auth +type `example.type.foo`, it might submit something like this: + + POST /_matrix/client/r0/endpoint HTTP/1.1 + Content-Type: application/json + +```json +{ + "a_request_parameter": "something", + "another_request_parameter": "something else", + "auth": { + "type": "example.type.foo", + "session": "xxxxxx", + "example_credential": "verypoorsharedsecret" + } +} +``` + +If the homeserver deems the authentication attempt to be successful but +still requires more stages to be completed, it returns HTTP status 401 +along with the same object as when no authentication was attempted, with +the addition of the `completed` key which is an array of auth types the +client has completed successfully: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + +```json +{ + "completed": [ "example.type.foo" ], + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" +} +``` + +Individual stages may require more than one request to complete, in +which case the response will be as if the request was unauthenticated +with the addition of any other keys as defined by the auth type. + +If the homeserver decides that an attempt on a stage was unsuccessful, +but the client may make a second attempt, it returns the same HTTP +status 401 response as above, with the addition of the standard +`errcode` and `error` fields describing the error. For example: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + +```json +{ + "errcode": "M_FORBIDDEN", + "error": "Invalid password", + "completed": [ "example.type.foo" ], + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" +} +``` + +If the request fails for a reason other than authentication, the server +returns an error message in the standard format. For example: + + HTTP/1.1 400 Bad request + Content-Type: application/json + +```json +{ + "errcode": "M_EXAMPLE_ERROR", + "error": "Something was wrong" +} +``` + +If the client has completed all stages of a flow, the homeserver +performs the API call and returns the result as normal. Completed stages +cannot be retried by clients, therefore servers must return either a 401 +response with the completed stages, or the result of the API call if all +stages were completed when a client retries a stage. + +Some authentication types may be completed by means other than through +the Matrix client, for example, an email confirmation may be completed +when the user clicks on the link in the email. In this case, the client +retries the request with an auth dict containing only the session key. +The response to this will be the same as if the client were attempting +to complete an auth state normally, i.e. the request will either +complete or request auth, with the presence or absence of that auth type +in the 'completed' array indicating whether that stage is complete. + +{{% boxes/note %}} +A request to an endpoint that uses User-Interactive Authentication never +succeeds without auth. Homeservers may allow requests that don't require +auth by offering a stage with only the `m.login.dummy` auth type, but they +must still give a 401 response to requests with no auth data. +{{% /boxes/note %}} + +#### Example + +At a high level, the requests made for an API call completing an auth +flow with three stages will resemble the following diagram: + +``` + _______________________ + | Stage 0 | + | No auth | + | ___________________ | + | |_Request_1_________| | <-- Returns "session" key which is used throughout. + |_______________________| + | + | + _________V_____________ + | Stage 1 | + | type: "" | + | ___________________ | + | |_Request_1_________| | + |_______________________| + | + | + _________V_____________ + | Stage 2 | + | type: "" | + | ___________________ | + | |_Request_1_________| | + | ___________________ | + | |_Request_2_________| | + | ___________________ | + | |_Request_3_________| | + |_______________________| + | + | + _________V_____________ + | Stage 3 | + | type: "" | + | ___________________ | + | |_Request_1_________| | <-- Returns API response + |_______________________| +``` + +#### Authentication types + +This specification defines the following auth types: +- `m.login.password` +- `m.login.recaptcha` +- `m.login.sso` +- `m.login.email.identity` +- `m.login.msisdn` +- `m.login.dummy` + +##### Password-based + + +| Type | Description | +|--------------------|--------------------------------------------------------------------------------| +| `m.login.password` | The client submits an identifier and secret password, both sent in plain-text. | + +To use this authentication type, clients should submit an auth dict as +follows: + +``` +{ + "type": "m.login.password", + "identifier": { + ... + }, + "password": "", + "session": "" +} +``` + +where the `identifier` property is a user identifier object, as +described in [Identifier types](#identifier-types). + +For example, to authenticate using the user's Matrix ID, clients would +submit: + +```json +{ + "type": "m.login.password", + "identifier": { + "type": "m.id.user", + "user": "" + }, + "password": "", + "session": "" +} +``` + +Alternatively reply using a 3PID bound to the user's account on the +homeserver using the `/account/3pid`\_ API rather than giving the `user` +explicitly as follows: + +```json +{ + "type": "m.login.password", + "identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" + }, + "password": "", + "session": "" +} +``` + +In the case that the homeserver does not know about the supplied 3PID, +the homeserver must respond with 403 Forbidden. + +##### Google ReCaptcha + +| Type | Description | +|---------------------|------------------------------------------------------| +| `m.login.recaptcha` | The user completes a Google ReCaptcha 2.0 challenge. | + +To use this authentication type, clients should submit an auth dict as +follows: + +```json +{ + "type": "m.login.recaptcha", + "response": "", + "session": "" +} +``` + +##### Single Sign-On + +| Type | Description | +|---------------|--------------------------------------------------------------------------------------| +| `m.login.sso` | Authentication is supported by authorising with an external single sign-on provider. | + +A client wanting to complete authentication using SSO should use the +[Fallback](#fallback) mechanism. See [SSO during User-Interactive +Authentication](#sso-during-user-interactive-authentication) for more information. + +##### Email-based (identity / homeserver) + +| Type | Description | +|--------------------------|------------------------------------------------------------------------------------------------------------------| +| `m.login.email.identity` | Authentication is supported by authorising an email address with an identity server, or homeserver if supported. | + +Prior to submitting this, the client should authenticate with an +identity server (or homeserver). After authenticating, the session +information should be submitted to the homeserver. + +To use this authentication type, clients should submit an auth dict as +follows: + +```json +{ + "type": "m.login.email.identity", + "threepidCreds": [ + { + "sid": "", + "client_secret": "", + "id_server": "", + "id_access_token": "" + } + ], + "session": "" +} +``` + +Note that `id_server` (and therefore `id_access_token`) is optional if +the `/requestToken` request did not include them. + +##### Phone number/MSISDN-based (identity / homeserver) + +| Type | Description | +|------------------|----------------------------------------------------------------------------------------------------------------| +| `m.login.msisdn` | Authentication is supported by authorising a phone number with an identity server, or homeserver if supported. | + +Prior to submitting this, the client should authenticate with an +identity server (or homeserver). After authenticating, the session +information should be submitted to the homeserver. + +To use this authentication type, clients should submit an auth dict as +follows: + +```json +{ + "type": "m.login.msisdn", + "threepidCreds": [ + { + "sid": "", + "client_secret": "", + "id_server": "", + "id_access_token": "" + } + ], + "session": "" +} +``` + +Note that `id_server` (and therefore `id_access_token`) is optional if +the `/requestToken` request did not include them. + +##### Dummy Auth + +| Type | Description | +|------------------|------------------------------------------------------------------------| +| `m.login.dummy` | Dummy authentication always succeeds and requires no extra parameters. | + +The purpose of dummy authentication is to allow servers to not require any form of +User-Interactive Authentication to perform a request. It can also be +used to differentiate flows where otherwise one flow would be a subset +of another flow. e.g. if a server offers flows `m.login.recaptcha` and +`m.login.recaptcha, m.login.email.identity` and the client completes the +recaptcha stage first, the auth would succeed with the former flow, even +if the client was intending to then complete the email auth stage. A +server can instead send flows `m.login.recaptcha, m.login.dummy` and +`m.login.recaptcha, m.login.email.identity` to fix the ambiguity. + +To use this authentication type, clients should submit an auth dict with +just the type and session, if provided: + +```json +{ + "type": "m.login.dummy", + "session": "" +} +``` + +#### Fallback + +Clients cannot be expected to be able to know how to process every +single login type. If a client does not know how to handle a given login +type, it can direct the user to a web browser with the URL of a fallback +page which will allow the user to complete that login step out-of-band +in their web browser. The URL it should open is: + + /_matrix/client/%CLIENT_MAJOR_VERSION%/auth//fallback/web?session= + +Where `auth type` is the type name of the stage it is attempting and +`session ID` is the ID of the session given by the homeserver. + +This MUST return an HTML page which can perform this authentication +stage. This page must use the following JavaScript when the +authentication has been completed: + +```js +if (window.onAuthDone) { + window.onAuthDone(); +} else if (window.opener && window.opener.postMessage) { + window.opener.postMessage("authDone", "*"); +} +``` + +This allows the client to either arrange for the global function +`onAuthDone` to be defined in an embedded browser, or to use the HTML5 +[cross-document +messaging](https://www.w3.org/TR/webmessaging/#web-messaging) API, to +receive a notification that the authentication stage has been completed. + +Once a client receives the notification that the authentication stage +has been completed, it should resubmit the request with an auth dict +with just the session ID: + +```json +{ + "session": "" +} +``` + +##### Example + +A client webapp might use the following JavaScript to open a popup +window which will handle unknown login types: + +```js +/** + * Arguments: + * homeserverUrl: the base url of the homeserver (e.g. "https://matrix.org") + * + * apiEndpoint: the API endpoint being used (e.g. + * "/_matrix/client/%CLIENT_MAJOR_VERSION%/account/password") + * + * loginType: the loginType being attempted (e.g. "m.login.recaptcha") + * + * sessionID: the session ID given by the homeserver in earlier requests + * + * onComplete: a callback which will be called with the results of the request + */ +function unknownLoginType(homeserverUrl, apiEndpoint, loginType, sessionID, onComplete) { + var popupWindow; + + var eventListener = function(ev) { + // check it's the right message from the right place. + if (ev.data !== "authDone" || ev.origin !== homeserverUrl) { + return; + } + + // close the popup + popupWindow.close(); + window.removeEventListener("message", eventListener); + + // repeat the request + var requestBody = { + auth: { + session: sessionID, + }, + }; + + request({ + method:'POST', url:apiEndpoint, json:requestBody, + }, onComplete); + }; + + window.addEventListener("message", eventListener); + + var url = homeserverUrl + + "/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/" + + encodeURIComponent(loginType) + + "/fallback/web?session=" + + encodeURIComponent(sessionID); + + popupWindow = window.open(url); +} +``` + +#### Identifier types + +Some authentication mechanisms use a user identifier object to identify +a user. The user identifier object has a `type` field to indicate the +type of identifier being used, and depending on the type, has other +fields giving the information required to identify the user as described +below. + +This specification defines the following identifier types: +- `m.id.user` +- `m.id.thirdparty` +- `m.id.phone` + +##### Matrix User ID + +| Type | Description | +|-------------|--------------------------------------------| +| `m.id.user` | The user is identified by their Matrix ID. | + +A client can identify a user using their Matrix ID. This can either be +the fully qualified Matrix user ID, or just the localpart of the user +ID. + +```json +"identifier": { + "type": "m.id.user", + "user": "" +} +``` + +##### Third-party ID + +| Type | Description | +|-------------------|---------------------------------------------------------------------------| +| `m.id.thirdparty` | The user is identified by a third-party identifier in canonicalised form. | + +A client can identify a user using a 3PID associated with the user's +account on the homeserver, where the 3PID was previously associated +using the `/account/3pid`\_ API. See the [3PID +Types](/appendices#3pid-types) Appendix for a list of Third-party +ID media. + +```json +"identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" +} +``` + +##### Phone number + +| Type | Description | +|--------------|-------------------------------------------| +| `m.id.phone` | The user is identified by a phone number. | + +A client can identify a user using a phone number associated with the +user's account, where the phone number was previously associated using +the `/account/3pid`\_ API. The phone number can be passed in as entered +by the user; the homeserver will be responsible for canonicalising it. +If the client wishes to canonicalise the phone number, then it can use +the `m.id.thirdparty` identifier type with a `medium` of `msisdn` +instead. + +```json +"identifier": { + "type": "m.id.phone", + "country": "", + "phone": "" +} +``` + +The `country` is the two-letter uppercase ISO-3166-1 alpha-2 country +code that the number in `phone` should be parsed as if it were dialled +from. + +### Login + +A client can obtain access tokens using the `/login` API. + +Note that this endpoint does not +currently use the [User-Interactive Authentication +API](#user-interactive-authentication-api). + +For a simple username/password login, clients should submit a `/login` +request as follows: + +```json +{ + "type": "m.login.password", + "identifier": { + "type": "m.id.user", + "user": "" + }, + "password": "" +} +``` + +Alternatively, a client can use a 3PID bound to the user's account on +the homeserver using the `/account/3pid`\_ API rather than giving the +`user` explicitly, as follows: + +```json +{ + "type": "m.login.password", + "identifier": { + "medium": "", + "address": "" + }, + "password": "" +} +``` + +In the case that the homeserver does not know about the supplied 3PID, +the homeserver must respond with `403 Forbidden`. + +To log in using a login token, clients should submit a `/login` request +as follows: + +```json +{ + "type": "m.login.token", + "token": "" +} +``` + +As with [token-based]() interactive login, the `token` must encode the +user ID. In the case that the token is not valid, the homeserver must +respond with `403 Forbidden` and an error code of `M_FORBIDDEN`. + +If the homeserver advertises `m.login.sso` as a viable flow, and the +client supports it, the client should redirect the user to the +`/redirect` endpoint for [client login via SSO](#client-login-via-sso). After authentication +is complete, the client will need to submit a `/login` request matching +`m.login.token`. + +{{login\_cs\_http\_api}} + +{{logout\_cs\_http\_api}} + +#### Login Fallback + +If a client does not recognize any or all login flows it can use the +fallback login API: + + GET /_matrix/static/client/login/ + +This returns an HTML and JavaScript page which can perform the entire +login process. The page will attempt to call the JavaScript function +`window.onLogin` when login has been successfully completed. + +Non-credential parameters valid for the `/login` endpoint can be +provided as query string parameters here. These are to be forwarded to +the login endpoint during the login process. For example: + + GET /_matrix/static/client/login/?device_id=GHTYAJCE + +### Account registration and management + +{{registration\_cs\_http\_api}} + +#### Notes on password management + +{{% boxes/warning %}} +Clients SHOULD enforce that the password provided is suitably complex. +The password SHOULD include a lower-case letter, an upper-case letter, a +number and a symbol and be at a minimum 8 characters in length. Servers +MAY reject weak passwords with an error code `M_WEAK_PASSWORD`. +{{% /boxes/warning %}} + +### Adding Account Administrative Contact Information + +A homeserver may keep some contact information for administrative use. +This is independent of any information kept by any identity servers, +though can be proxied (bound) to the identity server in many cases. + +{{% boxes/note %}} +This section deals with two terms: "add" and "bind". Where "add" (or +"remove") is used, it is speaking about an identifier that was not bound +to an identity server. As a result, "bind" (or "unbind") references an +identifier that is found in an identity server. Note that an identifier +can be added and bound at the same time, depending on context. +{{% /boxes/note %}} + +{{administrative\_contact\_cs\_http\_api}} + +### Current account information + +{{whoami\_cs\_http\_api}} + +#### Notes on identity servers + +Identity servers in Matrix store bindings (relationships) between a +user's third party identifier, typically email or phone number, and +their user ID. Once a user has chosen an identity server, that identity +server should be used by all clients. + +Clients can see which identity server the user has chosen through the +`m.identity_server` account data event, as described below. Clients +SHOULD refrain from making requests to any identity server until the +presence of `m.identity_server` is confirmed as (not) present. If +present, the client SHOULD check for the presence of the `base_url` +property in the event's content. If the `base_url` is present, the +client SHOULD use the identity server in that property as the identity +server for the user. If the `base_url` is missing, or the account data +event is not present, the client SHOULD use whichever default value it +normally would for an identity server, if applicable. Clients SHOULD NOT +update the account data with the default identity server when the user +is missing an identity server in their account data. + +Clients SHOULD listen for changes to the `m.identity_server` account +data event and update the identity server they are contacting as a +result. + +If the client offers a way to set the identity server to use, it MUST +update the value of `m.identity_server` accordingly. A `base_url` of +`null` MUST be treated as though the user does not want to use an +identity server, disabling all related functionality as a result. + +Clients SHOULD refrain from populating the account data as a migration +step for users who are lacking the account data, unless the user sets +the identity server within the client to a value. For example, a user +which has no `m.identity_server` account data event should not end up +with the client's default identity server in their account data, unless +the user first visits their account settings to set the identity server. + +{{m\_identity\_server\_event}} + +## Capabilities negotiation + +A homeserver may not support certain operations and clients must be able +to query for what the homeserver can and can't offer. For example, a +homeserver may not support users changing their password as it is +configured to perform authentication against an external system. + +The capabilities advertised through this system are intended to +advertise functionality which is optional in the API, or which depend in +some way on the state of the user or server. This system should not be +used to advertise unstable or experimental features - this is better +done by the `/versions` endpoint. + +Some examples of what a reasonable capability could be are: + +- Whether the server supports user presence. +- Whether the server supports optional features, such as the user or + room directories. +- The rate limits or file type restrictions imposed on clients by the + server. + +Some examples of what should **not** be a capability are: + +- Whether the server supports a feature in the `unstable` + specification. +- Media size limits - these are handled by the + `/media/%CLIENT_MAJOR_VERSION%/config` API. +- Optional encodings or alternative transports for communicating with + the server. + +Capabilities prefixed with `m.` are reserved for definition in the +Matrix specification while other values may be used by servers using the +Java package naming convention. The capabilities supported by the Matrix +specification are defined later in this section. + +{{capabilities\_cs\_http\_api}} + +### `m.change_password` capability + +This capability has a single flag, `enabled`, which indicates whether or +not the user can use the `/account/password` API to change their +password. If not present, the client should assume that password changes +are possible via the API. When present, clients SHOULD respect the +capability's `enabled` flag and indicate to the user if they are unable +to change their password. + +An example of the capability API's response for this capability is: + +```json +{ + "capabilities": { + "m.change_password": { + "enabled": false + } + } +} +``` + +### `m.room_versions` capability + +This capability describes the default and available room versions a +server supports, and at what level of stability. Clients should make use +of this capability to determine if users need to be encouraged to +upgrade their rooms. + +An example of the capability API's response for this capability is: + +```json +{ + "capabilities": { + "m.room_versions": { + "default": "1", + "available": { + "1": "stable", + "2": "stable", + "3": "unstable", + "custom-version": "unstable" + } + } + } +} +``` + +This capability mirrors the same restrictions of [room +versions](../index.html#room-versions) to describe which versions are +stable and unstable. Clients should assume that the `default` version is +`stable`. Any version not explicitly labelled as `stable` in the +`available` versions is to be treated as `unstable`. For example, a +version listed as `future-stable` should be treated as `unstable`. + +The `default` version is the version the server is using to create new +rooms. Clients should encourage users with sufficient permissions in a +room to upgrade their room to the `default` version when the room is +using an `unstable` version. + +When this capability is not listed, clients should use `"1"` as the +default and only stable `available` room version. + +## Pagination + +{{% boxes/note %}} +The paths referred to in this section are not actual endpoints. They +only serve as examples to explain how pagination functions. +{{% /boxes/note %}} + +Pagination is the process of dividing a dataset into multiple discrete +pages. Matrix makes use of pagination to allow clients to view extremely +large datasets. These datasets are not limited to events in a room (for +example clients may want to paginate a list of rooms in addition to +events within those rooms). Regardless of what is being paginated, there +is a common approach which is used to give clients an easy way of +selecting subsets of a potentially changing dataset. Each endpoint that +uses pagination may use different parameters. However the theme among +them is that they take a `from` and `to` token, and occasionally a +`limit` and `dir`. Together, these parameters describe the position in a +data set, where `from` and `to` are known as "stream tokens" matching +the regular expression `[a-zA-Z0-9.=_-]+`. If supported, the `dir` +defines the direction of events to return: either forwards (`f`) or +backwards (`b`). The response may contain tokens that can be used for +retrieving results before or after the returned set. These tokens may be +called start or prev\_batch for retrieving the previous result +set, or end, next\_batch or next\_token for retrieving the next result set. + +In the following examples, 'START' and 'END' are placeholders to signify +the start and end of the data sets respectively. + +For example, if an endpoint had events E1 -> E15. The client wants +the last 5 events and doesn't know any previous events: + +``` + S E + |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| + | | | + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END + Returns: + E15,E14,E13,E12,E11 +``` + +Another example: a public room list has rooms R1 -> R17. The client +is showing 5 rooms at a time on screen, and is on page 2. They want to +now show page 3 (rooms R11 -> 15): + +``` + S E + | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token + |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room + |____________| |________________| + | | + Currently | + viewing | + | + GET /roomslist?from=9&to=END&limit=5 + Returns: R11,R12,R13,R14,R15 +``` + +Note that tokens are treated in an *exclusive*, not inclusive, manner. +The end token from the initial request was '9' which corresponded to +R10. When the 2nd request was made, R10 did not appear again, even +though from=9 was specified. If you know the token, you already have the +data. + +Responses for pagination-capable endpoints SHOULD have a `chunk` array +alongside the applicable stream tokens to represent the result set. + +In general, when the end of a result set is reached the applicable +stream token will be excluded from the response. For example, if a user +was backwards-paginating events in a room they'd eventually reach the +first event in the room. In this scenario, the `prev_batch` token would +be excluded from the response. Some paginated endpoints are open-ended +in one direction, such as endpoints which expose an event stream for an +active room. In this case, it is not possible for the client to reach +the true "end" of the data set and therefore should always be presented +with a token to keep moving forwards. + +## Filtering + +Filters can be created on the server and can be passed as a parameter to +APIs which return events. These filters alter the data returned from +those APIs. Not all APIs accept filters. + +### Lazy-loading room members + +Membership events often take significant resources for clients to track. +In an effort to reduce the number of resources used, clients can enable +"lazy-loading" for room members. By doing this, servers will attempt to +only send membership events which are relevant to the client. + +It is important to understand that lazy-loading is not intended to be a +perfect optimisation, and that it may not be practical for the server to +calculate precisely which membership events are relevant to the client. +As a result, it is valid for the server to send redundant membership +events to the client to ease implementation, although such redundancy +should be minimised where possible to conserve bandwidth. + +In terms of filters, lazy-loading is enabled by enabling +`lazy_load_members` on a `RoomEventFilter` (or a `StateFilter` in the +case of `/sync` only). When enabled, lazy-loading aware endpoints (see +below) will only include membership events for the `sender` of events +being included in the response. For example, if a client makes a `/sync` +request with lazy-loading enabled, the server will only return +membership events for the `sender` of events in the timeline, not all +members of a room. + +When processing a sequence of events (e.g. by looping on `/sync` or +paginating `/messages`), it is common for blocks of events in the +sequence to share a similar set of senders. Rather than responses in the +sequence sending duplicate membership events for these senders to the +client, the server MAY assume that clients will remember membership +events they have already been sent, and choose to skip sending +membership events for members whose membership has not changed. These +are called 'redundant membership events'. Clients may request that +redundant membership events are always included in responses by setting +`include_redundant_members` to true in the filter. + +The expected pattern for using lazy-loading is currently: + +- Client performs an initial /sync with lazy-loading enabled, and + receives only the membership events which relate to the senders of + the events it receives. +- Clients which support display-name tab-completion or other + operations which require rapid access to all members in a room + should call /members for the currently selected room, with an `?at` + parameter set to the /sync response's from token. The member list + for the room is then maintained by the state in subsequent + incremental /sync responses. +- Clients which do not support tab-completion may instead pull in + profiles for arbitrary users (e.g. read receipts, typing + notifications) on demand by querying the room state or `/profile`. + +The current endpoints which support lazy-loading room members are: + +- `/sync`\_ +- `/rooms//messages`\_ +- `/rooms/{roomId}/context/{eventId}`\_ + +### API endpoints + +{{filter\_cs\_http\_api}} + +## Events + +The model of conversation history exposed by the client-server API can +be considered as a list of events. The server 'linearises' the +eventually-consistent event graph of events into an 'event stream' at +any given point in time: + + [E0]->[E1]->[E2]->[E3]->[E4]->[E5] + +{{% boxes/warning %}} +The format of events can change depending on room version. Check the +[room version specification](../index.html#room-versions) for specific +details on what to expect for event formats. Examples contained within +the client-server specification are expected to be compatible with all +specified room versions, however some differences may still apply. + +For this version of the specification, clients only need to worry about +the event ID format being different depending on room version. Clients +should not be parsing the event ID, and instead be treating it as an +opaque string. No changes should be required to support the currently +available room versions. +{{% /boxes/warning %}} + +### Types of room events + +Room events are split into two categories: + +State Events +These are events which update the metadata state of the room (e.g. room +topic, room membership etc). State is keyed by a tuple of event `type` +and a `state_key`. State in the room with the same key-tuple will be +overwritten. + +Message events +These are events which describe transient "once-off" activity in a room: +typically communication such as sending an instant message or setting up +a VoIP call. + +This specification outlines several events, all with the event type +prefix `m.`. (See [Room Events](#room-events) for the m. event +specification.) However, applications may wish to add their own type of +event, and this can be achieved using the REST API detailed in the +following sections. If new events are added, the event `type` key SHOULD +follow the Java package naming convention, e.g. +`com.example.myapp.event`. This ensures event types are suitably +namespaced for each application and reduces the risk of clashes. + +{{% boxes/note %}} +Events are not limited to the types defined in this specification. New +or custom event types can be created on a whim using the Java package +naming convention. For example, a `com.example.game.score` event can be +sent by clients and other clients would receive it through Matrix, +assuming the client has access to the `com.example` namespace. +{{% /boxes/note %}} + +Note that the structure of these events may be different than those in +the server-server API. + +{{common\_event\_fields}} + +{{common\_room\_event\_fields}} + +#### State Event Fields + +In addition to the fields of a Room Event, State Events have the +following fields. + + +| Key | Type | Description | +|--------------|--------------|--------------------------------------------------------------------------------------------------------------| +| state_key | string | **Required.** A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. State keys starting with an `@` are reserved for referencing user IDs, such as room members. With the exception of a few events, state events set with a given user's ID as the state key MUST only be set by that user. | +| prev_content | EventContent | Optional. The previous `content` for this event. If there is no previous content, this key will be missing. | + +### Size limits + +The complete event MUST NOT be larger than 65535 bytes, when formatted +as a [PDU for the Server-Server +protocol](/server-server-api/#pdus), including any +signatures, and encoded as [Canonical +JSON](/appendices#canonical-json). + +There are additional restrictions on sizes per key: + +- `sender` MUST NOT exceed 255 bytes (including domain). +- `room_id` MUST NOT exceed 255 bytes. +- `state_key` MUST NOT exceed 255 bytes. +- `type` MUST NOT exceed 255 bytes. +- `event_id` MUST NOT exceed 255 bytes. + +Some event types have additional size restrictions which are specified +in the description of the event. Additional keys have no limit other +than that implied by the total 65 KB limit on events. + +### Room Events + +{{% boxes/note %}} +This section is a work in progress. +{{% /boxes/note %}} + +This specification outlines several standard event types, all of which +are prefixed with `m.` + +{{m\_room\_canonical\_alias\_event}} + +{{m\_room\_create\_event}} + +{{m\_room\_join\_rules\_event}} + +{{m\_room\_member\_event}} + +{{m\_room\_power\_levels\_event}} + +#### Historical events + +Some events within the `m.` namespace might appear in rooms, however +they serve no significant meaning in this version of the specification. +They are: + +- `m.room.aliases` + +Previous versions of the specification have more information on these +events. + +### Syncing + +To read events, the intended flow of operation is for clients to first +call the `/sync`\_ API without a `since` parameter. This returns the +most recent message events for each room, as well as the state of the +room at the start of the returned timeline. The response also includes a +`next_batch` field, which should be used as the value of the `since` +parameter in the next call to `/sync`. Finally, the response includes, +for each room, a `prev_batch` field, which can be passed as a `start` +parameter to the `/rooms//messages`\_ API to retrieve earlier +messages. + +You can visualise the range of events being returned as: + +``` + [E0]->[E1]->[E2]->[E3]->[E4]->[E5] + ^ ^ + | | + prev_batch: '1-2-3' next_batch: 'a-b-c' +``` + +Clients then receive new events by "long-polling" the homeserver via the +`/sync` API, passing the value of the `next_batch` field from the +response to the previous call as the `since` parameter. The client +should also pass a `timeout` parameter. The server will then hold open +the HTTP connection for a short period of time waiting for new events, +returning early if an event occurs. Only the `/sync` API (and the +deprecated `/events` API) support long-polling in this way. + +The response for such an incremental sync can be visualised as: + +``` + [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6] + ^ ^ + | | + | next_batch: 'x-y-z' + prev_batch: 'a-b-c' +``` + +Normally, all new events which are visible to the client will appear in +the response to the `/sync` API. However, if a large number of events +arrive between calls to `/sync`, a "limited" timeline is returned, +containing only the most recent message events. A state "delta" is also +returned, summarising any state changes in the omitted part of the +timeline. The client may therefore end up with "gaps" in its knowledge +of the message timeline. The client can fill these gaps using the +`/rooms//messages`\_ API. This situation looks like this: + +``` + | gap | + | <-> | + [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6]->[E7]->[E8]->[E9]->[E10] + ^ ^ + | | + prev_batch: 'd-e-f' next_batch: 'u-v-w' +``` + +{{% boxes/warning %}} +Events are ordered in this API according to the arrival time of the +event on the homeserver. This can conflict with other APIs which order +events based on their partial ordering in the event graph. This can +result in duplicate events being received (once per distinct API +called). Clients SHOULD de-duplicate events based on the event ID when +this happens. +{{% /boxes/warning %}} + +{{% boxes/note %}} +The `/sync` API returns a `state` list which is separate from the +`timeline`. This `state` list allows clients to keep their model of the +room state in sync with that on the server. In the case of an initial +(`since`-less) sync, the `state` list represents the complete state of +the room at the **start** of the returned timeline (so in the case of a +recently-created room whose state fits entirely in the `timeline`, the +`state` list will be empty). + +In the case of an incremental sync, the `state` list gives a delta +between the state of the room at the `since` parameter and that at the +start of the returned `timeline`. (It will therefore be empty unless the +timeline was `limited`.) + +In both cases, it should be noted that the events returned in the +`state` list did **not** necessarily take place just before the returned +`timeline`, so clients should not display them to the user in the +timeline. +{{% /boxes/note %}} + +{{% boxes/rationale %}} +An early design of this specification made the `state` list represent +the room state at the end of the returned timeline, instead of the +start. This was unsatisfactory because it led to duplication of events +between the `state` list and the `timeline`, but more importantly, it +made it difficult for clients to show the timeline correctly. + +In particular, consider a returned timeline \[M0, S1, M2\], where M0 and +M2 are both messages sent by the same user, and S1 is a state event +where that user changes their displayname. If the `state` list +represents the room state at the end of the timeline, the client must +take a copy of the state dictionary, and *rewind* S1, in order to +correctly calculate the display name for M0. +{{% /boxes/rationale %}} + +{{sync\_cs\_http\_api}} + +{{old\_sync\_cs\_http\_api}} + +### Getting events for a room + +There are several APIs provided to `GET` events for a room: + +{{rooms\_cs\_http\_api}} + +{{message\_pagination\_cs\_http\_api}} + +{{room\_initial\_sync\_cs\_http\_api}} + +### Sending events to a room + +{{room\_state\_cs\_http\_api}} + +**Examples** + +Valid requests look like: + +``` +PUT /rooms/!roomid:domain/state/m.example.event +{ "key" : "without a state key" } +``` +``` +PUT /rooms/!roomid:domain/state/m.another.example.event/foo +{ "key" : "with 'foo' as the state key" } +``` + +In contrast, these requests are invalid: + +``` +POST /rooms/!roomid:domain/state/m.example.event/ +{ "key" : "cannot use POST here" } +``` +``` +PUT /rooms/!roomid:domain/state/m.another.example.event/foo/11 +{ "key" : "txnIds are not supported" } +``` + +Care should be taken to avoid setting the wrong `state key`: + +``` +PUT /rooms/!roomid:domain/state/m.another.example.event/11 +{ "key" : "with '11' as the state key, but was probably intended to be a txnId" } +``` + +The `state_key` is often used to store state about individual users, by +using the user ID as the `state_key` value. For example: + +``` +PUT /rooms/!roomid:domain/state/m.favorite.animal.event/%40my_user%3Aexample.org +{ "animal" : "cat", "reason": "fluffy" } +``` + +In some cases, there may be no need for a `state_key`, so it can be +omitted: + +``` +PUT /rooms/!roomid:domain/state/m.room.bgd.color +{ "color": "red", "hex": "#ff0000" } +``` + +{{room\_send\_cs\_http\_api}} + +### Redactions + +Since events are extensible it is possible for malicious users and/or +servers to add keys that are, for example offensive or illegal. Since +some events cannot be simply deleted, e.g. membership events, we instead +'redact' events. This involves removing all keys from an event that are +not required by the protocol. This stripped down event is thereafter +returned anytime a client or remote server requests it. Redacting an +event cannot be undone, allowing server owners to delete the offending +content from the databases. Events that have been redacted include a +`redacted_because` key whose value is the event that caused it to be +redacted, which may include a reason. + +The exact algorithm to apply against an event is defined in the [room +version specification](../index.html#room-versions). + +The server should add the event causing the redaction to the `unsigned` +property of the redacted event, under the `redacted_because` key. When a +client receives a redaction event it should change the redacted event in +the same way a server does. + +{{% boxes/note %}} +Redacted events can still affect the state of the room. When redacted, +state events behave as though their properties were simply not +specified, except those protected by the redaction algorithm. For +example, a redacted `join` event will still result in the user being +considered joined. Similarly, a redacted topic does not necessarily +cause the topic to revert to what it was prior to the event - it causes +the topic to be removed from the room. +{{% /boxes/note %}} + +#### Events + +{{m\_room\_redaction\_event}} + +#### Client behaviour + +{{redaction\_cs\_http\_api}} + +## Rooms + +### Creation + +The homeserver will create an `m.room.create` event when a room is +created, which serves as the root of the event graph for this room. This +event also has a `creator` key which contains the user ID of the room +creator. It will also generate several other events in order to manage +permissions in this room. This includes: + +- `m.room.power_levels` : Sets the power levels of users and required power + levels for various actions within the room such as sending events. + +- `m.room.join_rules` : Whether the room is "invite-only" or not. + +See [Room Events](#room-events) for more information on these events. To +create a room, a client has to use the following API. + +{{create\_room\_cs\_http\_api}} + +### Room aliases + +Servers may host aliases for rooms with human-friendly names. Aliases +take the form `#friendlyname:server.name`. + +As room aliases are scoped to a particular homeserver domain name, it is +likely that a homeserver will reject attempts to maintain aliases on +other domain names. This specification does not provide a way for +homeservers to send update requests to other servers. However, +homeservers MUST handle `GET` requests to resolve aliases on other +servers; they should do this using the federation API if necessary. + +Rooms do not store a list of all aliases present on a room, though +members of the room with relevant permissions may publish preferred +aliases through the `m.room.canonical_alias` state event. The aliases in +the state event should point to the room ID they are published within, +however room aliases can and do drift to other room IDs over time. +Clients SHOULD NOT treat the aliases as accurate. They SHOULD be checked +before they are used or shared with another user. If a room appears to +have a room alias of `#alias:example.com`, this SHOULD be checked to +make sure that the room's ID matches the `room_id` returned from the +request. + +{{directory\_cs\_http\_api}} + +### Permissions + +{{% boxes/note %}} +This section is a work in progress. +{{% /boxes/note %}} + +Permissions for rooms are done via the concept of power levels - to do +any action in a room a user must have a suitable power level. Power +levels are stored as state events in a given room. The power levels +required for operations and the power levels for users are defined in +`m.room.power_levels`, where both a default and specific users' power +levels can be set. By default all users have a power level of 0, other +than the room creator whose power level defaults to 100. Users can grant +other users increased power levels up to their own power level. For +example, user A with a power level of 50 could increase the power level +of user B to a maximum of level 50. Power levels for users are tracked +per-room even if the user is not present in the room. The keys contained +in `m.room.power_levels` determine the levels required for certain +operations such as kicking, banning and sending state events. See +[m.room.power\_levels](#room-events) for more information. + +Clients may wish to assign names to particular power levels. A suggested +mapping is as follows: - 0 User - 50 Moderator - 100 Admin + +### Room membership + +Users need to be a member of a room in order to send and receive events +in that room. There are several states in which a user may be, in +relation to a room: + +- Unrelated (the user cannot send or receive events in the room) +- Invited (the user has been invited to participate in the room, but + is not yet participating) +- Joined (the user can send and receive events in the room) +- Banned (the user is not allowed to join the room) + +There is an exception to the requirement that a user join a room before +sending events to it: users may send an `m.room.member` event to a room +with `content.membership` set to `leave` to reject an invitation if they +have currently been invited to a room but have not joined it. + +Some rooms require that users be invited to it before they can join; +others allow anyone to join. Whether a given room is an "invite-only" +room is determined by the room config key `m.room.join_rules`. It can +have one of the following values: + +`public` +This room is free for anyone to join without an invite. + +`invite` +This room can only be joined if you were invited. + +The allowable state transitions of membership are: + +``` + /ban + +------------------------------------------------------+ + | | + | +----------------+ +----------------+ | + | | /leave | | | | + | | v v | | + /invite +--------+ +-------+ | | + ------------>| invite |<----------| leave |----+ | | + +--------+ /invite +-------+ | | | + | | ^ | | | + | | | | | | + /join | +---------------+ | | | | + | | /join if | | | | + | | join_rules | | /ban | /unban | + | | public /leave | | | | + v v or | | | | + +------+ /kick | | | | + ------------>| join |-------------------+ | | | + /join +------+ v | | + if | +-----+ | | + join_rules +-------------------------->| ban |-----+ | + public /ban +-----+ | + ^ ^ | + | | | + ----------------------------------------------+ +----------------------+ + /ban +``` + +{{list\_joined\_rooms\_cs\_http\_api}} + +#### Joining rooms + +{{inviting\_cs\_http\_api}} + +{{joining\_cs\_http\_api}} + +#### Leaving rooms + +A user can leave a room to stop receiving events for that room. A user +must have been invited to or have joined the room before they are +eligible to leave the room. Leaving a room to which the user has been +invited rejects the invite. Once a user leaves a room, it will no longer +appear in the response to the `/sync`\_ API unless it is explicitly +requested via a filter with the `include_leave` field set to `true`. + +Whether or not they actually joined the room, if the room is an +"invite-only" room the user will need to be re-invited before they can +re-join the room. + +A user can also forget a room which they have left. Rooms which have +been forgotten will never appear the response to the `/sync`\_ API, +until the user re-joins or is re-invited. + +A user may wish to force another user to leave a room. This can be done +by 'kicking' the other user. To do so, the user performing the kick MUST +have the required power level. Once a user has been kicked, the +behaviour is the same as if they had left of their own accord. In +particular, the user is free to re-join if the room is not +"invite-only". + +{{leaving\_cs\_http\_api}} + +{{kicking\_cs\_http\_api}} + +##### Banning users in a room + +A user may decide to ban another user in a room. 'Banning' forces the +target user to leave the room and prevents them from re-joining the +room. A banned user will not be treated as a joined user, and so will +not be able to send or receive events in the room. In order to ban +someone, the user performing the ban MUST have the required power level. +To ban a user, a request should be made to `/rooms//ban`\_ +with: + +```json +{ + "user_id": "", + "reason": "string: " +} +```` + +Banning a user adjusts the banned member's membership state to `ban`. +Like with other membership changes, a user can directly adjust the +target member's state, by making a request to +`/rooms//state/m.room.member/`: + +```json +{ + "membership": "ban" +} +``` + +A user must be explicitly unbanned with a request to +`/rooms//unban`\_ before they can re-join the room or be +re-invited. + +{{banning\_cs\_http\_api}} + +### Listing rooms + +{{list\_public\_rooms\_cs\_http\_api}} + +## User Data + +### User Directory + +{{users\_cs\_http\_api}} + +### Profiles + +{{profile\_cs\_http\_api}} + +#### Events on Change of Profile Information + +Because the profile display name and avatar information are likely to be +used in many places of a client's display, changes to these fields cause +an automatic propagation event to occur, informing likely-interested +parties of the new values. This change is conveyed using two separate +mechanisms: + +- an `m.room.member` event (with a `join` membership) is sent to every + room the user is a member of, to update the `displayname` and + `avatar_url`. +- an `m.presence` presence status update is sent, again containing the + new values of the `displayname` and `avatar_url` keys, in addition + to the required `presence` key containing the current presence state + of the user. + +Both of these should be done automatically by the homeserver when a user +successfully changes their display name or avatar URL fields. + +Additionally, when homeservers emit room membership events for their own +users, they should include the display name and avatar URL fields in +these events so that clients already have these details to hand, and do +not have to perform extra round trips to query it. + +## Security + +### Rate limiting + +Homeservers SHOULD implement rate limiting to reduce the risk of being +overloaded. If a request is refused due to rate limiting, it should +return a standard error response of the form: + +```json +{ + "errcode": "M_LIMIT_EXCEEDED", + "error": "string", + "retry_after_ms": integer (optional) +} +``` + +The `retry_after_ms` key SHOULD be included to tell the client how long +they have to wait in milliseconds before they can try again. + +## Modules + +Modules are parts of the Client-Server API which are not universal to +all endpoints. Modules are strictly defined within this specification +and should not be mistaken for experimental extensions or optional +features. A compliant server implementation MUST support all modules and +supporting specification (unless the implementation only targets clients +of certain profiles, in which case only the required modules for those +feature profiles MUST be implemented). A compliant client implementation +MUST support all the required modules and supporting specification for +the [Feature Profile](#feature-profiles) it targets. + +### Feature Profiles + +Matrix supports many different kinds of clients: from embedded IoT +devices to desktop clients. Not all clients can provide the same feature +sets as other clients e.g. due to lack of physical hardware such as not +having a screen. Clients can fall into one of several profiles and each +profile contains a set of features that the client MUST support. This +section details a set of "feature profiles". Clients are expected to +implement a profile in its entirety in order for it to be classified as +that profile. + +#### Summary + +| Module / Profile | Web | Mobile | Desktop | CLI | Embedded | +|------------------------------------------------------------|-----------|----------|----------|----------|----------| +| [Instant Messaging](#instant-messaging) | Required | Required | Required | Required | Optional | +| [Direct Messaging](#direct-messaging) | Required | Required | Required | Required | Optional | +| [Mentions](#user-room-and-group-mentions) | Required | Required | Required | Optional | Optional | +| [Presence](#presence) | Required | Required | Required | Required | Optional | +| [Push Notifications](#push-notifications) | Optional | Required | Optional | Optional | Optional | +| [Receipts](#receipts) | Required | Required | Required | Required | Optional | +| [Fully read markers](#fully-read-markers) | Optional | Optional | Optional | Optional | Optional | +| [Typing Notifications](#typing-notifications) | Required | Required | Required | Required | Optional | +| [VoIP](#voice-over-ip) | Required | Required | Required | Optional | Optional | +| [Ignoring Users](#ignoring-users) | Required | Required | Required | Optional | Optional | +| [Reporting Content](#reporting-content) | Optional | Optional | Optional | Optional | Optional | +| [Content Repository](#content-repository) | Required | Required | Required | Optional | Optional | +| [Managing History Visibility](#room-history-visibility) | Required | Required | Required | Required | Optional | +| [Server Side Search](#server-side-search) | Optional | Optional | Optional | Optional | Optional | +| [Room Upgrades](#room-upgrades) | Required | Required | Required | Required | Optional | +| [Server Administration](#server-administration) | Optional | Optional | Optional | Optional | Optional | +| [Event Context](#event-context) | Optional | Optional | Optional | Optional | Optional | +| [Third Party Networks](#third-party-networks) | Optional | Optional | Optional | Optional | Optional | +| [Send-to-Device Messaging](#send-to-device-messaging) | Optional | Optional | Optional | Optional | Optional | +| [Device Management](#device-management) | Optional | Optional | Optional | Optional | Optional | +| [End-to-End Encryption](#end-to-end-encryption) | Optional | Optional | Optional | Optional | Optional | +| [Guest Accounts](#guest-access) | Optional | Optional | Optional | Optional | Optional | +| [Room Previews](#room-previews) | Optional | Optional | Optional | Optional | Optional | +| [Client Config](#client-config) | Optional | Optional | Optional | Optional | Optional | +| [SSO Login](#sso-client-loginauthentication) | Optional | Optional | Optional | Optional | Optional | +| [OpenID](#openid) | Optional | Optional | Optional | Optional | Optional | +| [Stickers](#sticker-messages) | Optional | Optional | Optional | Optional | Optional | +| [Server ACLs](#server-access-control-lists-acls-for-rooms) | Optional | Optional | Optional | Optional | Optional | +| [Server Notices](#server-notices) | Optional | Optional | Optional | Optional | Optional | +| [Moderation policies](#moderation-policy-lists) | Optional | Optional | Optional | Optional | Optional | + +*Please see each module for more details on what clients need to +implement.* + +#### Clients + +##### Stand-alone web (`Web`) + +This is a web page which heavily uses Matrix for communication. +Single-page web apps would be classified as a stand-alone web client, as +would multi-page web apps which use Matrix on nearly every page. + +##### Mobile (`Mobile`) + +This is a Matrix client specifically designed for consumption on mobile +devices. This is typically a mobile app but need not be so provided the +feature set can be reached (e.g. if a mobile site could display push +notifications it could be classified as a mobile client). + +##### Desktop (`Desktop`) + +This is a native GUI application which can run in its own environment +outside a browser. + +##### Command Line Interface (`CLI`) + +This is a client which is used via a text-based terminal. + +##### Embedded (`Embedded`) + +This is a client which is embedded into another application or an +embedded device. + +###### Application + +This is a Matrix client which is embedded in another website, e.g. using +iframes. These embedded clients are typically for a single purpose +related to the website in question, and are not intended to be +fully-fledged communication apps. + +###### Device + +This is a client which is typically running on an embedded device such +as a kettle, fridge or car. These clients tend to perform a few +operations and run in a resource constrained environment. Like embedded +applications, they are not intended to be fully-fledged communication +systems. + +{{% cs-modules %}} diff --git a/content/client-server-api/modules/account_data.md b/content/client-server-api/modules/account_data.md new file mode 100644 index 00000000..ea78dc82 --- /dev/null +++ b/content/client-server-api/modules/account_data.md @@ -0,0 +1,32 @@ +--- +type: module +weight: 190 +--- + +### Client Config + +Clients can store custom config data for their account on their +homeserver. This account data will be synced between different devices +and can persist across installations on a particular device. Users may +only view the account data for their own account + +The account\_data may be either global or scoped to a particular rooms. + +#### Events + +The client receives the account data as events in the `account_data` +sections of a `/sync`. + +These events can also be received in a `/events` response or in the +`account_data` section of a room in `/sync`. `m.tag` events appearing in +`/events` will have a `room_id` with the room the tags are for. + +#### Client Behaviour + +{{account\_data\_cs\_http\_api}} + +#### Server Behaviour + +Servers MUST reject clients from setting account data for event types +that the server manages. Currently, this only includes +[m.fully\_read](#mfully_read). diff --git a/content/client-server-api/modules/admin.md b/content/client-server-api/modules/admin.md new file mode 100644 index 00000000..28176bd9 --- /dev/null +++ b/content/client-server-api/modules/admin.md @@ -0,0 +1,13 @@ +--- +type: module +weight: 200 +--- + +### Server Administration + +This module adds capabilities for server administrators to inspect +server state and data. + +#### Client Behaviour + +{{admin\_cs\_http\_api}} diff --git a/content/client-server-api/modules/content_repo.md b/content/client-server-api/modules/content_repo.md new file mode 100644 index 00000000..f615e7f7 --- /dev/null +++ b/content/client-server-api/modules/content_repo.md @@ -0,0 +1,118 @@ +--- +type: module +weight: 70 +--- + +### Content repository + +The content repository (or "media repository") allows users to upload +files to their homeserver for later use. For example, files which the +user wants to send to a room would be uploaded here, as would an avatar +the user wants to use. + +Uploads are POSTed to a resource on the user's local homeserver which +returns a MXC URI which can later be used to GET the download. Content +is downloaded from the recipient's local homeserver, which must first +transfer the content from the origin homeserver using the same API +(unless the origin and destination homeservers are the same). + +When serving content, the server SHOULD provide a +`Content-Security-Policy` header. The recommended policy is +`sandbox; default-src 'none'; script-src 'none'; plugin-types application/pdf; style-src 'unsafe-inline'; object-src 'self';`. + +#### Matrix Content (MXC) URIs + +Content locations are represented as Matrix Content (MXC) URIs. They +look like: + + mxc:/// + + : The name of the homeserver where this content originated, e.g. matrix.org + : An opaque ID which identifies the content. + +#### Client behaviour + +Clients can upload and download content using the following HTTP APIs. + +{{content\_repo\_cs\_http\_api}} + +##### Thumbnails + +The homeserver SHOULD be able to supply thumbnails for uploaded images +and videos. The exact file types which can be thumbnailed are not +currently specified - see [Issue +\#1938](https://github.com/matrix-org/matrix-doc/issues/1938) for more +information. + +The thumbnail methods are "crop" and "scale". "scale" tries to return an +image where either the width or the height is smaller than the requested +size. The client should then scale and letterbox the image if it needs +to fit within a given rectangle. "crop" tries to return an image where +the width and height are close to the requested size and the aspect +matches the requested size. The client should scale the image if it +needs to fit within a given rectangle. + +The dimensions given to the thumbnail API are the minimum size the +client would prefer. Servers must never return thumbnails smaller than +the client's requested dimensions, unless the content being thumbnailed +is smaller than the dimensions. When the content is smaller than the +requested dimensions, servers should return the original content rather +than thumbnail it. + +Servers SHOULD produce thumbnails with the following dimensions and +methods: + +- 32x32, crop +- 96x96, crop +- 320x240, scale +- 640x480, scale +- 800x600, scale + +In summary: +- "scale" maintains the original aspect ratio of the image +- "crop" provides an image in the aspect ratio of the sizes given in + the request +- The server will return an image larger than or equal to the + dimensions requested where possible. + +Servers MUST NOT upscale thumbnails under any circumstance. Servers MUST +NOT return a smaller thumbnail than requested, unless the original +content makes that impossible. + +#### Security considerations + +The HTTP GET endpoint does not require any authentication. Knowing the +URL of the content is sufficient to retrieve the content, even if the +entity isn't in the room. + +MXC URIs are vulnerable to directory traversal attacks such as +`mxc://127.0.0.1/../../../some_service/etc/passwd`. This would cause the +target homeserver to try to access and return this file. As such, +homeservers MUST sanitise MXC URIs by allowing only alphanumeric +(`A-Za-z0-9`), `_` and `-` characters in the `server-name` and +`media-id` values. This set of whitelisted characters allows URL-safe +base64 encodings specified in RFC 4648. Applying this character +whitelist is preferable to blacklisting `.` and `/` as there are +techniques around blacklisted characters (percent-encoded characters, +UTF-8 encoded traversals, etc). + +Homeservers have additional content-specific concerns: + +- Clients may try to upload very large files. Homeservers should not + store files that are too large and should not serve them to clients, + returning a HTTP 413 error with the `M_TOO_LARGE` code. +- Clients may try to upload very large images. Homeservers should not + attempt to generate thumbnails for images that are too large, + returning a HTTP 413 error with the `M_TOO_LARGE` code. +- Remote homeservers may host very large files or images. Homeservers + should not proxy or thumbnail large files or images from remote + homeservers, returning a HTTP 502 error with the `M_TOO_LARGE` code. +- Clients may try to upload a large number of files. Homeservers + should limit the number and total size of media that can be uploaded + by clients, returning a HTTP 403 error with the `M_FORBIDDEN` code. +- Clients may try to access a large number of remote files through a + homeserver. Homeservers should restrict the number and size of + remote files that it caches. +- Clients or remote homeservers may try to upload malicious files + targeting vulnerabilities in either the homeserver thumbnailing or + the client decoders. diff --git a/content/client-server-api/modules/device_management.md b/content/client-server-api/modules/device_management.md new file mode 100644 index 00000000..c6fa6c45 --- /dev/null +++ b/content/client-server-api/modules/device_management.md @@ -0,0 +1,28 @@ +--- +type: module +weight: 90 +--- + +### Device Management + +This module provides a means for a user to manage their [devices](/#devices). + +#### Client behaviour + +Clients that implement this module should offer the user a list of +registered devices, as well as the means to update their display names. +Clients should also allow users to delete disused devices. + +{{device\_management\_cs\_http\_api}} + +#### Security considerations + +Deleting devices has security implications: it invalidates the +access\_token assigned to the device, so an attacker could use it to log +out the real user (and do it repeatedly every time the real user tries +to log in to block the attacker). Servers should require additional +authentication beyond the access token when deleting devices (for +example, requiring that the user resubmit their password). + +The display names of devices are publicly visible. Clients should +consider advising the user of this. diff --git a/content/client-server-api/modules/dm.md b/content/client-server-api/modules/dm.md new file mode 100644 index 00000000..c9f4511a --- /dev/null +++ b/content/client-server-api/modules/dm.md @@ -0,0 +1,47 @@ +--- +type: module +weight: 230 +--- + +### Direct Messaging + +All communication over Matrix happens within a room. It is sometimes +desirable to offer users the concept of speaking directly to one +particular person. This module defines a way of marking certain rooms as +'direct chats' with a given person. This does not restrict the chat to +being between exactly two people since this would preclude the presence +of automated 'bot' users or even a 'personal assistant' who is able to +answer direct messages on behalf of the user in their absence. + +A room may not necessarily be considered 'direct' by all members of the +room, but a signalling mechanism exists to propagate the information of +whether a chat is 'direct' to an invitee. + +#### Events + +{{m\_direct\_event}} + +#### Client behaviour + +To start a direct chat with another user, the inviting user's client +should set the `is_direct` flag to \_. The client should do this +whenever the flow the user has followed is one where their intention is +to speak directly with another person, as opposed to bringing that +person in to a shared room. For example, clicking on 'Start Chat' beside +a person's profile picture would imply the `is_direct` flag should be +set. + +The invitee's client may use the `is_direct` flag in the +[m.room.member](#mroommember) event to automatically mark the room as a direct chat +but this is not required: it may for example, prompt the user, or ignore +the flag altogether. + +Both the inviting client and the invitee's client should record the fact +that the room is a direct chat by storing an `m.direct` event in the +account data using \_. + +#### Server behaviour + +When the `is_direct` flag is given to \_, the home server must set the +`is_direct` flag in the invite member event for any users invited in the +\_ call. diff --git a/content/client-server-api/modules/end_to_end_encryption.md b/content/client-server-api/modules/end_to_end_encryption.md new file mode 100644 index 00000000..d8d1051c --- /dev/null +++ b/content/client-server-api/modules/end_to_end_encryption.md @@ -0,0 +1,1454 @@ +--- +type: module +weight: 100 +--- + +### End-to-End Encryption + +Matrix optionally supports end-to-end encryption, allowing rooms to be +created whose conversation contents are not decryptable or interceptable +on any of the participating homeservers. + +#### Key Distribution + +Encryption and Authentication in Matrix is based around public-key +cryptography. The Matrix protocol provides a basic mechanism for +exchange of public keys, though an out-of-band channel is required to +exchange fingerprints between users to build a web of trust. + +##### Overview + +1) Bob publishes the public keys and supported algorithms for his +device. This may include long-term identity keys, and/or one-time +keys. + +``` + +----------+ +--------------+ + | Bob's HS | | Bob's Device | + +----------+ +--------------+ + | | + |<=============| + /keys/upload +``` + +2) Alice requests Bob's public identity keys and supported algorithms. + +``` + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/query +``` + +3) Alice selects an algorithm and claims any one-time keys needed. + +``` + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/claim +``` + +##### Key algorithms + +The name `ed25519` corresponds to the +[Ed25519](http://ed25519.cr.yp.to/) signature algorithm. The key is a +32-byte Ed25519 public key, encoded using [unpadded Base64](/appendices/#unpadded-base64). Example: + + "SogYyrkTldLz0BXP+GYWs0qaYacUI0RleEqNT8J3riQ" + +The name `curve25519` corresponds to the +[Curve25519](https://cr.yp.to/ecdh.html) ECDH algorithm. The key is a +32-byte Curve25519 public key, encoded using [unpadded Base64](/appendices/#unpadded-base64). +Example: + + "JGLn/yafz74HB2AbPLYJWIVGnKAtqECOBf11yyXac2Y" + +The name `signed_curve25519` also corresponds to the Curve25519 +algorithm, but a key using this algorithm is represented by an object +with the following properties: + +`KeyObject` + +| Parameter | Type | Description | +|------------|------------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| key | string | **Required.** The unpadded Base64-encoded 32-byte Curve25519 public key. | +| signatures | Signatures | **Required.** Signatures of the key object. The signature is calculated using the process described at [Signing JSON](/appendices/#signing-json). | + +Example: + +```json +{ + "key":"06UzBknVHFMwgi7AVloY7ylC+xhOhEX4PkNge14Grl8", + "signatures": { + "@user:example.com": { + "ed25519:EGURVBUNJP": "YbJva03ihSj5mPk+CHMJKUKlCXCPFXjXOK6VqBnN9nA2evksQcTGn6hwQfrgRHIDDXO2le49x7jnWJHMJrJoBQ" + } + } +} +``` + +##### Device keys + +Each device should have one Ed25519 signing key. This key should be +generated on the device from a cryptographically secure source, and the +private part of the key should never be exported from the device. This +key is used as the fingerprint for a device by other clients. + +A device will generally need to generate a number of additional keys. +Details of these will vary depending on the messaging algorithm in use. + +Algorithms generally require device identity keys as well as signing +keys. Some algorithms also require one-time keys to improve their +secrecy and deniability. These keys are used once during session +establishment, and are then thrown away. + +For Olm version 1, each device requires a single Curve25519 identity +key, and a number of signed Curve25519 one-time keys. + +##### Uploading keys + +A device uploads the public parts of identity keys to their homeserver +as a signed JSON object, using the `/keys/upload`\_ API. The JSON object +must include the public part of the device's Ed25519 key, and must be +signed by that key, as described in [Signing +JSON](/appendices/#signing-json). + +One-time keys are also uploaded to the homeserver using the +`/keys/upload`\_ API. + +Devices must store the private part of each key they upload. They can +discard the private part of a one-time key when they receive a message +using that key. However it's possible that a one-time key given out by a +homeserver will never be used, so the device that generates the key will +never know that it can discard the key. Therefore a device could end up +trying to store too many private keys. A device that is trying to store +too many private keys may discard keys starting with the oldest. + +##### Tracking the device list for a user + +Before Alice can send an encrypted message to Bob, she needs a list of +each of his devices and the associated identity keys, so that she can +establish an encryption session with each device. This list can be +obtained by calling `/keys/query`\_, passing Bob's user ID in the +`device_keys` parameter. + +From time to time, Bob may add new devices, and Alice will need to know +this so that she can include his new devices for later encrypted +messages. A naive solution to this would be to call `/keys/query`\_ +before sending each message -however, the number of users and devices +may be large and this would be inefficient. + +It is therefore expected that each client will maintain a list of +devices for a number of users (in practice, typically each user with +whom we share an encrypted room). Furthermore, it is likely that this +list will need to be persisted between invocations of the client +application (to preserve device verification data and to alert Alice if +Bob suddenly gets a new device). + +Alice's client can maintain a list of Bob's devices via the following +process: + +1. It first sets a flag to record that it is now tracking Bob's device + list, and a separate flag to indicate that its list of Bob's devices + is outdated. Both flags should be in storage which persists over + client restarts. +2. It then makes a request to `/keys/query`\_, passing Bob's user ID in + the `device_keys` parameter. When the request completes, it stores + the resulting list of devices in persistent storage, and clears the + 'outdated' flag. +3. During its normal processing of responses to \_, Alice's client + inspects the `changed` property of the `device_lists`\_ field. If it + is tracking the device lists of any of the listed users, then it + marks the device lists for those users outdated, and initiates + another request to `/keys/query`\_ for them. +4. Periodically, Alice's client stores the `next_batch` field of the + result from \_ in persistent storage. If Alice later restarts her + client, it can obtain a list of the users who have updated their + device list while it was offline by calling `/keys/changes`\_, + passing the recorded `next_batch` field as the `from` parameter. If + the client is tracking the device list of any of the users listed in + the response, it marks them as outdated. It combines this list with + those already flagged as outdated, and initiates a `/keys/query`\_ + request for all of them. + +{{% boxes/warning %}} +Bob may update one of his devices while Alice has a request to +`/keys/query` in flight. Alice's client may therefore see Bob's user ID +in the `device_lists` field of the `/sync` response while the first +request is in flight, and initiate a second request to `/keys/query`. +This may lead to either of two related problems. + +The first problem is that, when the first request completes, the client +will clear the 'outdated' flag for Bob's devices. If the second request +fails, or the client is shut down before it completes, this could lead +to Alice using an outdated list of Bob's devices. + +The second possibility is that, under certain conditions, the second +request may complete *before* the first one. When the first request +completes, the client could overwrite the later results from the second +request with those from the first request. + +Clients MUST guard against these situations. For example, a client could +ensure that only one request to `/keys/query` is in flight at a time for +each user, by queuing additional requests until the first completes. +Alternatively, the client could make a new request immediately, but +ensure that the first request's results are ignored (possibly by +cancelling the request). +{{% /boxes/warning %}} + +{{% boxes/note %}} +When Bob and Alice share a room, with Bob tracking Alice's devices, she +may leave the room and then add a new device. Bob will not be notified +of this change, as he doesn't share a room anymore with Alice. When they +start sharing a room again, Bob has an out-of-date list of Alice's +devices. In order to address this issue, Bob's homeserver will add +Alice's user ID to the `changed` property of the `device_lists` field, +thus Bob will update his list of Alice's devices as part of his normal +processing. Note that Bob can also be notified when he stops sharing any +room with Alice by inspecting the `left` property of the `device_lists` +field, and as a result should remove her from its list of tracked users. +{{% /boxes/note %}} + +##### Sending encrypted attachments + +When encryption is enabled in a room, files should be uploaded encrypted +on the homeserver. + +In order to achieve this, a client should generate a single-use 256-bit +AES key, and encrypt the file using AES-CTR. The counter should be +64-bit long, starting at 0 and prefixed by a random 64-bit +Initialization Vector (IV), which together form a 128-bit unique counter +block. + +{{% boxes/warning %}} +An IV must never be used multiple times with the same key. This implies +that if there are multiple files to encrypt in the same message, +typically an image and its thumbnail, the files must not share both the +same key and IV. +{{% /boxes/warning %}} + +Then, the encrypted file can be uploaded to the homeserver. The key and +the IV must be included in the room event along with the resulting +`mxc://` in order to allow recipients to decrypt the file. As the event +containing those will be Megolm encrypted, the server will never have +access to the decrypted file. + +A hash of the ciphertext must also be included, in order to prevent the +homeserver from changing the file content. + +A client should send the data as an encrypted `m.room.message` event, +using either `m.file` as the msgtype, or the appropriate msgtype for the +file type. The key is sent using the [JSON Web +Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) format, with a +[W3C extension](https://w3c.github.io/webcrypto/#iana-section-jwk). + +Extensions to `m.room.message` msgtypes +<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +This module adds `file` and `thumbnail_file` properties, of type +`EncryptedFile`, to `m.room.message` msgtypes that reference files, such +as [m.file](#mfile) and [m.image](#mimage), replacing the `url` and `thumbnail_url` +properties. + +`EncryptedFile` + +| Parameter | Type | Description | +|-----------|------------------|------------------------------------------------------------------------------------------------| +| url | string | **Required.** The URL to the file. | +| key | JWK | **Required.** A [JSON Web Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) object. | +| iv | string | **Required.** The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64. | +| hashes | {string: string} | **Required.** A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64. Clients should support the SHA-256 hash, which uses the key `sha256`. | +| v | string | **Required.** Version of the encrypted attachments protocol. Must be `v2`. | + +`JWK` + +| Parameter | Type | Description | +| --------- |----------|--------------------------------------------------------------------------------------------------------------------------| +| kty | string | **Required.** Key type. Must be `oct`. | +| key_ops | [string] | **Required.** Key operations. Must at least contain `encrypt` and `decrypt`. | +| alg | string | **Required.** Algorithm. Must be `A256CTR`. | +| k | string | **Required.** The key, encoded as urlsafe unpadded base64. | +| ext | boolean | **Required.** Extractable. Must be `true`. This is a [W3C extension](https://w3c.github.io/webcrypto/#iana-section-jwk). | + +Example: + +```json +{ + "content": { + "body": "something-important.jpg", + "file": { + "url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe", + "mimetype": "image/jpeg", + "v": "v2", + "key": { + "alg": "A256CTR", + "ext": true, + "k": "aWF6-32KGYaC3A_FEUCk1Bt0JA37zP0wrStgmdCaW-0", + "key_ops": ["encrypt","decrypt"], + "kty": "oct" + }, + "iv": "w+sE15fzSc0AAAAAAAAAAA", + "hashes": { + "sha256": "fdSLu/YkRx3Wyh3KQabP3rd6+SFiKg5lsJZQHtkSAYA" + } + }, + "info": { + "mimetype": "image/jpeg", + "h": 1536, + "size": 422018, + "thumbnail_file": { + "hashes": { + "sha256": "/NogKqW5bz/m8xHgFiH5haFGjCNVmUIPLzfvOhHdrxY" + }, + "iv": "U+k7PfwLr6UAAAAAAAAAAA", + "key": { + "alg": "A256CTR", + "ext": true, + "k": "RMyd6zhlbifsACM1DXkCbioZ2u0SywGljTH8JmGcylg", + "key_ops": ["encrypt", "decrypt"], + "kty": "oct" + }, + "mimetype": "image/jpeg", + "url": "mxc://example.org/pmVJxyxGlmxHposwVSlOaEOv", + "v": "v2" + }, + "thumbnail_info": { + "h": 768, + "mimetype": "image/jpeg", + "size": 211009, + "w": 432 + }, + "w": 864 + }, + "msgtype": "m.image" + }, + "event_id": "$143273582443PhrSn:example.org", + "origin_server_ts": 1432735824653, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "sender": "@example:example.org", + "type": "m.room.message", + "unsigned": { + "age": 1234 + } +} +``` + +##### Claiming one-time keys + +A client wanting to set up a session with another device can claim a +one-time key for that device. This is done by making a request to the +`/keys/claim`\_ API. + +A homeserver should rate-limit the number of one-time keys that a given +user or remote server can claim. A homeserver should discard the public +part of a one time key once it has given that key to another user. + +#### Device verification + +Before Alice sends Bob encrypted data, or trusts data received from him, +she may want to verify that she is actually communicating with him, +rather than a man-in-the-middle. This verification process requires an +out-of-band channel: there is no way to do it within Matrix without +trusting the administrators of the homeservers. + +In Matrix, verification works by Alice meeting Bob in person, or +contacting him via some other trusted medium, and use [SAS +Verification](#SAS Verification) to interactively verify Bob's devices. +Alice and Bob may also read aloud their unpadded base64 encoded Ed25519 +public key, as returned by `/keys/query`. + +Device verification may reach one of several conclusions. For example: + +- Alice may "accept" the device. This means that she is satisfied that + the device belongs to Bob. She can then encrypt sensitive material + for that device, and knows that messages received were sent from + that device. +- Alice may "reject" the device. She will do this if she knows or + suspects that Bob does not control that device (or equivalently, + does not trust Bob). She will not send sensitive material to that + device, and cannot trust messages apparently received from it. +- Alice may choose to skip the device verification process. She is not + able to verify that the device actually belongs to Bob, but has no + reason to suspect otherwise. The encryption protocol continues to + protect against passive eavesdroppers. + +{{% boxes/note %}} +Once the signing key has been verified, it is then up to the encryption +protocol to verify that a given message was sent from a device holding +that Ed25519 private key, or to encrypt a message so that it may only be +decrypted by such a device. For the Olm protocol, this is documented at +. +{{% /boxes/note %}} + +##### Key verification framework + +Verifying keys manually by reading out the Ed25519 key is not very +user-friendly, and can lead to errors. In order to help mitigate errors, +and to make the process easier for users, some verification methods are +supported by the specification. The methods all use a common framework +for negotiating the key verification. + +To use this framework, Alice's client would send +`m.key.verification.request` events to Bob's devices. All of the +`to_device` messages sent to Bob MUST have the same `transaction_id` to +indicate they are part of the same request. This allows Bob to reject +the request on one device, and have it apply to all of his devices. +Similarly, it allows Bob to process the verification on one device +without having to involve all of his devices. + +When Bob's device receives an `m.key.verification.request`, it should +prompt Bob to verify keys with Alice using one of the supported methods +in the request. If Bob's device does not understand any of the methods, +it should not cancel the request as one of his other devices may support +the request. Instead, Bob's device should tell Bob that an unsupported +method was used for starting key verification. The prompt for Bob to +accept/reject Alice's request (or the unsupported method prompt) should +be automatically dismissed 10 minutes after the `timestamp` field or 2 +minutes after Bob's client receives the message, whichever comes first, +if Bob does not interact with the prompt. The prompt should additionally +be hidden if an appropriate `m.key.verification.cancel` message is +received. + +If Bob rejects the request, Bob's client must send an +`m.key.verification.cancel` message to Alice's device. Upon receipt, +Alice's device should tell her that Bob does not want to verify her +device and send `m.key.verification.cancel` messages to all of Bob's +devices to notify them that the request was rejected. + +If Bob accepts the request, Bob's device starts the key verification +process by sending an `m.key.verification.start` message to Alice's +device. Upon receipt of this message, Alice's device should send an +`m.key.verification.cancel` message to all of Bob's other devices to +indicate the process has been started. The start message must use the +same `transaction_id` from the original key verification request if it +is in response to the request. The start message can be sent +independently of any request. + +Individual verification methods may add additional steps, events, and +properties to the verification messages. Event types for methods defined +in this specification must be under the `m.key.verification` namespace +and any other event types must be namespaced according to the Java +package naming convention. + +Any of Alice's or Bob's devices can cancel the key verification request +or process at any time with an `m.key.verification.cancel` message to +all applicable devices. + +This framework yields the following handshake, assuming both Alice and +Bob each have 2 devices, Bob's first device accepts the key verification +request, and Alice's second device initiates the request. Note how +Alice's first device is not involved in the request or verification +process. + +``` + +---------------+ +---------------+ +-------------+ +-------------+ + | AliceDevice1 | | AliceDevice2 | | BobDevice1 | | BobDevice2 | + +---------------+ +---------------+ +-------------+ +-------------+ + | | | | + | | m.key.verification.request | | + | |---------------------------------->| | + | | | | + | | m.key.verification.request | | + | |-------------------------------------------------->| + | | | | + | | m.key.verification.start | | + | |<----------------------------------| | + | | | | + | | m.key.verification.cancel | | + | |-------------------------------------------------->| + | | | | +``` + +After the handshake, the verification process begins. + +{{m\_key\_verification\_request\_event}} + +{{m\_key\_verification\_start\_event}} + +{{m\_key\_verification\_cancel\_event}} + +##### Short Authentication String (SAS) verification + +SAS verification is a user-friendly key verification process built off +the common framework outlined above. SAS verification is intended to be +a highly interactive process for users, and as such exposes verification +methods which are easier for users to use. + +The verification process is heavily inspired by Phil Zimmermann's ZRTP +key agreement handshake. A key part of key agreement in ZRTP is the hash +commitment: the party that begins the Diffie-Hellman key sharing sends a +hash of their part of the Diffie-Hellman exchange, and does not send +their part of the Diffie-Hellman exchange until they have received the +other party's part. Thus an attacker essentially only has one attempt to +attack the Diffie-Hellman exchange, and hence we can verify fewer bits +while still achieving a high degree of security: if we verify n bits, +then an attacker has a 1 in 2n chance of success. For +example, if we verify 40 bits, then an attacker has a 1 in +1,099,511,627,776 chance (or less than 1 in 1012 chance) of +success. A failed attack would result in a mismatched Short +Authentication String, alerting users to the attack. + +The verification process takes place over [to-device](#send-to-device-messaging) messages in two +phases: + +1. Key agreement phase (based on [ZRTP key + agreement](https://tools.ietf.org/html/rfc6189#section-4.4.1)). +2. Key verification phase (based on HMAC). + +The process between Alice and Bob verifying each other would be: + +1. Alice and Bob establish a secure out-of-band connection, such as + meeting in-person or a video call. "Secure" here means that either + party cannot be impersonated, not explicit secrecy. +2. Alice and Bob communicate which devices they'd like to verify with + each other. +3. Alice selects Bob's device from the device list and begins + verification. +4. Alice's client ensures it has a copy of Bob's device key. +5. Alice's device sends Bob's device an `m.key.verification.start` + message. +6. Bob's device receives the message and selects a key agreement + protocol, hash algorithm, message authentication code, and SAS + method supported by Alice's device. +7. Bob's device ensures it has a copy of Alice's device key. +8. Bob's device creates an ephemeral Curve25519 key pair + (*KBprivate*, *KBpublic*), + and calculates the hash (using the chosen algorithm) of the public + key *KBpublic*. +9. Bob's device replies to Alice's device with an + `m.key.verification.accept` message. +10. Alice's device receives Bob's message and stores the commitment hash + for later use. +11. Alice's device creates an ephemeral Curve25519 key pair + (*KAprivate*, *KApublic*) + and replies to Bob's device with an `m.key.verification.key`, + sending only the public key + *KApublic*. +12. Bob's device receives Alice's message and replies with its own + `m.key.verification.key` message containing its public key + *KBpublic*. +13. Alice's device receives Bob's message and verifies the commitment + hash from earlier matches the hash of the key Bob's device just sent + and the content of Alice's `m.key.verification.start` message. +14. Both Alice and Bob's devices perform an Elliptic-curve + Diffie-Hellman + (*ECDH(KAprivate*, *KBpublic*)), + using the result as the shared secret. +15. Both Alice and Bob's devices display a SAS to their users, which is + derived from the shared key using one of the methods in this + section. If multiple SAS methods are available, clients should allow + the users to select a method. +16. Alice and Bob compare the strings shown by their devices, and tell + their devices if they match or not. +17. Assuming they match, Alice and Bob's devices calculate the HMAC of + their own device keys and a comma-separated sorted list of the key + IDs that they wish the other user to verify, using SHA-256 as the + hash function. HMAC is defined in [RFC + 2104](https://tools.ietf.org/html/rfc2104). The key for the HMAC is + different for each item and is calculated by generating 32 bytes + (256 bits) using [the key verification HKDF](#hkdf-calculation). +18. Alice's device sends Bob's device an `m.key.verification.mac` + message containing the MAC of Alice's device keys and the MAC of her + key IDs to be verified. Bob's device does the same for Bob's device + keys and key IDs concurrently with Alice. +19. When the other device receives the `m.key.verification.mac` message, + the device calculates the HMAC of its copies of the other device's + keys given in the message, as well as the HMAC of the + comma-separated, sorted, list of key IDs in the message. The device + compares these with the HMAC values given in the message, and if + everything matches then the device keys are verified. + +The wire protocol looks like the following between Alice and Bob's +devices: + +``` + +-------------+ +-----------+ + | AliceDevice | | BobDevice | + +-------------+ +-----------+ + | | + | m.key.verification.start | + |-------------------------------->| + | | + | m.key.verification.accept | + |<--------------------------------| + | | + | m.key.verification.key | + |-------------------------------->| + | | + | m.key.verification.key | + |<--------------------------------| + | | + | m.key.verification.mac | + |-------------------------------->| + | | + | m.key.verification.mac | + |<--------------------------------| + | | +``` + +###### Error and exception handling + +At any point the interactive verification can go wrong. The following +describes what to do when an error happens: + +- Alice or Bob can cancel the verification at any time. An + `m.key.verification.cancel` message must be sent to signify the + cancellation. +- The verification can time out. Clients should time out a + verification that does not complete within 10 minutes. Additionally, + clients should expire a `transaction_id` which goes unused for 10 + minutes after having last sent/received it. The client should inform + the user that the verification timed out, and send an appropriate + `m.key.verification.cancel` message to the other device. +- When the same device attempts to initiate multiple verification + attempts, the recipient should cancel all attempts with that device. +- When a device receives an unknown `transaction_id`, it should send + an appropriate `m.key.verification.cancel` message to the other + device indicating as such. This does not apply for inbound + `m.key.verification.start` or `m.key.verification.cancel` messages. +- If the two devices do not share a common key share, hash, HMAC, or + SAS method then the device should notify the other device with an + appropriate `m.key.verification.cancel` message. +- If the user claims the Short Authentication Strings do not match, + the device should send an appropriate `m.key.verification.cancel` + message to the other device. +- If the device receives a message out of sequence or that it was not + expecting, it should notify the other device with an appropriate + `m.key.verification.cancel` message. + +###### Verification messages specific to SAS + +Building off the common framework, the following events are involved in +SAS verification. + +The `m.key.verification.cancel` event is unchanged, however the +following error codes are used in addition to those already specified: + +- `m.unknown_method`: The devices are unable to agree on the key + agreement, hash, MAC, or SAS method. +- `m.mismatched_commitment`: The hash commitment did not match. +- `m.mismatched_sas`: The SAS did not match. + +{{m\_key\_verification\_start\_m\_sas\_v1\_event}} + +{{m\_key\_verification\_accept\_event}} + +{{m\_key\_verification\_key\_event}} + +{{m\_key\_verification\_mac\_event}} + +###### HKDF calculation + +In all of the SAS methods, HKDF is as defined in [RFC +5869](https://tools.ietf.org/html/rfc5869) and uses the previously +agreed-upon hash function for the hash function. The shared secret is +supplied as the input keying material. No salt is used. When the +`key_agreement_protocol` is `curve25519-hkdf-sha256`, the info parameter +is the concatenation of: + +- The string `MATRIX_KEY_VERIFICATION_SAS|`. +- The Matrix ID of the user who sent the `m.key.verification.start` + message, followed by `|`. +- The Device ID of the device which sent the + `m.key.verification.start` message, followed by `|`. +- The public key from the `m.key.verification.key` message sent by + the device which sent the `m.key.verification.start` message, + followed by `|`. +- The Matrix ID of the user who sent the `m.key.verification.accept` + message, followed by `|`. +- The Device ID of the device which sent the + `m.key.verification.accept` message, followed by `|`. +- The public key from the `m.key.verification.key` message sent by + the device which sent the `m.key.verification.accept` message, + followed by `|`. +- The `transaction_id` being used. + +When the `key_agreement_protocol` is the deprecated method `curve25519`, +the info parameter is the concatenation of: + +- The string `MATRIX_KEY_VERIFICATION_SAS`. +- The Matrix ID of the user who sent the `m.key.verification.start` + message. +- The Device ID of the device which sent the + `m.key.verification.start` message. +- The Matrix ID of the user who sent the `m.key.verification.accept` + message. +- The Device ID of the device which sent the + `m.key.verification.accept` message. +- The `transaction_id` being used. + +New implementations are discouraged from implementing the `curve25519` +method. + +{{% boxes/rationale %}} +HKDF is used over the plain shared secret as it results in a harder +attack as well as more uniform data to work with. +{{% /boxes/rationale %}} + +For verification of each party's device keys, HKDF is as defined in RFC +5869 and uses SHA-256 as the hash function. The shared secret is +supplied as the input keying material. No salt is used, and in the info +parameter is the concatenation of: + +- The string `MATRIX_KEY_VERIFICATION_MAC`. +- The Matrix ID of the user whose key is being MAC-ed. +- The Device ID of the device sending the MAC. +- The Matrix ID of the other user. +- The Device ID of the device receiving the MAC. +- The `transaction_id` being used. +- The Key ID of the key being MAC-ed, or the string `KEY_IDS` if the + item being MAC-ed is the list of key IDs. + +###### SAS method: `decimal` + +Generate 5 bytes using [HKDF](#hkdf-calculation) then take sequences of 13 bits +to convert to decimal numbers (resulting in 3 numbers between 0 and 8191 +inclusive each). Add 1000 to each calculated number. + +The bitwise operations to get the numbers given the 5 bytes +*B0*, *B1*, *B2*, *B3*, *B4* +would be: + +- First: (*B0* ≪ 5|*B1* ≫ 3) + 1000 +- Second: + ((*B1*&0x7) ≪ 10|*B2* ≪ 2|*B3* ≫ 6) + 1000 +- Third: ((*B3*&0x3F) ≪ 7|*B4* ≫ 1) + 1000 + +The digits are displayed to the user either with an appropriate +separator, such as dashes, or with the numbers on individual lines. + +###### SAS method: `emoji` + +Generate 6 bytes using [HKDF](#hkdf-calculation) then split the first 42 bits +into 7 groups of 6 bits, similar to how one would base64 encode +something. Convert each group of 6 bits to a number and use the +following table to get the corresponding emoji: + +{{% sas-emojis %}} + +{{% boxes/note %}} +This table is available as JSON at + +{{% /boxes/note %}} + +{{% boxes/rationale %}} +The emoji above were chosen to: + +- Be recognisable without colour. +- Be recognisable at a small size. +- Be recognisable by most cultures. +- Be distinguishable from each other. +- Easily described by a few words. +- Avoid symbols with negative connotations. +- Be likely similar across multiple platforms. +{{% /boxes/rationale %}} + +Clients SHOULD show the emoji with the descriptions from the table, or +appropriate translation of those descriptions. Client authors SHOULD +collaborate to create a common set of translations for all languages. + +{{% boxes/note %}} +Known translations for the emoji are available from + +and can be translated online: + +{{% /boxes/note %}} + +##### Cross-signing + +Rather than requiring Alice to verify each of Bob's devices with each of +her own devices and vice versa, the cross-signing feature allows users +to sign their device keys such that Alice and Bob only need to verify +once. With cross-signing, each user has a set of cross-signing keys that +are used to sign their own device keys and other users' keys, and can be +used to trust device keys that were not verified directly. + +Each user has three ed25519 key pairs for cross-signing: + +- a master key (MSK) that serves as the user's identity in + cross-signing and signs their other cross-signing keys; +- a user-signing key (USK) -- only visible to the user that it belongs + to --that signs other users' master keys; and +- a self-signing key (SSK) that signs the user's own device keys. + +The master key may also be used to sign other items such as the backup +key. The master key may also be signed by the user's own device keys to +aid in migrating from device verifications: if Alice's device had +previously verified Bob's device and Bob's device has signed his master +key, then Alice's device can trust Bob's master key, and she can sign it +with her user-signing key. + +Users upload their cross-signing keys to the server using [POST +/\_matrix/client/r0/keys/device\_signing/upload](). When Alice uploads +new cross-signing keys, her user ID will appear in the `changed` +property of the `device_lists` field of the `/sync` of response of all +users who share an encrypted room with her. When Bob sees Alice's user +ID in his `/sync`, he will call [POST /\_matrix/client/r0/keys/query]() +to retrieve Alice's device and cross-signing keys. + +If Alice has a device and wishes to send an encrypted message to Bob, +she can trust Bob's device if: + +- Alice's device is using a master key that has signed her + user-signing key, +- Alice's user-signing key has signed Bob's master key, +- Bob's master key has signed Bob's self-signing key, and +- Bob's self-signing key has signed Bob's device key. + +The following diagram illustrates how keys are signed: + +``` + +------------------+ .................. +----------------+ + | +--------------+ | .................. : | +------------+ | + | | v v v : : v v v | | + | | +-----------+ : : +-----------+ | | + | | | Alice MSK | : : | Bob MSK | | | + | | +-----------+ : : +-----------+ | | + | | | : : : : | | | + | | +--+ :... : : ...: +--+ | | + | | v v : : v v | | + | | +-----------+ ............. : : ............. +-----------+ | | + | | | Alice SSK | : Alice USK : : : : Bob USK : | Bob SSK | | | + | | +-----------+ :...........: : : :...........: +-----------+ | | + | | | ... | : : : : | ... | | | + | | V V :........: :........: V V | | + | | +---------+ -+ +---------+ -+ | | + | | | Devices | ...| | Devices | ...| | | + | | +---------+ -+ +---------+ -+ | | + | | | ... | | ... | | | + | +------+ | | +----+ | + +----------------+ +--------------+ +``` + +In the diagram, boxes represent keys and lines represent signatures with +the arrows pointing from the signing key to the key being signed. Dotted +boxes and lines represent keys and signatures that are only visible to +the user who created them. + +The following diagram illustrates Alice's view, hiding the keys and +signatures that she cannot see: + +``` + +------------------+ +----------------+ +----------------+ + | +--------------+ | | | | +------------+ | + | | v v | v v v | | + | | +-----------+ | +-----------+ | | + | | | Alice MSK | | | Bob MSK | | | + | | +-----------+ | +-----------+ | | + | | | | | | | | + | | +--+ +--+ | +--+ | | + | | v v | v | | + | | +-----------+ +-----------+ | +-----------+ | | + | | | Alice SSK | | Alice USK | | | Bob SSK | | | + | | +-----------+ +-----------+ | +-----------+ | | + | | | ... | | | | ... | | | + | | V V +--------+ V V | | + | | +---------+ -+ +---------+ -+ | | + | | | Devices | ...| | Devices | ...| | | + | | +---------+ -+ +---------+ -+ | | + | | | ... | | ... | | | + | +------+ | | +----+ | + +----------------+ +--------------+ +``` + +[Verification methods](#device-verification) can be used to verify a +user's master key by using the master public key, encoded using unpadded +base64, as the device ID, and treating it as a normal device. For +example, if Alice and Bob verify each other using SAS, Alice's +`m.key.verification.mac` message to Bob may include +`"ed25519:alices+master+public+key": "alices+master+public+key"` in the +`mac` property. Servers therefore must ensure that device IDs will not +collide with cross-signing public keys. + +###### Key and signature security + +A user's master key could allow an attacker to impersonate that user to +other users, or other users to that user. Thus clients must ensure that +the private part of the master key is treated securely. If clients do +not have a secure means of storing the master key (such as a secret +storage system provided by the operating system), then clients must not +store the private part. + +If a user's client sees that any other user has changed their master +key, that client must notify the user about the change before allowing +communication between the users to continue. + +A user's user-signing and self-signing keys are intended to be easily +replaceable if they are compromised by re-issuing a new key signed by +the user's master key and possibly by re-verifying devices or users. +However, doing so relies on the user being able to notice when their +keys have been compromised, and it involves extra work for the user, and +so although clients do not have to treat the private parts as +sensitively as the master key, clients should still make efforts to +store the private part securely, or not store it at all. Clients will +need to balance the security of the keys with the usability of signing +users and devices when performing key verification. + +To avoid leaking of social graphs, servers will only allow users to see: + +- signatures made by the user's own master, self-signing or + user-signing keys, +- signatures made by the user's own devices about their own master + key, +- signatures made by other users' self-signing keys about their + respective devices, +- signatures made by other users' master keys about their respective + self-signing key, or +- signatures made by other users' devices about their respective + master keys. + +Users will not be able to see signatures made by other users' +user-signing keys. + +{{cross\_signing\_cs\_http\_api}} + +#### Sharing keys between devices + +If Bob has an encrypted conversation with Alice on his computer, and +then logs in through his phone for the first time, he may want to have +access to the previously exchanged messages. To address this issue, +several methods are provided to allow users to transfer keys from one +device to another. + +##### Key requests + +When a device is missing keys to decrypt messages, it can request the +keys by sending [m.room\_key\_request](#mroom_key_request) to-device messages to other +devices with `action` set to `request`. + +If a device wishes to share the keys with that device, it can forward +the keys to the first device by sending an encrypted +[m.forwarded\_room\_key](#mforwarded_room_key) to-device message. The first device should +then send an [m.room\_key\_request](#mroom_key_request) to-device message with `action` +set to `request_cancellation` to the other devices that it had +originally sent the key request to; a device that receives a +`request_cancellation` should disregard any previously-received +`request` message with the same `request_id` and `requesting_device_id`. + +If a device does not wish to share keys with that device, it can +indicate this by sending an [m.room\_key.withheld](#mroom_key.withheld) to-device message, +as described in [Reporting that decryption keys are +withheld](#reporting-that-decryption-keys-are-withheld). + +{{% boxes/note %}} +Key sharing can be a big attack vector, thus it must be done very +carefully. A reasonable strategy is for a user's client to only send +keys requested by the verified devices of the same user. +{{% /boxes/note %}} + +##### Server-side key backups + +Devices may upload encrypted copies of keys to the server. When a device +tries to read a message that it does not have keys for, it may request +the key from the server and decrypt it. Backups are per-user, and users +may replace backups with new backups. + +In contrast with [Key requests](#key-requests), Server-side key backups +do not require another device to be online from which to request keys. +However, as the session keys are stored on the server encrypted, it +requires users to enter a decryption key to decrypt the session keys. + +To create a backup, a client will call [POST +/\_matrix/client/r0/room\_keys/version](#post_matrixclientr0room_keysversion) and define how the keys are to +be encrypted through the backup's `auth_data`; other clients can +discover the backup by calling [GET +/\_matrix/client/r0/room\_keys/version](#get_matrixclientr0room_keysversion). Keys are encrypted according +to the backup's `auth_data` and added to the backup by calling [PUT +/\_matrix/client/r0/room\_keys/keys]() or one of its variants, and can +be retrieved by calling [GET /\_matrix/client/r0/room\_keys/keys](#put_matrixclientr0room_keyskeys) or +one of its variants. Keys can only be written to the most recently +created version of the backup. Backups can also be deleted using [DELETE +/\_matrix/client/r0/room\_keys/version/{version}](#delete_matrixclientr0room_keysversionversion), or individual keys +can be deleted using [DELETE /\_matrix/client/r0/room\_keys/keys](#delete_matrixclientr0room_keyskeys) or +one of its variants. + +Clients must only store keys in backups after they have ensured that the +`auth_data` is trusted, either by checking the signatures on it, or by +deriving the public key from a private key that it obtained from a +trusted source. + +When a client uploads a key for a session that the server already has a +key for, the server will choose to either keep the existing key or +replace it with the new key based on the key metadata as follows: + +- if the keys have different values for `is_verified`, then it will + keep the key that has `is_verified` set to `true`; +- if they have the same values for `is_verified`, then it will keep + the key with a lower `first_message_index`; +- and finally, is `is_verified` and `first_message_index` are equal, + then it will keep the key with a lower `forwarded_count`. + +###### Recovery key + +If the recovery key (the private half of the backup encryption key) is +presented to the user to save, it is presented as a string constructed +as follows: + +1. The 256-bit curve25519 private key is prepended by the bytes `0x8B` + and `0x01` +2. All the bytes in the string above, including the two header bytes, + are XORed together to form a parity byte. This parity byte is + appended to the byte string. +3. The byte string is encoded using base58, using the same [mapping as + is used for Bitcoin + addresses](https://en.bitcoin.it/wiki/Base58Check_encoding#Base58_symbol_chart), + that is, using the alphabet + `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz`. +4. A space should be added after every 4th character. + +When reading in a recovery key, clients must disregard whitespace, and +perform the reverse of steps 1 through 3. + +###### Backup algorithm: `m.megolm_backup.v1.curve25519-aes-sha2` + +When a backup is created with the `algorithm` set to +`m.megolm_backup.v1.curve25519-aes-sha2`, the `auth_data` should have +the following format: + +`AuthData` + +| Parameter | Type | Description | +| -----------| -----------|--------------------------------------------------------------------------------------------------| +| public_key | string | **Required.** The curve25519 public key used to encrypt the backups, encoded in unpadded base64. | +| signatures | Signatures | Optional. Signatures of the ``auth_data``, as Signed JSON | + +The `session_data` field in the backups is constructed as follows: + +1. Encode the session key to be backed up as a JSON object with the + properties: + +| Parameter | Type | Description | +| --------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| algorithm | string | **Required.** The end-to-end message encryption algorithm that the key is for. Must be `m.megolm.v1.aes-sha2`. | +| forwarding_curve25519_key_chain | [string] | **Required.** Chain of Curve25519 keys through which this session was forwarded, via [m.forwarded_room_key](#mforwarded_room_key) events. | +| sender_key | string | **Required.** Unpadded base64-encoded device curve25519 key. | +| sender_claimed_keys | {string: string} | **Required.** A map from algorithm name (`ed25519`) to the identity key for the sending device. | +| session_key | string | **Required.** Unpadded base64-encoded session key in [session-sharing format](https://gitlab.matrix.org/matrix-org/olm/blob/master/docs/megolm.md#session-sharing-format). | + +2. Generate an ephemeral curve25519 key, and perform an ECDH with the + ephemeral key and the backup's public key to generate a shared + secret. The public half of the ephemeral key, encoded using unpadded + base64, becomes the `ephemeral` property of the `session_data`. + +3. Using the shared secret, generate 80 bytes by performing an HKDF + using SHA-256 as the hash, with a salt of 32 bytes of 0, and with + the empty string as the info. The first 32 bytes are used as the AES + key, the next 32 bytes are used as the MAC key, and the last 16 + bytes are used as the AES initialization vector. + +4. Stringify the JSON object, and encrypt it using AES-CBC-256 with + PKCS\#7 padding. This encrypted data, encoded using unpadded base64, + becomes the `ciphertext` property of the `session_data`. + +5. Pass the raw encrypted data (prior to base64 encoding) through + HMAC-SHA-256 using the MAC key generated above. The first 8 bytes of + the resulting MAC are base64-encoded, and become the `mac` property + of the `session_data`. + +{{key\_backup\_cs\_http\_api}} + +##### Key exports + +Keys can be manually exported from one device to an encrypted file, +copied to another device, and imported. The file is encrypted using a +user-supplied passphrase, and is created as follows: + +1. Encode the sessions as a JSON object, formatted as described in [Key + export format](#key-export-format). + +2. Generate a 512-bit key from the user-entered passphrase by computing + [PBKDF2](https://tools.ietf.org/html/rfc2898#section-5.2)(HMAC-SHA-512, + passphrase, S, N, 512), where S is a 128-bit + cryptographically-random salt and N is the number of rounds. N + should be at least 100,000. The keys K and K' are set to the first + and last 256 bits of this generated key, respectively. K is used as + an AES-256 key, and K' is used as an HMAC-SHA-256 key. + +3. Serialize the JSON object as a UTF-8 string, and encrypt it using + AES-CTR-256 with the key K generated above, and with a 128-bit + cryptographically-random initialization vector, IV, that has bit 63 + set to zero. (Setting bit 63 to zero in IV is needed to work around + differences in implementations of AES-CTR.) + +4. Concatenate the following data: + +| Size (bytes)| Description | +| ------------|-----------------------------------------------------------------------------------------| +| 1 | Export format version, which must be `0x01`. | +| 16 | The salt S. | +| 16 | The initialization vector IV. | +| 4 | The number of rounds N, as a big-endian unsigned 32-bit integer. | +| variable | The encrypted JSON object. | +| 32 | The HMAC-SHA-256 of all the above string concatenated together, using K' as the key. | + +5. Base64-encode the string above. Newlines may be added to avoid + overly long lines. + +6. Prepend the resulting string with + `-----BEGIN MEGOLM SESSION DATA-----`, with a trailing newline, and + append `-----END MEGOLM SESSION DATA-----`, with a leading and + trailing newline. + +###### Key export format + +The exported sessions are formatted as a JSON array of `SessionData` +objects described as follows: + +`SessionData` + +| Parameter | Type | Description | +|-----------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| algorithm | string | Required. The encryption algorithm that the session uses. Must be `m.megolm.v1.aes-sha2`. | +| forwarding_curve25519_key_chain | [string] | Required. Chain of Curve25519 keys through which this session was forwarded, via [m.forwarded_room_key](#mforwarded_room_key) events. | +| room_id | string | Required. The room where the session is used. | +| sender_key | string | Required. The Curve25519 key of the device which initiated the session originally. | +| sender_claimed_keys | {string: string} | Required. The Ed25519 key of the device which initiated the session originally. | +| session_id | string | Required. The ID of the session. | +| session_key | string | Required. The key for the session. | + +This is similar to the format before encryption used for the session +keys in [Server-side key backups](#server-side-key-backups) but adds the +`room_id` and `session_id` fields. + +Example: + +``` +[ + { + "algorithm": "m.megolm.v1.aes-sha2", + "forwarding_curve25519_key_chain": [ + "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" + ], + "room_id": "!Cuyf34gef24t:localhost", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU", + "sender_claimed_keys": { + "ed25519": "", + }, + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8Llf..." + }, + ... +] +``` + +#### Messaging Algorithms + +##### Messaging Algorithm Names + +Messaging algorithm names use the extensible naming scheme used +throughout this specification. Algorithm names that start with `m.` are +reserved for algorithms defined by this specification. Implementations +wanting to experiment with new algorithms must be uniquely globally +namespaced following Java's package naming conventions. + +Algorithm names should be short and meaningful, and should list the +primitives used by the algorithm so that it is easier to see if the +algorithm is using a broken primitive. + +A name of `m.olm.v1` is too short: it gives no information about the +primitives in use, and is difficult to extend for different primitives. +However a name of +`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +is too long despite giving a more precise description of the algorithm: +it adds to the data transfer overhead and sacrifices clarity for human +readers without adding any useful extra information. + +##### `m.olm.v1.curve25519-aes-sha2` + +The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the +Olm ratchet, as defined by the [Olm +specification](http://matrix.org/docs/spec/olm.html). This uses: + +- Curve25519 for the initial key agreement. +- HKDF-SHA-256 for ratchet key derivation. +- Curve25519 for the root key ratchet. +- HMAC-SHA-256 for the chain key ratchet. +- HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 + for authenticated encryption. + +Devices that support Olm must include "m.olm.v1.curve25519-aes-sha2" in +their list of supported messaging algorithms, must list a Curve25519 +device key, and must publish Curve25519 one-time keys. + +An event encrypted using Olm has the following format: + +```json +{ + "type": "m.room.encrypted", + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "", + "ciphertext": { + "": { + "type": 0, + "body": "" + } + } + } +} +``` + +`ciphertext` is a mapping from device Curve25519 key to an encrypted +payload for that device. `body` is a Base64-encoded Olm message body. +`type` is an integer indicating the type of the message body: 0 for the +initial pre-key message, 1 for ordinary messages. + +Olm sessions will generate messages with a type of 0 until they receive +a message. Once a session has decrypted a message it will produce +messages with a type of 1. + +When a client receives a message with a type of 0 it must first check if +it already has a matching session. If it does then it will use that +session to try to decrypt the message. If there is no existing session +then the client must create a new session and use the new session to +decrypt the message. A client must not persist a session or remove +one-time keys used by a session until it has successfully decrypted a +message using that session. + +Messages with type 1 can only be decrypted with an existing session. If +there is no matching session, the client must treat this as an invalid +message. + +The plaintext payload is of the form: + +```json +{ + "type": "", + "content": "", + "sender": "", + "recipient": "", + "recipient_keys": { + "ed25519": "" + }, + "keys": { + "ed25519": "" + } +} +``` + +The type and content of the plaintext message event are given in the +payload. + +Other properties are included in order to prevent an attacker from +publishing someone else's curve25519 keys as their own and subsequently +claiming to have sent messages which they didn't. `sender` must +correspond to the user who sent the event, `recipient` to the local +user, and `recipient_keys` to the local ed25519 key. + +Clients must confirm that the `sender_key` and the `ed25519` field value +under the `keys` property match the keys returned by `/keys/query`\_ for +the given user, and must also verify the signature of the payload. +Without this check, a client cannot be sure that the sender device owns +the private part of the ed25519 key it claims to have in the Olm +payload. This is crucial when the ed25519 key corresponds to a verified +device. + +If a client has multiple sessions established with another device, it +should use the session from which it last received and successfully +decrypted a message. For these purposes, a session that has not received +any messages should use its creation time as the time that it last +received a message. A client may expire old sessions by defining a +maximum number of olm sessions that it will maintain for each device, +and expiring sessions on a Least Recently Used basis. The maximum number +of olm sessions maintained per device should be at least 4. + +###### Recovering from undecryptable messages + +Occasionally messages may be undecryptable by clients due to a variety +of reasons. When this happens to an Olm-encrypted message, the client +should assume that the Olm session has become corrupted and create a new +one to replace it. + +{{% boxes/note %}} +Megolm-encrypted messages generally do not have the same problem. +Usually the key for an undecryptable Megolm-encrypted message will come +later, allowing the client to decrypt it successfully. Olm does not have +a way to recover from the failure, making this session replacement +process required. +{{% /boxes/note %}} + +To establish a new session, the client sends an [m.dummy](#mdummy) +to-device event to the other party to notify them of the new session +details. + +Clients should rate-limit the number of sessions it creates per device +that it receives a message from. Clients should not create a new session +with another device if it has already created one for that given device +in the past 1 hour. + +Clients should attempt to mitigate loss of the undecryptable messages. +For example, Megolm sessions that were sent using the old session would +have been lost. The client can attempt to retrieve the lost sessions +through `m.room_key_request` messages. + +##### `m.megolm.v1.aes-sha2` + +The name `m.megolm.v1.aes-sha2` corresponds to version 1 of the Megolm +ratchet, as defined by the [Megolm +specification](http://matrix.org/docs/spec/megolm.html). This uses: + +- HMAC-SHA-256 for the hash ratchet. +- HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 + for authenticated encryption. +- Ed25519 for message authenticity. + +Devices that support Megolm must support Olm, and include +"m.megolm.v1.aes-sha2" in their list of supported messaging algorithms. + +An event encrypted using Megolm has the following format: + +```json +{ + "type": "m.room.encrypted", + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "sender_key": "", + "device_id": "", + "session_id": "", + "ciphertext": "" + } +} +``` + +The encrypted payload can contain any message event. The plaintext is of +the form: + +```json +{ + "type": "", + "content": "", + "room_id": "" +} +``` + +We include the room ID in the payload, because otherwise the homeserver +would be able to change the room a message was sent in. + +Clients must guard against replay attacks by keeping track of the +ratchet indices of Megolm sessions. They should reject messages with a +ratchet index that they have already decrypted. Care should be taken in +order to avoid false positives, as a client may decrypt the same event +twice as part of its normal processing. + +As with Olm events, clients must confirm that the `sender_key` belongs +to the user who sent the message. The same reasoning applies, but the +sender ed25519 key has to be inferred from the `keys.ed25519` property +of the event which established the Megolm session. + +In order to enable end-to-end encryption in a room, clients can send an +`m.room.encryption` state event specifying `m.megolm.v1.aes-sha2` as its +`algorithm` property. + +When creating a Megolm session in a room, clients must share the +corresponding session key using Olm with the intended recipients, so +that they can decrypt future messages encrypted using this session. An +`m.room_key` event is used to do this. Clients must also handle +`m.room_key` events sent by other devices in order to decrypt their +messages. + +#### Protocol definitions + +##### Events + +{{m\_room\_encryption\_event}} + +{{m\_room\_encrypted\_event}} + +{{m\_room\_key\_event}} + +{{m\_room\_key\_request\_event}} + +{{m\_forwarded\_room\_key\_event}} + +{{m\_dummy\_event}} + +##### Key management API + +{{keys\_cs\_http\_api}} + +##### Extensions to /sync + +This module adds an optional `device_lists` property to the \_ response, +as specified below. The server need only populate this property for an +incremental `/sync` (i.e., one where the `since` parameter was +specified). The client is expected to use `/keys/query`\_ or +`/keys/changes`\_ for the equivalent functionality after an initial +sync, as documented in [Tracking the device list for a +user](#tracking-the-device-list-for-a-user). + +It also adds a `one_time_keys_count` property. Note the spelling +difference with the `one_time_key_counts` property in the +`/keys/upload`\_ response. + +| Parameter | Type | Description | +|----------------------------|--------------------|------------------------------------------------------------------------------------------------------------------------| +| device_lists | DeviceLists | Optional. Information on e2e device updates. Note: only present on an incremental sync. | +| device_one_time_keys_count | {string: integer} | Optional. For each key algorithm, the number of unclaimed one-time keys currently held on the server for this device. | + +`DeviceLists` + +| Parameter | Type | Description | +|------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| changed | [string] | List of users who have updated their device identity or cross-signing keys, or who now share an encrypted room with the client since the previous sync response. | +| left | [string] | List of users with whom we do not share any encrypted rooms anymore since the previous sync response. | + +{{% boxes/note %}} +For optimal performance, Alice should be added to `changed` in Bob's +sync only when she updates her devices or cross-signing keys, or when +Alice and Bob now share a room but didn't share any room previously. +However, for the sake of simpler logic, a server may add Alice to +`changed` when Alice and Bob share a new room, even if they previously +already shared a room. +{{% /boxes/note %}} + +Example response: + +```json +{ + "next_batch": "s72595_4483_1934", + "rooms": {"leave": {}, "join": {}, "invite": {}}, + "device_lists": { + "changed": [ + "@alice:example.com", + ], + "left": [ + "@bob:example.com", + ], + }, + "device_one_time_keys_count": { + "curve25519": 10, + "signed_curve25519": 20 + } +} +``` + +#### Reporting that decryption keys are withheld + +When sending an encrypted event to a room, a client can optionally +signal to other devices in that room that it is not sending them the +keys needed to decrypt the event. In this way, the receiving client can +indicate to the user why it cannot decrypt the event, rather than just +showing a generic error message. + +In the same way, when one device requests keys from another using [Key +requests](#key-requests), the device from which the key is being +requested may want to tell the requester that it is purposely not +sharing the key. + +If Alice withholds a megolm session from Bob for some messages in a +room, and then later on decides to allow Bob to decrypt later messages, +she can send Bob the megolm session, ratcheted up to the point at which +she allows Bob to decrypt the messages. If Bob logs into a new device +and uses key sharing to obtain the decryption keys, the new device will +be sent the megolm sessions that have been ratcheted up. Bob's old +device can include the reason that the session was initially not shared +by including a `withheld` property in the `m.forwarded_room_key` message +that is an object with the `code` and `reason` properties from the +`m.room_key.withheld` message. + +{{m\_room\_key\_withheld\_event}} diff --git a/content/client-server-api/modules/event_context.md b/content/client-server-api/modules/event_context.md new file mode 100644 index 00000000..32d7b9e8 --- /dev/null +++ b/content/client-server-api/modules/event_context.md @@ -0,0 +1,21 @@ +--- +type: module +weight: 210 +--- + +### Event Context + +This API returns a number of events that happened just before and after +the specified event. This allows clients to get the context surrounding +an event. + +#### Client behaviour + +There is a single HTTP API for retrieving event context, documented +below. + +{{event\_context\_cs\_http\_api}} + +#### Security considerations + +The server must only return results that the user has permission to see. diff --git a/content/client-server-api/modules/guest_access.md b/content/client-server-api/modules/guest_access.md new file mode 100644 index 00000000..b68a8e01 --- /dev/null +++ b/content/client-server-api/modules/guest_access.md @@ -0,0 +1,91 @@ +--- +type: module +weight: 160 +--- + +### Guest Access + +There are times when it is desirable for clients to be able to interact +with rooms without having to fully register for an account on a +homeserver or join the room. This module specifies how these clients +should interact with servers in order to participate in rooms as guests. + +Guest users retrieve access tokens from a homeserver using the ordinary +[register +endpoint](#post_matrixclientr0register), +specifying the `kind` parameter as `guest`. They may then interact with +the client-server API as any other user would, but will only have access +to a subset of the API as described the Client behaviour subsection +below. Homeservers may choose not to allow this access at all to their +local users, but have no information about whether users on other +homeservers are guests or not. + +Guest users can also upgrade their account by going through the ordinary +`register` flow, but specifying the additional POST parameter +`guest_access_token` containing the guest's access token. They are also +required to specify the `username` parameter to the value of the local +part of their username, which is otherwise optional. + +This module does not fully factor in federation; it relies on individual +homeservers properly adhering to the rules set out in this module, +rather than allowing all homeservers to enforce the rules on each other. + +#### Events + +{{m\_room\_guest\_access\_event}} + +#### Client behaviour + +The following API endpoints are allowed to be accessed by guest accounts +for retrieving events: + +- [GET /rooms/:room\_id/state](#get_matrixclientr0roomsroomidstate) +- [GET /rooms/:room\_id/context/:event\_id](#get_matrixclientr0roomsroomidcontexteventid) +- [GET /rooms/:room\_id/event/:event\_id](#get_matrixclientr0roomsroomideventeventid) +- [GET /rooms/:room\_id/state/:event\_type/:state\_key](#get_matrixclientr0roomsroomidstateeventtypestatekey) +- [GET /rooms/:room\_id/messages](#get_matrixclientr0roomsroomidmessages) +- [GET /rooms/:room\_id/members](#get_matrixclientr0roomsroomidmembers) +- [GET /rooms/:room\_id/initialSync](#get_matrixclientr0roomsroomidinitialsync) +- [GET /sync](#get_matrixclientr0sync) +- [GET /events](#get_matrixclientr0events) as used for room previews. + +The following API endpoints are allowed to be accessed by guest accounts +for sending events: + +- [POST /rooms/:room\_id/join](#post_matrixclientr0roomsroomidjoin) +- [POST /rooms/:room\_id/leave](#post_matrixclientr0roomsroomidleave) +- [PUT /rooms/:room\_id/send/m.room.message/:txn\_id](#put_matrixclientr0roomsroomidsendeventtypetxnid) +- [PUT /sendToDevice/{eventType}/{txnId}](#put_matrixclientr0sendtodeviceeventtypetxnid) + +The following API endpoints are allowed to be accessed by guest accounts +for their own account maintenance: + +- [PUT /profile/:user\_id/displayname](#put_matrixclientr0profileuseriddisplayname) +- [GET /devices](#get_matrixclientr0devices) +- [GET /devices/{deviceId}](#get_matrixclientr0devicesdeviceid) +- [PUT /devices/{deviceId}](#put_matrixclientr0devicesdeviceid) + +The following API endpoints are allowed to be accessed by guest accounts +for end-to-end encryption: + +- [POST /keys/upload](#post_matrixclientr0keysupload) +- [POST /keys/query](#post_matrixclientr0keysquery) +- [POST /keys/claim](#post_matrixclientr0keysclaim) + +#### Server behaviour + +Servers MUST only allow guest users to join rooms if the +`m.room.guest_access` state event is present on the room, and has the +`guest_access` value `can_join`. If the `m.room.guest_access` event is +changed to stop this from being the case, the server MUST set those +users' `m.room.member` state to `leave`. + +#### Security considerations + +Each homeserver manages its own guest accounts itself, and whether an +account is a guest account or not is not information passed from server +to server. Accordingly, any server participating in a room is trusted to +properly enforce the permissions outlined in this section. + +Homeservers may want to enable protections such as captchas for guest +registration to prevent spam, denial of service, and similar attacks. diff --git a/content/client-server-api/modules/history_visibility.md b/content/client-server-api/modules/history_visibility.md new file mode 100644 index 00000000..8eba3643 --- /dev/null +++ b/content/client-server-api/modules/history_visibility.md @@ -0,0 +1,91 @@ +--- +type: module +weight: 120 +--- + +### Room History Visibility + +This module adds support for controlling the visibility of previous +events in a room. + +In all cases except `world_readable`, a user needs to join a room to +view events in that room. Once they have joined a room, they will gain +access to a subset of events in the room. How this subset is chosen is +controlled by the `m.room.history_visibility` event outlined below. +After a user has left a room, they may see any events which they were +allowed to see before they left the room, but no events received after +they left. + +The four options for the `m.room.history_visibility` event are: + +- `world_readable` - All events while this is the + `m.room.history_visibility` value may be shared by any participating + homeserver with anyone, regardless of whether they have ever joined + the room. +- `shared` - Previous events are always accessible to newly joined + members. All events in the room are accessible, even those sent when + the member was not a part of the room. +- `invited` - Events are accessible to newly joined members from the + point they were invited onwards. Events stop being accessible when + the member's state changes to something other than `invite` or + `join`. +- `joined` - Events are accessible to newly joined members from the + point they joined the room onwards. Events stop being accessible + when the member's state changes to something other than `join`. + +{{% boxes/warning %}} +These options are applied at the point an event is *sent*. Checks are +performed with the state of the `m.room.history_visibility` event when +the event in question is added to the DAG. This means clients cannot +retrospectively choose to show or hide history to new users if the +setting at that time was more restrictive. +{{% /boxes/warning %}} + +#### Events + +{{m\_room\_history\_visibility\_event}} + +#### Client behaviour + +Clients that implement this module MUST present to the user the possible +options for setting history visibility when creating a room. + +Clients may want to display a notice that their events may be read by +non-joined people if the value is set to `world_readable`. + +#### Server behaviour + +By default if no `history_visibility` is set, or if the value is not +understood, the visibility is assumed to be `shared`. The rules +governing whether a user is allowed to see an event depend on the state +of the room *at that event*. + +1. If the `history_visibility` was set to `world_readable`, allow. +2. If the user's `membership` was `join`, allow. +3. If `history_visibility` was set to `shared`, and the user joined the + room at any point after the event was sent, allow. +4. If the user's `membership` was `invite`, and the + `history_visibility` was set to `invited`, allow. +5. Otherwise, deny. + +For `m.room.history_visibility` events themselves, the user should be +allowed to see the event if the `history_visibility` before *or* after +the event would allow them to see it. (For example, a user should be +able to see `m.room.history_visibility` events which change the +`history_visibility` from `world_readable` to `joined` *or* from +`joined` to `world_readable`, even if that user was not a member of the +room.) + +Likewise, for the user's own `m.room.member` events, the user should be +allowed to see the event if their `membership` before *or* after the +event would allow them to see it. (For example, a user can always see +`m.room.member` events which set their membership to `join`, or which +change their membership from `join` to any other value, even if +`history_visibility` is `joined`.) + +#### Security considerations + +The default value for `history_visibility` is `shared` for +backwards-compatibility reasons. Clients need to be aware that by not +setting this event they are exposing all of their room history to anyone +in the room. diff --git a/content/client-server-api/modules/ignore_users.md b/content/client-server-api/modules/ignore_users.md new file mode 100644 index 00000000..18fadb67 --- /dev/null +++ b/content/client-server-api/modules/ignore_users.md @@ -0,0 +1,50 @@ +--- +type: module +weight: 240 +--- + +### Ignoring Users + +With all the communication through Matrix it may be desirable to ignore +a particular user for whatever reason. This module defines how clients +and servers can implement the ignoring of users. + +#### Events + +{{m\_ignored\_user\_list\_event}} + +#### Client behaviour + +To ignore a user, effectively blocking them, the client should add the +target user to the `m.ignored_user_list` event in their account data +using \_. Once ignored, the client will no longer receive events sent by +that user, with the exception of state events. The client should either +hide previous content sent by the newly ignored user or perform a new +`/sync` with no previous token. + +Invites to new rooms by ignored users will not be sent to the client. +The server may optionally reject the invite on behalf of the client. + +State events will still be sent to the client, even if the user is +ignored. This is to ensure parts, such as the room name, do not appear +different to the user just because they ignored the sender. + +To remove a user from the ignored users list, remove them from the +account data event. The server will resume sending events from the +previously ignored user, however it should not send events that were +missed while the user was ignored. To receive the events that were sent +while the user was ignored the client should perform a fresh sync. The +client may also un-hide any events it previously hid due to the user +becoming ignored. + +#### Server behaviour + +Following an update of the `m.ignored_user_list`, the sync API for all +clients should immediately start ignoring (or un-ignoring) the user. +Clients are responsible for determining if they should hide previously +sent events or to start a new sync stream. + +Servers must still send state events sent by ignored users to clients. + +Servers must not send room invites from ignored users to clients. +Servers may optionally decide to reject the invite, however. diff --git a/content/client-server-api/modules/instant_messaging.md b/content/client-server-api/modules/instant_messaging.md new file mode 100644 index 00000000..8550a7ae --- /dev/null +++ b/content/client-server-api/modules/instant_messaging.md @@ -0,0 +1,485 @@ +--- +type: module +weight: 10 +--- + +### Instant Messaging + +This module adds support for sending human-readable messages to a room. +It also adds support for associating human-readable information with the +room itself such as a room name and topic. + +#### Events + +{{m\_room\_message\_event}} + +{{m\_room\_message\_feedback\_event}} + +Usage of this event is discouraged for several reasons: +- The number of feedback events will grow very quickly with the number + of users in the room. This event provides no way to "batch" + feedback, unlike the [receipts module](#receipts). +- Pairing feedback to messages gets complicated when paginating as + feedback arrives before the message it is acknowledging. +- There are no guarantees that the client has seen the event ID being + acknowledged. + +{{m\_room\_name\_event}} + +{{m\_room\_topic\_event}} + +{{m\_room\_avatar\_event}} + +{{m\_room\_pinned\_events\_event}} + +##### m.room.message msgtypes + +Each [m.room.message](#m.room.message) MUST have a `msgtype` key which identifies the +type of message being sent. Each type has their own required and +optional keys, as outlined below. If a client cannot display the given +`msgtype` then it SHOULD display the fallback plain text `body` key +instead. + +Some message types support HTML in the event content that clients should +prefer to display if available. Currently `m.text`, `m.emote`, and +`m.notice` support an additional `format` parameter of +`org.matrix.custom.html`. When this field is present, a `formatted_body` +with the HTML must be provided. The plain text version of the HTML +should be provided in the `body`. + +Clients should limit the HTML they render to avoid Cross-Site Scripting, +HTML injection, and similar attacks. The strongly suggested set of HTML +tags to permit, denying the use and rendering of anything else, is: +`font`, `del`, `h1`, `h2`, `h3`, `h4`, `h5`, `h6`, `blockquote`, `p`, +`a`, `ul`, `ol`, `sup`, `sub`, `li`, `b`, `i`, `u`, `strong`, `em`, +`strike`, `code`, `hr`, `br`, `div`, `table`, `thead`, `tbody`, `tr`, +`th`, `td`, `caption`, `pre`, `span`, `img`. + +Not all attributes on those tags should be permitted as they may be +avenues for other disruption attempts, such as adding `onclick` handlers +or excessively large text. Clients should only permit the attributes +listed for the tags below. Where `data-mx-bg-color` and `data-mx-color` +are listed, clients should translate the value (a 6-character hex color +code) to the appropriate CSS/attributes for the tag. + +`font` +`data-mx-bg-color`, `data-mx-color` + +`span` +`data-mx-bg-color`, `data-mx-color` + +`a` +`name`, `target`, `href` (provided the value is not relative and has a +scheme matching one of: `https`, `http`, `ftp`, `mailto`, `magnet`) + +`img` +`width`, `height`, `alt`, `title`, `src` (provided it is a [Matrix +Content (MXC) URI](#matrix-content-mxc-uris)) + +`ol` +`start` + +`code` +`class` (only classes which start with `language-` for syntax +highlighting) + +Additionally, web clients should ensure that *all* `a` tags get a +`rel="noopener"` to prevent the target page from referencing the +client's tab/window. + +Tags must not be nested more than 100 levels deep. Clients should only +support the subset of tags they can render, falling back to other +representations of the tags where possible. For example, a client may +not be able to render tables correctly and instead could fall back to +rendering tab-delimited text. + +In addition to not rendering unsafe HTML, clients should not emit unsafe +HTML in events. Likewise, clients should not generate HTML that is not +needed, such as extra paragraph tags surrounding text due to Rich Text +Editors. HTML included in events should otherwise be valid, such as +having appropriate closing tags, appropriate attributes (considering the +custom ones defined in this specification), and generally valid +structure. + +A special tag, `mx-reply`, may appear on rich replies (described below) +and should be allowed if, and only if, the tag appears as the very first +tag in the `formatted_body`. The tag cannot be nested and cannot be +located after another tag in the tree. Because the tag contains HTML, an +`mx-reply` is expected to have a partner closing tag and should be +treated similar to a `div`. Clients that support rich replies will end +up stripping the tag and its contents and therefore may wish to exclude +the tag entirely. + +{{% boxes/note %}} +A future iteration of the specification will support more powerful and +extensible message formatting options, such as the proposal +[MSC1767](https://github.com/matrix-org/matrix-doc/pull/1767). +{{% /boxes/note %}} + +{{msgtype\_events}} + +#### Client behaviour + +Clients SHOULD verify the structure of incoming events to ensure that +the expected keys exist and that they are of the right type. Clients can +discard malformed events or display a placeholder message to the user. +Redacted `m.room.message` events MUST be removed from the client. This +can either be replaced with placeholder text (e.g. "\[REDACTED\]") or +the redacted message can be removed entirely from the messages view. + +Events which have attachments (e.g. `m.image`, `m.file`) SHOULD be +uploaded using the [content repository module](#content-repository) +where available. The resulting `mxc://` URI can then be used in the `url` +key. + +Clients MAY include a client generated thumbnail image for an attachment +under a `info.thumbnail_url` key. The thumbnail SHOULD also be a +`mxc://` URI. Clients displaying events with attachments can either use +the client generated thumbnail or ask its homeserver to generate a +thumbnail from the original attachment using the [content repository +module](#content-repository). + +##### Recommendations when sending messages + +In the event of send failure, clients SHOULD retry requests using an +exponential-backoff algorithm for a certain amount of time T. It is +recommended that T is no longer than 5 minutes. After this time, the +client should stop retrying and mark the message as "unsent". Users +should be able to manually resend unsent messages. + +Users may type several messages at once and send them all in quick +succession. Clients SHOULD preserve the order in which they were sent by +the user. This means that clients should wait for the response to the +previous request before sending the next request. This can lead to +head-of-line blocking. In order to reduce the impact of head-of-line +blocking, clients should use a queue per room rather than a global +queue, as ordering is only relevant within a single room rather than +between rooms. + +##### Local echo + +Messages SHOULD appear immediately in the message view when a user +presses the "send" button. This should occur even if the message is +still sending. This is referred to as "local echo". Clients SHOULD +implement "local echo" of messages. Clients MAY display messages in a +different format to indicate that the server has not processed the +message. This format should be removed when the server responds. + +Clients need to be able to match the message they are sending with the +same message which they receive from the event stream. The echo of the +same message from the event stream is referred to as "remote echo". Both +echoes need to be identified as the same message in order to prevent +duplicate messages being displayed. Ideally this pairing would occur +transparently to the user: the UI would not flicker as it transitions +from local to remote. Flickering can be reduced through clients making +use of the transaction ID they used to send a particular event. The +transaction ID used will be included in the event's `unsigned` data as +`transaction_id` when it arrives through the event stream. + +Clients unable to make use of the transaction ID are likely to +experience flickering when the remote echo arrives on the event stream +*before* the request to send the message completes. In that case the +event arrives before the client has obtained an event ID, making it +impossible to identify it as a remote echo. This results in the client +displaying the message twice for some time (depending on the server +responsiveness) before the original request to send the message +completes. Once it completes, the client can take remedial actions to +remove the duplicate event by looking for duplicate event IDs. + +##### Calculating the display name for a user + +Clients may wish to show the human-readable display name of a room +member as part of a membership list, or when they send a message. +However, different members may have conflicting display names. Display +names MUST be disambiguated before showing them to the user, in order to +prevent spoofing of other users. + +To ensure this is done consistently across clients, clients SHOULD use +the following algorithm to calculate a disambiguated display name for a +given user: + +1. Inspect the `m.room.member` state event for the relevant user id. +2. If the `m.room.member` state event has no `displayname` field, or if + that field has a `null` value, use the raw user id as the display + name. Otherwise: +3. If the `m.room.member` event has a `displayname` which is unique + among members of the room with `membership: join` or + `membership: invite`, use the given `displayname` as the + user-visible display name. Otherwise: +4. The `m.room.member` event has a non-unique `displayname`. This + should be disambiguated using the user id, for example "display name + (@id:homeserver.org)". + +Developers should take note of the following when implementing the above +algorithm: + +- The user-visible display name of one member can be affected by + changes in the state of another member. For example, if + `@user1:matrix.org` is present in a room, with `displayname: Alice`, + then when `@user2:example.com` joins the room, also with + `displayname: Alice`, *both* users must be given disambiguated + display names. Similarly, when one of the users then changes their + display name, there is no longer a clash, and *both* users can be + given their chosen display name. Clients should be alert to this + possibility and ensure that all affected users are correctly + renamed. +- The display name of a room may also be affected by changes in the + membership list. This is due to the room name sometimes being based + on user display names (see [Calculating the display name for a + room](#calculating-the-display-name-for-a-room)). +- If the entire membership list is searched for clashing display + names, this leads to an O(N^2) implementation for building the list + of room members. This will be very inefficient for rooms with large + numbers of members. It is recommended that client implementations + maintain a hash table mapping from `displayname` to a list of room + members using that name. Such a table can then be used for efficient + calculation of whether disambiguation is needed. + +##### Displaying membership information with messages + +Clients may wish to show the display name and avatar URL of the room +member who sent a message. This can be achieved by inspecting the +`m.room.member` state event for that user ID (see [Calculating the +display name for a user](#calculating-the-display-name-for-a-user)). + +When a user paginates the message history, clients may wish to show the +**historical** display name and avatar URL for a room member. This is +possible because older `m.room.member` events are returned when +paginating. This can be implemented efficiently by keeping two sets of +room state: old and current. As new events arrive and/or the user +paginates back in time, these two sets of state diverge from each other. +New events update the current state and paginated events update the old +state. When paginated events are processed sequentially, the old state +represents the state of the room *at the time the event was sent*. This +can then be used to set the historical display name and avatar URL. + +##### Calculating the display name for a room + +Clients may wish to show a human-readable name for a room. There are a +number of possibilities for choosing a useful name. To ensure that rooms +are named consistently across clients, clients SHOULD use the following +algorithm to choose a name: + +1. If the room has an [m.room.name](#m.room.name) state event with a non-empty + `name` field, use the name given by that field. +2. If the room has an [m.room.canonical\_alias](#m.room.canonical_alias) state event with a + valid `alias` field, use the alias given by that field as the name. + Note that clients should avoid using `alt_aliases` when calculating + the room name. +3. If none of the above conditions are met, a name should be composed + based on the members of the room. Clients should consider + [m.room.member](#m.room.member) events for users other than the logged-in user, as + defined below. + 1. If the number of `m.heroes` for the room are greater or equal to + `m.joined_member_count + m.invited_member_count - 1`, then use + the membership events for the heroes to calculate display names + for the users ([disambiguating them if + required](#calculating-the-display-name-for-a-user)) and + concatenating them. For example, the client may choose to show + "Alice, Bob, and Charlie (@charlie:example.org)" as the room + name. The client may optionally limit the number of users it + uses to generate a room name. + 2. If there are fewer heroes than + `m.joined_member_count + m.invited_member_count - 1`, and + `m.joined_member_count + m.invited_member_count` is greater than + 1, the client should use the heroes to calculate display names + for the users ([disambiguating them if + required](#calculating-the-display-name-for-a-user)) and + concatenating them alongside a count of the remaining users. For + example, "Alice, Bob, and 1234 others". + 3. If `m.joined_member_count + m.invited_member_count` is less than + or equal to 1 (indicating the member is alone), the client + should use the rules above to indicate that the room was empty. + For example, "Empty Room (was Alice)", "Empty Room (was Alice + and 1234 others)", or "Empty Room" if there are no heroes. + +Clients SHOULD internationalise the room name to the user's language +when using the `m.heroes` to calculate the name. Clients SHOULD use +minimum 5 heroes to calculate room names where possible, but may use +more or less to fit better with their user experience. + +##### Forming relationships between events + +In some cases, events may wish to reference other events. This could be +to form a thread of messages for the user to follow along with, or to +provide more context as to what a particular event is describing. +Currently, the only kind of relation defined is a "rich reply" where a +user may reference another message to create a thread-like conversation. + +Relationships are defined under an `m.relates_to` key in the event's +`content`. If the event is of the type `m.room.encrypted`, the +`m.relates_to` key MUST NOT be covered by the encryption and instead be +put alongside the encryption information held in the `content`. + +###### Rich replies + +Users may wish to reference another message when forming their own +message, and clients may wish to better embed the referenced message for +the user to have a better context for the conversation being had. This +sort of embedding another message in a message is known as a "rich +reply", or occasionally just a "reply". + +A rich reply is formed through use of an `m.relates_to` relation for +`m.in_reply_to` where a single key, `event_id`, is used to reference the +event being replied to. The referenced event ID SHOULD belong to the +same room where the reply is being sent. Clients should be cautious of +the event ID belonging to another room, or being invalid entirely. Rich +replies can only be constructed in the form of `m.room.message` events +with a `msgtype` of `m.text` or `m.notice`. Due to the fallback +requirements, rich replies cannot be constructed for types of `m.emote`, +`m.file`, etc. Rich replies may reference any other `m.room.message` +event, however. Rich replies may reference another event which also has +a rich reply, infinitely. + +An `m.in_reply_to` relationship looks like the following: + +``` +{ + ... + "type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "", + "format": "org.matrix.custom.html", + "formatted_body": "", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$another:event.com" + } + } + } +} +``` + +####### Fallbacks and event representation + +Some clients may not have support for rich replies and therefore need a +fallback to use instead. Clients that do not support rich replies should +render the event as if rich replies were not special. + +Clients that do support rich replies MUST provide the fallback format on +replies, and MUST strip the fallback before rendering the reply. Rich +replies MUST have a `format` of `org.matrix.custom.html` and therefore a +`formatted_body` alongside the `body` and appropriate `msgtype`. The +specific fallback text is different for each `msgtype`, however the +general format for the `body` is: + + > <@alice:example.org> This is the original body + + This is where the reply goes + +The `formatted_body` should use the following template: + + +
+ In reply to + @alice:example.org +
+ +
+ + This is where the reply goes. + +If the related event does not have a `formatted_body`, the event's +`body` should be considered after encoding any HTML special characters. +Note that the `href` in both of the anchors use a [matrix.to +URI](/appendices#matrixto-navigation). + +######## Stripping the fallback + +Clients which support rich replies MUST strip the fallback from the +event before rendering the event. This is because the text provided in +the fallback cannot be trusted to be an accurate representation of the +event. After removing the fallback, clients are recommended to represent +the event referenced by `m.in_reply_to` similar to the fallback's +representation, although clients do have creative freedom for their user +interface. Clients should prefer the `formatted_body` over the `body`, +just like with other `m.room.message` events. + +To strip the fallback on the `body`, the client should iterate over each +line of the string, removing any lines that start with the fallback +prefix ("> ", including the space, without quotes) and stopping when +a line is encountered without the prefix. This prefix is known as the +"fallback prefix sequence". + +To strip the fallback on the `formatted_body`, the client should remove +the entirety of the `mx-reply` tag. + +######## Fallback for `m.text`, `m.notice`, and unrecognised message types + +Using the prefix sequence, the first line of the related event's `body` +should be prefixed with the user's ID, followed by each line being +prefixed with the fallback prefix sequence. For example: + + > <@alice:example.org> This is the first line + > This is the second line + + This is the reply + +The `formatted_body` uses the template defined earlier in this section. + +######## Fallback for `m.emote` + +Similar to the fallback for `m.text`, each line gets prefixed with the +fallback prefix sequence. However an asterisk should be inserted before +the user's ID, like so: + + > * <@alice:example.org> feels like today is going to be a great day + + This is the reply + +The `formatted_body` has a subtle difference for the template where the +asterisk is also inserted ahead of the user's ID: + + +
+ In reply to + * @alice:example.org +
+ +
+ + This is where the reply goes. + +######## Fallback for `m.image`, `m.video`, `m.audio`, and `m.file` + +The related event's `body` would be a file name, which may not be very +descriptive. The related event should additionally not have a `format` +or `formatted_body` in the `content` - if the event does have a `format` +and/or `formatted_body`, those fields should be ignored. Because the +filename alone may not be descriptive, the related event's `body` should +be considered to be `"sent a file."` such that the output looks similar +to the following: + + > <@alice:example.org> sent a file. + + This is the reply + + +
+ In reply to + @alice:example.org +
+ sent a file. +
+ + This is where the reply goes. + +For `m.image`, the text should be `"sent an image."`. For `m.video`, the +text should be `"sent a video."`. For `m.audio`, the text should be +`"sent an audio file"`. + +#### Server behaviour + +Homeservers SHOULD reject `m.room.message` events which don't have a +`msgtype` key, or which don't have a textual `body` key, with an HTTP +status code of 400. + +#### Security considerations + +Messages sent using this module are not encrypted, although end to end +encryption is in development (see [E2E module](#end-to-end-encryption)). + +Clients should sanitise **all displayed keys** for unsafe HTML to +prevent Cross-Site Scripting (XSS) attacks. This includes room names and +topics. diff --git a/content/client-server-api/modules/mentions.md b/content/client-server-api/modules/mentions.md new file mode 100644 index 00000000..f6f9084a --- /dev/null +++ b/content/client-server-api/modules/mentions.md @@ -0,0 +1,60 @@ +--- +type: module +weight: 300 +--- + +### User, room, and group mentions + +This module allows users to mention other users, rooms, and groups +within a room message. This is achieved by including a [matrix.to +URI](/appendices/#matrixto-navigation) in the HTML body of an +[m.room.message](#mroommessage) event. This module does not have any server-specific +behaviour to it. + +Mentions apply only to [m.room.message](#mroommessage) events where the `msgtype` is +`m.text`, `m.emote`, or `m.notice`. The `format` for the event must be +`org.matrix.custom.html` and therefore requires a `formatted_body`. + +To make a mention, reference the entity being mentioned in the +`formatted_body` using an anchor, like so: + +```json +{ + "body": "Hello Alice!", + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": "Hello Alice!" +} +``` + +#### Client behaviour + +In addition to using the appropriate `matrix.to URI` for the mention, +clients should use the following guidelines when making mentions in +events to be sent: + +- When mentioning users, use the user's potentially ambiguous display + name for the anchor's text. If the user does not have a display + name, use the user's ID. +- When mentioning rooms, use the canonical alias for the room. If the + room does not have a canonical alias, prefer one of the aliases + listed on the room. If no alias can be found, fall back to the room + ID. In all cases, use the alias/room ID being linked to as the + anchor's text. +- When referencing groups, use the group ID as the anchor's text. + +The text component of the anchor should be used in the event's `body` +where the mention would normally be represented, as shown in the example +above. + +Clients should display mentions differently from other elements. For +example, this may be done by changing the background color of the +mention to indicate that it is different from a normal link. + +If the current user is mentioned in a message (either by a mention as +defined in this module or by a push rule), the client should show that +mention differently from other mentions, such as by using a red +background color to signify to the user that they were mentioned. + +When clicked, the mention should navigate the user to the appropriate +room, group, or user information. diff --git a/content/client-server-api/modules/moderation_policies.md b/content/client-server-api/modules/moderation_policies.md new file mode 100644 index 00000000..923df71e --- /dev/null +++ b/content/client-server-api/modules/moderation_policies.md @@ -0,0 +1,126 @@ +--- +type: module +weight: 330 +--- + +### Moderation policy lists + +With Matrix being an open network where anyone can participate, a very +wide range of content exists and it is important that users are +empowered to select which content they wish to see, and which content +they wish to block. By extension, room moderators and server admins +should also be able to select which content they do not wish to host in +their rooms and servers. + +The protocol's position on this is one of neutrality: it should not be +deciding what content is undesirable for any particular entity and +should instead be empowering those entities to make their own decisions. +As such, a generic framework for communicating "moderation policy lists" +or "moderation policy rooms" is described. Note that this module only +describes the data structures and not how they should be interpreting: +the entity making the decisions on filtering is best positioned to +interpret the rules how it sees fit. + +Moderation policy lists are stored as room state events. There are no +restrictions on how the rooms can be configured (they could be public, +private, encrypted, etc). + +There are currently 3 kinds of entities which can be affected by rules: +`user`, `server`, and `room`. All 3 are described with +`m.policy.rule.` state events. The `state_key` for a policy rule +is an arbitrary string decided by the sender of the rule. + +Rules contain recommendations and reasons for the rule existing. The +`reason` is a human-readable string which describes the +`recommendation`. Currently only one recommendation, `m.ban`, is +specified. + +#### `m.ban` recommendation + +When this recommendation is used, the entities affected by the rule +should be banned from participation where possible. The enforcement of +this is deliberately left as an implementation detail to avoid the +protocol imposing its opinion on how the policy list is to be +interpreted. However, a suggestion for a simple implementation is as +follows: + +- Is a `user` rule... + - Applied to a user: The user should be added to the subscriber's + ignore list. + - Applied to a room: The user should be banned from the room + (either on sight or immediately). + - Applied to a server: The user should not be allowed to send + invites to users on the server. +- Is a `room` rule... + - Applied to a user: The user should leave the room and not join + it + ([MSC2270](https://github.com/matrix-org/matrix-doc/pull/2270)-style + ignore). + - Applied to a room: No-op because a room cannot ban itself. + - Applied to a server: The server should prevent users from + joining the room and from receiving invites to it. +- Is a `server` rule... + - Applied to a user: The user should not receive events or invites + from the server. + - Applied to a room: The server is added as a denied server in the + ACLs. + - Applied to a server: The subscriber should avoid federating with + the server as much as possible by blocking invites from the + server and not sending traffic unless strictly required (no + outbound invites). + +#### Subscribing to policy lists + +This is deliberately left as an implementation detail. For +implementations using the Client-Server API, this could be as easy as +joining or peeking the room. Joining or peeking is not required, +however: an implementation could poll for updates or use a different +technique for receiving updates to the policy's rules. + +#### Sharing + +In addition to sharing a direct reference to the room which contains the +policy's rules, plain http or https URLs can be used to share links to +the list. When the URL is approached with a `Accept: application/json` +header or has `.json` appended to the end of the URL, it should return a +JSON object containing a `room_uri` property which references the room. +Currently this would be a `matrix.to` URI, however in future it could be +a Matrix-schemed URI instead. When not approached with the intent of +JSON, the service could return a user-friendly page describing what is +included in the ban list. + +#### Events + +The `entity` described by the state events can contain `*` and `?` to +match zero or more and one or more characters respectively. Note that +rules against rooms can describe a room ID or room alias - the +subscriber is responsible for resolving the alias to a room ID if +desired. + +{{m\_policy\_rule\_user\_event}} + +{{m\_policy\_rule\_room\_event}} + +{{m\_policy\_rule\_server\_event}} + +#### Client behaviour + +As described above, the client behaviour is deliberately left undefined. + +#### Server behaviour + +Servers have no additional requirements placed on them by this module. + +#### Security considerations + +This module could be used to build a system of shared blacklists, which +may create a divide within established communities if not carefully +deployed. This may well not be a suitable solution for all communities. + +Depending on how implementations handle subscriptions, user IDs may be +linked to policy lists and therefore expose the views of that user. For +example, a client implementation which joins the user to the policy room +would expose the user's ID to observers of the policy room. In future, +[MSC1228](https://github.com/matrix-org/matrix-doc/pulls/1228) and +[MSC1777](https://github.com/matrix-org/matrix-doc/pulls/1777) (or +similar) could help solve this concern. diff --git a/content/client-server-api/modules/openid.md b/content/client-server-api/modules/openid.md new file mode 100644 index 00000000..af8b31c9 --- /dev/null +++ b/content/client-server-api/modules/openid.md @@ -0,0 +1,13 @@ +--- +type: module +weight: 280 +--- + +### OpenID + +This module allows users to verify their identity with a third party +service. The third party service does need to be matrix-aware in that it +will need to know to resolve matrix homeservers to exchange the user's +token for identity information. + +{{openid\_cs\_http\_api}} diff --git a/content/client-server-api/modules/presence.md b/content/client-server-api/modules/presence.md new file mode 100644 index 00000000..83f9c560 --- /dev/null +++ b/content/client-server-api/modules/presence.md @@ -0,0 +1,76 @@ +--- +type: module +weight: 60 +--- + +### Presence + +Each user has the concept of presence information. This encodes: + +- Whether the user is currently online +- How recently the user was last active (as seen by the server) +- Whether a given client considers the user to be currently idle +- Arbitrary information about the user's current status (e.g. "in a + meeting"). + +This information is collated from both per-device (`online`, `idle`, +`last_active`) and per-user (status) data, aggregated by the user's +homeserver and transmitted as an `m.presence` event. Presence events are +sent to interested parties where users share a room membership. + +User's presence state is represented by the `presence` key, which is an +enum of one of the following: + +- `online` : The default state when the user is connected to an event + stream. +- `unavailable` : The user is not reachable at this time e.g. they are + idle. +- `offline` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. + +#### Events + +{{presence\_events}} + +#### Client behaviour + +Clients can manually set/get their presence using the HTTP APIs listed +below. + +{{presence\_cs\_http\_api}} + +##### Last active ago + +The server maintains a timestamp of the last time it saw a pro-active +event from the user. A pro-active event may be sending a message to a +room or changing presence state to `online`. This timestamp is presented +via a key called `last_active_ago` which gives the relative number of +milliseconds since the pro-active event. + +To reduce the number of presence updates sent to clients the server may +include a `currently_active` boolean field when the presence state is +`online`. When true, the server will not send further updates to the +last active time until an update is sent to the client with either a) +`currently_active` set to false or b) a presence state other than +`online`. During this period clients must consider the user to be +currently active, irrespective of the last active time. + +The last active time must be up to date whenever the server gives a +presence event to the client. The `currently_active` mechanism should +purely be used by servers to stop sending continuous presence updates, +as opposed to disabling last active tracking entirely. Thus clients can +fetch up to date last active times by explicitly requesting the presence +for a given user. + +##### Idle timeout + +The server will automatically set a user's presence to `unavailable` if +their last active time was over a threshold value (e.g. 5 minutes). +Clients can manually set a user's presence to `unavailable`. Any +activity that bumps the last active time on any of the user's clients +will cause the server to automatically set their presence to `online`. + +#### Security considerations + +Presence information is shared with all users who share a room with the +target user. In large public rooms this could be undesirable. diff --git a/content/client-server-api/modules/push.md b/content/client-server-api/modules/push.md new file mode 100644 index 00000000..00b24cb4 --- /dev/null +++ b/content/client-server-api/modules/push.md @@ -0,0 +1,748 @@ +--- +type: module +weight: 130 +--- + +### Push Notifications + +``` + +--------------------+ +-------------------+ + Matrix HTTP | | | | + Notification Protocol | App Developer | | Device Vendor | + | | | | + +-------------------+ | +----------------+ | | +---------------+ | + | | | | | | | | | | + | Matrix homeserver +-----> Push Gateway +------> Push Provider | | + | | | | | | | | | | + +-^-----------------+ | +----------------+ | | +----+----------+ | + | | | | | | + Matrix | | | | | | + Client/Server API + | | | | | + | | +--------------------+ +-------------------+ + | +--+-+ | + | | <-------------------------------------------+ + +---+ | + | | Provider Push Protocol + +----+ + Mobile Device or Client +``` + +This module adds support for push notifications. Homeservers send +notifications of events to user-configured HTTP endpoints. Users may +also configure a number of rules that determine which events generate +notifications. These are all stored and managed by the user's +homeserver. This allows user-specific push settings to be reused between +client applications. + +The above diagram shows the flow of push notifications being sent to a +handset where push notifications are submitted via the handset vendor, +such as Apple's APNS or Google's GCM. This happens as follows: + +1. The client app signs in to a homeserver. +2. The client app registers with its vendor's Push Provider and obtains + a routing token of some kind. +3. The mobile app uses the Client/Server API to add a 'pusher', + providing the URL of a specific Push Gateway which is configured for + that application. It also provides the routing token it has acquired + from the Push Provider. +4. The homeserver starts sending HTTP requests to the Push Gateway + using the supplied URL. The Push Gateway relays this notification to + the Push Provider, passing the routing token along with any + necessary private credentials the provider requires to send push + notifications. +5. The Push Provider sends the notification to the device. + +Definitions for terms used in this section are below: + +Push Provider +A push provider is a service managed by the device vendor which can send +notifications directly to the device. Google Cloud Messaging (GCM) and +Apple Push Notification Service (APNS) are two examples of push +providers. + +Push Gateway +A push gateway is a server that receives HTTP event notifications from +homeservers and passes them on to a different protocol such as APNS for +iOS devices or GCM for Android devices. Clients inform the homeserver +which Push Gateway to send notifications to when it sets up a Pusher. + +Pusher +A pusher is a worker on the homeserver that manages the sending of HTTP +notifications for a user. A user can have multiple pushers: one per +device. + +Push Rule +A push rule is a single rule that states under what *conditions* an +event should be passed onto a push gateway and *how* the notification +should be presented. These rules are stored on the user's homeserver. +They are manually configured by the user, who can create and view them +via the Client/Server API. + +Push Ruleset +A push ruleset *scopes a set of rules according to some criteria*. For +example, some rules may only be applied for messages from a particular +sender, a particular room, or by default. The push ruleset contains the +entire set of scopes and rules. + +#### Client behaviour + +Clients MUST configure a Pusher before they will receive push +notifications. There is a single API endpoint for this, as described +below. + +{{pusher\_cs\_http\_api}} + +##### Listing Notifications + +A client can retrieve a list of events that it has been notified about. +This may be useful so that users can see a summary of what important +messages they have received. + +{{notifications\_cs\_http\_api}} + +##### Receiving notifications + +Servers MUST include the number of unread notifications in a client's +`/sync` stream, and MUST update it as it changes. Notifications are +determined by the push rules which apply to an event. + +When the user updates their read receipt (either by using the API or by +sending an event), notifications prior to and including that event MUST +be marked as read. + +##### Push Rules + +A push rule is a single rule that states under what *conditions* an +event should be passed onto a push gateway and *how* the notification +should be presented. There are different "kinds" of push rules and each +rule has an associated priority. Every push rule MUST have a `kind` and +`rule_id`. The `rule_id` is a unique string within the kind of rule and +its' scope: `rule_ids` do not need to be unique between rules of the +same kind on different devices. Rules may have extra keys depending on +the value of `kind`. + +The different `kind`s of rule, in the order that they are checked, are: + +Override Rules `override` +The highest priority rules are user-configured overrides. + +Content-specific Rules `content` +These configure behaviour for (unencrypted) messages that match certain +patterns. Content rules take one parameter: `pattern`, that gives the +glob pattern to match against. This is treated in the same way as +`pattern` for `event_match`. + +Room-specific Rules `room` +These rules change the behaviour of all messages for a given room. The +`rule_id` of a room rule is always the ID of the room that it affects. + +Sender-specific rules `sender` +These rules configure notification behaviour for messages from a +specific Matrix user ID. The `rule_id` of Sender rules is always the +Matrix user ID of the user whose messages they'd apply to. + +Underride rules `underride` +These are identical to `override` rules, but have a lower priority than +`content`, `room` and `sender` rules. + +Rules with the same `kind` can specify an ordering priority. This +determines which rule is selected in the event of multiple matches. For +example, a rule matching "tea" and a separate rule matching "time" would +both match the sentence "It's time for tea". The ordering of the rules +would then resolve the tiebreak to determine which rule is executed. +Only `actions` for highest priority rule will be sent to the Push +Gateway. + +Each rule can be enabled or disabled. Disabled rules never match. If no +rules match an event, the homeserver MUST NOT notify the Push Gateway +for that event. Homeservers MUST NOT notify the Push Gateway for events +that the user has sent themselves. + +###### Actions + +All rules have an associated list of `actions`. An action affects if and +how a notification is delivered for a matching event. The following +actions are defined: + +`notify` +This causes each matching event to generate a notification. + +`dont_notify` +This prevents each matching event from generating a notification + +`coalesce` +This enables notifications for matching events but activates homeserver +specific behaviour to intelligently coalesce multiple events into a +single notification. Not all homeservers may support this. Those that do +not support it should treat it as the `notify` action. + +`set_tweak` +Sets an entry in the `tweaks` dictionary key that is sent in the +notification request to the Push Gateway. This takes the form of a +dictionary with a `set_tweak` key whose value is the name of the tweak +to set. It may also have a `value` key which is the value to which it +should be set. + +Actions that have no parameters are represented as a string. Otherwise, +they are represented as a dictionary with a key equal to their name and +other keys as their parameters, e.g. +`{ "set_tweak": "sound", "value": "default" }` + +####### Tweaks + +The `set_tweak` action is used to add an entry to the 'tweaks' +dictionary that is sent in the notification request to the Push Gateway. +The following tweaks are defined: + +`sound` +A string representing the sound to be played when this notification +arrives. A value of `default` means to play a default sound. A device +may choose to alert the user by some other means if appropriate, eg. +vibration. + +`highlight` +A boolean representing whether or not this message should be highlighted +in the UI. This will normally take the form of presenting the message in +a different colour and/or style. The UI might also be adjusted to draw +particular attention to the room in which the event occurred. If a +`highlight` tweak is given with no value, its value is defined to be +`true`. If no highlight tweak is given at all then the value of +`highlight` is defined to be false. + +Tweaks are passed transparently through the homeserver so client +applications and Push Gateways may agree on additional tweaks. For +example, a tweak may be added to specify how to flash the notification +light on a mobile device. + +###### Conditions + +`override` and `underride` rules MAY have a list of 'conditions'. All +conditions must hold true for an event in order for the rule to match. A +rule with no conditions always matches. The following conditions are +defined: + +`event_match` +This is a glob pattern match on a field of the event. Parameters: + +- `key`: The dot-separated field of the event to match, e.g. + `content.body` +- `pattern`: The glob-style pattern to match against. Patterns with no + special glob characters should be treated as having asterisks + prepended and appended when testing the condition. + +`contains_display_name` +This matches unencrypted messages where `content.body` contains the +owner's display name in that room. This is a separate rule because +display names may change and as such it would be hard to maintain a rule +that matched the user's display name. This condition has no parameters. + +`room_member_count` +This matches the current number of members in the room. Parameters: + +- `is`: A decimal integer optionally prefixed by one of, `==`, `<`, + `>`, `>=` or `<=`. A prefix of `<` matches rooms where the member + count is strictly less than the given number and so forth. If no + prefix is present, this parameter defaults to `==`. + +`sender_notification_permission` +This takes into account the current power levels in the room, ensuring +the sender of the event has high enough power to trigger the +notification. + +Parameters: + +- `key`: A string that determines the power level the sender must have + to trigger notifications of a given type, such as `room`. Refer to + the [m.room.power\_levels](#mroompower_levels) event schema for information about what + the defaults are and how to interpret the event. The `key` is used + to look up the power level required to send a notification type from + the `notifications` object in the power level event content. + +Unrecognised conditions MUST NOT match any events, effectively making +the push rule disabled. + +`room`, `sender` and `content` rules do not have conditions in the same +way, but instead have predefined conditions. In the cases of `room` and +`sender` rules, the `rule_id` of the rule determines its behaviour. + +###### Predefined Rules + +Homeservers can specify "server-default rules" which operate at a lower +priority than "user-defined rules". The `rule_id` for all server-default +rules MUST start with a dot (".") to identify them as "server-default". +The following server-default rules are specified: + +####### Default Override Rules + +######## `.m.rule.master` + +Matches all events. This can be enabled to turn off all push +notifications other than those generated by override rules set by the +user. By default this rule is disabled. + +Definition + +```json +{ + "rule_id": ".m.rule.master", + "default": true, + "enabled": false, + "conditions": [], + "actions": [ + "dont_notify" + ] +} +``` + +######## `.m.rule.suppress_notices` + +Matches messages with a `msgtype` of `notice`. + +Definition: + +```json +{ + "rule_id": ".m.rule.suppress_notices", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "content.msgtype", + "pattern": "m.notice", + } + ], + "actions": [ + "dont_notify", + ] +} +``` + +######## `.m.rule.invite_for_me` + +Matches any invites to a new room for this user. + +Definition: + +```json +{ + "rule_id": ".m.rule.invite_for_me", + "default": true, + "enabled": true, + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.room.member" + }, + { + "key": "content.membership", + "kind": "event_match", + "pattern": "invite" + }, + { + "key": "state_key", + "kind": "event_match", + "pattern": "[the user's Matrix ID]" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ] +} +``` + +######## `.m.rule.member_event` + +Matches any `m.room.member_event`. + +Definition: + +```json +{ + "rule_id": ".m.rule.member_event", + "default": true, + "enabled": true, + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.room.member" + } + ], + "actions": [ + "dont_notify" + ] +} +``` + +######## `.m.rule.contains_display_name` + +Matches any message whose content is unencrypted and contains the user's +current display name in the room in which it was sent. + +Definition: + +```json +{ + "rule_id": ".m.rule.contains_display_name", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "contains_display_name" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight" + } + ] +} +``` + +######## `.m.rule.tombstone` + +Matches any state event whose type is `m.room.tombstone`. This is +intended to notify users of a room when it is upgraded, similar to what +an `@room` notification would accomplish. + +Definition: + +```json +{ + "rule_id": ".m.rule.tombstone", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.tombstone" + }, + { + "kind": "event_match", + "key": "state_key", + "pattern": "" + } + ], + "actions": [ + "notify", + { + "set_tweak": "highlight" + } + ] +} +``` + +######## `.m.rule.roomnotif` + +Matches any message whose content is unencrypted and contains the text +`@room`, signifying the whole room should be notified of the event. + +Definition: + +```json +{ + "rule_id": ".m.rule.roomnotif", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "content.body", + "pattern": "@room" + }, + { + "kind": "sender_notification_permission", + "key": "room" + } + ], + "actions": [ + "notify", + { + "set_tweak": "highlight" + } + ] +} +``` + +####### Default Content Rules + +######## `.m.rule.contains_user_name` + +Matches any message whose content is unencrypted and contains the local +part of the user's Matrix ID, separated by word boundaries. + +Definition (as a `content` rule): + +```json +{ + "rule_id": ".m.rule.contains_user_name", + "default": true, + "enabled": true, + "pattern": "[the local part of the user's Matrix ID]", + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + }, + { + "set_tweak": "highlight" + } + ] +} +``` + +####### Default Underride Rules + +######## `.m.rule.call` + +Matches any incoming VOIP call. + +Definition: + +```json +{ + "rule_id": ".m.rule.call", + "default": true, + "enabled": true, + "conditions": [ + { + "key": "type", + "kind": "event_match", + "pattern": "m.call.invite" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "ring" + } + ] +} +``` + +######## `.m.rule.encrypted_room_one_to_one` + +Matches any encrypted event sent in a room with exactly two members. +Unlike other push rules, this rule cannot be matched against the content +of the event by nature of it being encrypted. This causes the rule to be +an "all or nothing" match where it either matches *all* events that are +encrypted (in 1:1 rooms) or none. + +Definition: + +```json +{ + "rule_id": ".m.rule.encrypted_room_one_to_one", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "room_member_count", + "is": "2" + }, + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.encrypted" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ] +} +``` + +######## `.m.rule.room_one_to_one` + +Matches any message sent in a room with exactly two members. + +Definition: + +```json +{ + "rule_id": ".m.rule.room_one_to_one", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "room_member_count", + "is": "2" + }, + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.message" + } + ], + "actions": [ + "notify", + { + "set_tweak": "sound", + "value": "default" + } + ] +} +``` + +######## `.m.rule.message` + +Matches all chat messages. + +Definition: + +```json +{ + "rule_id": ".m.rule.message", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.message" + } + ], + "actions": [ + "notify" + ] +} +``` + +######## `.m.rule.encrypted` + +Matches all encrypted events. Unlike other push rules, this rule cannot +be matched against the content of the event by nature of it being +encrypted. This causes the rule to be an "all or nothing" match where it +either matches *all* events that are encrypted (in group rooms) or none. + +Definition: + +```json +{ + "rule_id": ".m.rule.encrypted", + "default": true, + "enabled": true, + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.encrypted" + } + ], + "actions": [ + "notify" + ] +} +``` + +##### Push Rules: API + +Clients can retrieve, add, modify and remove push rules globally or +per-device using the APIs below. + +{{pushrules\_cs\_http\_api}} + +##### Push Rules: Events + +When a user changes their push rules a `m.push_rules` event is sent to +all clients in the `account_data` section of their next `/sync` request. +The content of the event is the current push rules for the user. + +{{m\_push\_rules\_event}} + +###### Examples + +To create a rule that suppresses notifications for the room with ID +`!dj234r78wl45Gh4D:matrix.org`: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ + '{ + "actions" : ["dont_notify"] + }' + +To suppress notifications for the user `@spambot:matrix.org`: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ + '{ + "actions" : ["dont_notify"] + }' + +To always notify for messages that contain the work 'cake' and set a +specific sound (with a rule\_id of `SSByZWFsbHkgbGlrZSBjYWtl`): + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ + '{ + "pattern": "cake", + "actions" : ["notify", {"set_sound":"cakealarm.wav"}] + }' + +To add a rule suppressing notifications for messages starting with +'cake' but ending with 'lie', superseding the previous rule: + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ + '{ + "pattern": "cake*lie", + "actions" : ["notify"] + }' + +To add a custom sound for notifications messages containing the word +'beer' in any rooms with 10 members or fewer (with greater importance +than the room, sender and content rules): + + curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ + '{ + "conditions": [ + {"kind": "event_match", "key": "content.body", "pattern": "beer" }, + {"kind": "room_member_count", "is": "<=10"} + ], + "actions" : [ + "notify", + {"set_sound":"beeroclock.wav"} + ] + }' + +#### Server behaviour + +#### Push Gateway behaviour + +##### Recommendations for APNS + +The exact format for sending APNS notifications is flexible and up to +the client app and its' push gateway to agree on. As APNS requires that +the sender has a private key owned by the app developer, each app must +have its own push gateway. It is recommended that: + +- The APNS token be base64 encoded and used as the pushkey. +- A different app\_id be used for apps on the production and sandbox + APS environments. +- APNS push gateways do not attempt to wait for errors from the APNS + gateway before returning and instead to store failures and return + 'rejected' responses next time that pushkey is used. + +#### Security considerations + +Clients specify the Push Gateway URL to use to send event notifications +to. This URL should be over HTTPS and *never* over HTTP. + +As push notifications will pass through a Push Provider, message content +shouldn't be sent in the push itself where possible. Instead, Push +Gateways should send a "sync" command to instruct the client to get new +events from the homeserver directly. diff --git a/content/client-server-api/modules/read_markers.md b/content/client-server-api/modules/read_markers.md new file mode 100644 index 00000000..ccbafd26 --- /dev/null +++ b/content/client-server-api/modules/read_markers.md @@ -0,0 +1,54 @@ +--- +type: module +weight: 50 +--- + +### Fully read markers + +The history for a given room may be split into three sections: messages +the user has read (or indicated they aren't interested in them), +messages the user might have read some but not others, and messages the +user hasn't seen yet. The "fully read marker" (also known as a "read +marker") marks the last event of the first section, whereas the user's +read receipt marks the last event of the second section. + +#### Events + +The user's fully read marker is kept as an event in the room's [account +data](#client-config). The event may be read to determine the user's +current fully read marker location in the room, and just like other +account data events the event will be pushed down the event stream when +updated. + +The fully read marker is kept under an `m.fully_read` event. If the +event does not exist on the user's account data, the fully read marker +should be considered to be the user's read receipt location. + +{{m\_fully\_read\_event}} + +#### Client behaviour + +The client cannot update fully read markers by directly modifying the +`m.fully_read` account data event. Instead, the client must make use of +the read markers API to change the values. + +The read markers API can additionally update the user's read receipt +(`m.read`) location in the same operation as setting the fully read +marker location. This is because read receipts and read markers are +commonly updated at the same time, and therefore the client might wish +to save an extra HTTP call. Providing an `m.read` location performs the +same task as a request to `/receipt/m.read/$event:example.org`. + +{{read\_markers\_cs\_http\_api}} + +#### Server behaviour + +The server MUST prevent clients from setting `m.fully_read` directly in +room account data. The server must additionally ensure that it treats +the presence of `m.read` in the `/read_markers` request the same as how +it would for a request to `/receipt/m.read/$event:example.org`. + +Upon updating the `m.fully_read` event due to a request to +`/read_markers`, the server MUST send the updated account data event +through to the client via the event stream (eg: `/sync`), provided any +applicable filters are also satisfied. diff --git a/content/client-server-api/modules/receipts.md b/content/client-server-api/modules/receipts.md new file mode 100644 index 00000000..423decea --- /dev/null +++ b/content/client-server-api/modules/receipts.md @@ -0,0 +1,89 @@ +--- +type: module +weight: 40 +--- + +### Receipts + +This module adds in support for receipts. These receipts are a form of +acknowledgement of an event. This module defines a single +acknowledgement: `m.read` which indicates that the user has read up to a +given event. + +Sending a receipt for each event can result in sending large amounts of +traffic to a homeserver. To prevent this from becoming a problem, +receipts are implemented using "up to" markers. This marker indicates +that the acknowledgement applies to all events "up to and including" the +event specified. For example, marking an event as "read" would indicate +that the user had read all events *up to* the referenced event. See the +[Receiving notifications](#receiving-notifications) section for more +information on how read receipts affect notification counts. + +#### Events + +Each `user_id`, `receipt_type` pair must be associated with only a +single `event_id`. + +{{m\_receipt\_event}} + +#### Client behaviour + +In `/sync`, receipts are listed under the `ephemeral` array of events +for a given room. New receipts that come down the event streams are +deltas which update existing mappings. Clients should replace older +receipt acknowledgements based on `user_id` and `receipt_type` pairs. +For example: + + Client receives m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $aaa:example.com + + Client receives another m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $bbb:example.com + + The client should replace the older acknowledgement for $aaa:example.com with + this one for $bbb:example.com + +Clients should send read receipts when there is some certainty that the +event in question has been **displayed** to the user. Simply receiving +an event does not provide enough certainty that the user has seen the +event. The user SHOULD need to *take some action* such as viewing the +room that the event was sent to or dismissing a notification in order +for the event to count as "read". Clients SHOULD NOT send read receipts +for events sent by their own user. + +A client can update the markers for its user by interacting with the +following HTTP APIs. + +{{receipts\_cs\_http\_api}} + +#### Server behaviour + +For efficiency, receipts SHOULD be batched into one event per room +before delivering them to clients. + +Receipts are sent across federation as EDUs with type `m.receipt`. The +format of the EDUs are: + +``` +{ + : { + : { + : { } + }, + ... + }, + ... +} +``` + +These are always sent as deltas to previously sent receipts. Currently +only a single `` should be used: `m.read`. + +#### Security considerations + +As receipts are sent outside the context of the event graph, there are +no integrity checks performed on the contents of `m.receipt` events. diff --git a/content/client-server-api/modules/report_content.md b/content/client-server-api/modules/report_content.md new file mode 100644 index 00000000..ff90207d --- /dev/null +++ b/content/client-server-api/modules/report_content.md @@ -0,0 +1,24 @@ +--- +type: module +weight: 260 +--- + +### Reporting Content + +Users may encounter content which they find inappropriate and should be +able to report it to the server administrators or room moderators for +review. This module defines a way for users to report content. + +Content is reported based upon a negative score, where -100 is "most +offensive" and 0 is "inoffensive". + +#### Client behaviour + +{{report\_content\_cs\_http\_api}} + +#### Server behaviour + +Servers are free to handle the reported content however they desire. +This may be a dedicated room to alert server administrators to the +reported content or some other mechanism for notifying the appropriate +people. diff --git a/content/client-server-api/modules/room_previews.md b/content/client-server-api/modules/room_previews.md new file mode 100644 index 00000000..f3368067 --- /dev/null +++ b/content/client-server-api/modules/room_previews.md @@ -0,0 +1,42 @@ +--- +type: module +weight: 170 +--- + +### Room Previews + +It is sometimes desirable to offer a preview of a room, where a user can +"lurk" and read messages posted to the room, without joining the room. +This can be particularly effective when combined with [Guest Access](#guest-access). + +Previews are implemented via the `world_readable` [Room History +Visibility](#room-history-visibility). setting, along with a special version of the [GET +/events](#get_matrixclientr0events) endpoint. + +#### Client behaviour + +A client wishing to view a room without joining it should call [GET +/rooms/:room\_id/initialSync](#get_matrixclientr0roomsroomidinitialsync), +followed by [GET /events](#get_matrixclientr0events). Clients will need to do +this in parallel for each room they wish to view. + +Clients can of course also call other endpoints such as [GET +/rooms/:room\_id/messages](#get_matrixclientr0roomsroomidmessages) +and [GET /search](#post_matrixclientr0search) to +access events outside the `/events` stream. + +{{peeking\_events\_cs\_http\_api}} + +#### Server behaviour + +For clients which have not joined a room, servers are required to only +return events where the room state at the event had the +`m.room.history_visibility` state event present with +`history_visibility` value `world_readable`. + +#### Security considerations + +Clients may wish to display to their users that rooms which are +`world_readable` *may* be showing messages to non-joined users. There is +no way using this module to find out whether any non-joined guest users +*do* see events in the room, or to list or count any lurking users. diff --git a/content/client-server-api/modules/room_upgrades.md b/content/client-server-api/modules/room_upgrades.md new file mode 100644 index 00000000..1e5437f3 --- /dev/null +++ b/content/client-server-api/modules/room_upgrades.md @@ -0,0 +1,73 @@ +--- +type: module +weight: 310 +--- + +### Room Upgrades + +From time to time, a room may need to be upgraded to a different room +version for a variety for reasons. This module defines a way for rooms +to upgrade to a different room version when needed. + +#### Events + +{{m\_room\_tombstone\_event}} + +#### Client behaviour + +Clients which understand `m.room.tombstone` events and the `predecessor` +field on `m.room.create` events should communicate to the user that the +room was upgraded. One way of accomplishing this would be hiding the old +room from the user's room list and showing banners linking between the +old and new room - ensuring that permalinks work when referencing the +old room. Another approach may be to virtually merge the rooms such that +the old room's timeline seamlessly continues into the new timeline +without the user having to jump between the rooms. + +{{room\_upgrades\_cs\_http\_api}} + +#### Server behaviour + +When the client requests to upgrade a known room to a known version, the +server: + +1. Checks that the user has permission to send `m.room.tombstone` + events in the room. + +2. Creates a replacement room with a `m.room.create` event containing a + `predecessor` field and the applicable `room_version`. + +3. Replicates transferable state events to the new room. The exact + details for what is transferred is left as an implementation detail, + however the recommended state events to transfer are: + + - `m.room.server_acl` + - `m.room.encryption` + - `m.room.name` + - `m.room.avatar` + - `m.room.topic` + - `m.room.guest_access` + - `m.room.history_visibility` + - `m.room.join_rules` + - `m.room.power_levels` + + Membership events should not be transferred to the new room due to + technical limitations of servers not being able to impersonate + people from other homeservers. Additionally, servers should not + transfer state events which are sensitive to who sent them, such as + events outside of the Matrix namespace where clients may rely on the + sender to match certain criteria. + +4. Moves any local aliases to the new room. + +5. Sends a `m.room.tombstone` event to the old room to indicate that it + is not intended to be used any further. + +6. If possible, the power levels in the old room should also be + modified to prevent sending of events and inviting new users. For + example, setting `events_default` and `invite` to the greater of + `50` and `users_default + 1`. + +When a user joins the new room, the server should automatically +transfer/replicate some of the user's personalized settings such as +notifications, tags, etc. diff --git a/content/client-server-api/modules/search.md b/content/client-server-api/modules/search.md new file mode 100644 index 00000000..3dfe6463 --- /dev/null +++ b/content/client-server-api/modules/search.md @@ -0,0 +1,91 @@ +--- +type: module +weight: 150 +--- + +### Server Side Search + +The search API allows clients to perform full text search across events +in all rooms that the user has been in, including those that they have +left. Only events that the user is allowed to see will be searched, e.g. +it won't include events in rooms that happened after you left. + +#### Client behaviour + +There is a single HTTP API for performing server-side search, documented +below. + +{{search\_cs\_http\_api}} + +#### Search Categories + +The search API allows clients to search in different categories of +items. Currently the only specified category is `room_events`. + +##### `room_events` + +This category covers all events that the user is allowed to see, +including events in rooms that they have left. The search is performed +on certain keys of certain event types. + +The supported keys to search over are: + +- `content.body` in `m.room.message` +- `content.name` in `m.room.name` +- `content.topic` in `m.room.topic` + +The search will *not* include rooms that are end to end encrypted. + +The results include a `rank` key that can be used to sort the results by +relevancy. The higher the `rank` the more relevant the result is. + +The value of `count` gives an approximation of the total number of +results. Homeservers may give an estimate rather than an exact value for +this field. + +#### Ordering + +The client can specify the ordering that the server returns results in. +The two allowed orderings are: + +- `rank`, which returns the most relevant results first. +- `recent`, which returns the most recent results first. + +The default ordering is `rank`. + +#### Groups + +The client can request that the results are returned along with grouping +information, e.g. grouped by `room_id`. In this case the response will +contain a group entry for each distinct value of `room_id`. Each group +entry contains at least a list of the `event_ids` that are in that +group, as well as potentially other metadata about the group. + +The current required supported groupings are: + +- `room_id` +- `sender` + +#### Pagination + +The server may return a `next_batch` key at various places in the +response. These are used to paginate the results. To fetch more results, +the client should send the *same* request to the server with a +`next_batch` query parameter set to that of the token. + +The scope of the pagination is defined depending on where the +`next_batch` token was returned. For example, using a token inside a +group will return more results from within that group. + +The currently supported locations for the `next_batch` token are: + +- `search_categories..next_batch` +- `search_categories..groups...next_batch` + +A server need not support pagination, even if there are more matching +results. In that case, they must not return a `next_batch` token in the +response. + +#### Security considerations + +The server must only return results that the user has permission to see. diff --git a/content/client-server-api/modules/secrets.md b/content/client-server-api/modules/secrets.md new file mode 100644 index 00000000..8db2bb3d --- /dev/null +++ b/content/client-server-api/modules/secrets.md @@ -0,0 +1,315 @@ +--- +type: module +weight: 110 +--- + +### Secrets + +Clients may have secret information that they wish to be made available +to other authorised clients, but that the server should not be able to +see, so the information must be encrypted as it passes through the +server. This can be done either asynchronously, by storing encrypted +data on the server for later retrieval, or synchronously, by sending +messages to each other. + +Each secret has an identifier that is used by clients to refer to the +secret when storing, fetching, requesting, or sharing the secret. +Secrets are plain strings; structured data can be stored by encoding it +as a string. + +#### Storage + +When secrets are stored on the server, they are stored in the user's +[account-data](#client-config), using an event type equal to the +secret's identifier. The keys that secrets are encrypted with are +described by data that is also stored in the user's account-data. Users +can have multiple keys, allowing them to control what sets of secrets +clients can access, depending on what keys are given to them. + +##### Key storage + +Each key has an ID, and the description of the key is stored in the +user's account\_data using the event type +`m.secret_storage.key.[key ID]`. The contents of the account data for +the key will include an `algorithm` property, which indicates the +encryption algorithm used, as well as a `name` property, which is a +human-readable name. Key descriptions may also have a `passphrase` +property for generating the key from a user-entered passphrase, as +described in [deriving keys from +passphrases](#deriving-keys-from-passphrases). + +`KeyDescription` + +| Parameter | Type | Description +|------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------| +| name | string | **Required.** The name of the key. | +| algorithm | string | **Required.** The encryption algorithm to be used for this key. Currently, only `m.secret_storage.v1.aes-hmac-sha2` is supported. | +| passphrase | string | See [deriving keys from passphrases](#deriving-keys-from-passphrases) section for a description of this property. | + +Other properties depend on the encryption algorithm, and are described +below. + +A key can be marked as the "default" key by setting the user's +account\_data with event type `m.secret_storage.default_key` to an +object that has the ID of the key as its `key` property. The default key +will be used to encrypt all secrets that the user would expect to be +available on all their clients. Unless the user specifies otherwise, +clients will try to use the default key to decrypt secrets. + +##### Secret storage + +Encrypted data is stored in the user's account\_data using the event +type defined by the feature that uses the data. The account\_data will +have an `encrypted` property that is a map from key ID to an object. The +algorithm from the `m.secret_storage.key.[key ID]` data for the given +key defines how the other properties are interpreted, though it's +expected that most encryption schemes would have `ciphertext` and `mac` +properties, where the `ciphertext` property is the unpadded +base64-encoded ciphertext, and the `mac` is used to ensure the integrity +of the data. + +`Secret` + +| Parameter | Type | Description | +|-----------|------------------|-------------| +| encrypted | {string: object} | **Required.** Map from key ID the encrypted data. The exact format for the encrypted data is dependent on the key algorithm. See the definition of `AesHmacSha2EncryptedData` in the [m.secret_storage.v1.aes-hmac-sha2](#msecret_storagev1aes-hmac-sha2) section. | + +Example: + +Some secret is encrypted using keys with ID `key_id_1` and `key_id_2`: + +`org.example.some.secret`: + +``` +{ + "encrypted": { + "key_id_1": { + "ciphertext": "base64+encoded+encrypted+data", + "mac": "base64+encoded+mac", + // ... other properties according to algorithm property in + // m.secret_storage.key.key_id_1 + }, + "key_id_2": { + // ... + } + } +} +``` + +and the key descriptions for the keys would be: + +`m.secret_storage.key.key_id_1`: + +``` +{ + "name": "Some key", + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + // ... other properties according to algorithm +} +``` + +`m.secret_storage.key.key_id_2`: + +``` +{ + "name": "Some other key", + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + // ... other properties according to algorithm +} +``` + +###### `m.secret_storage.v1.aes-hmac-sha2` + +Secrets encrypted using the `m.secret_storage.v1.aes-hmac-sha2` +algorithm are encrypted using AES-CTR-256, and authenticated using +HMAC-SHA-256. The secret is encrypted as follows: + +1. Given the secret storage key, generate 64 bytes by performing an + HKDF with SHA-256 as the hash, a salt of 32 bytes of 0, and with the + secret name as the info. The first 32 bytes are used as the AES key, + and the next 32 bytes are used as the MAC key +2. Generate 16 random bytes, set bit 63 to 0 (in order to work around + differences in AES-CTR implementations), and use this as the AES + initialization vector. This becomes the `iv` property, encoded using + base64. +3. Encrypt the data using AES-CTR-256 using the AES key generated + above. This encrypted data, encoded using base64, becomes the + `ciphertext` property. +4. Pass the raw encrypted data (prior to base64 encoding) through + HMAC-SHA-256 using the MAC key generated above. The resulting MAC is + base64-encoded and becomes the `mac` property. + +`AesHmacSha2EncryptedData` + +| Parameter | Type | Description +|------------|---------|------------------------------------------------------------------------| +| iv | string | **Required.** The 16-byte initialization vector, encoded as base64. | +| ciphertext | string | **Required.** The AES-CTR-encrypted data, encoded as base64. | +| mac | string | **Required.** The MAC, encoded as base64. | + +For the purposes of allowing clients to check whether a user has +correctly entered the key, clients should: + +1. encrypt and MAC a message consisting of 32 bytes of 0 as described + above, using the empty string as the info parameter to the HKDF in + step 1. +2. store the `iv` and `mac` in the `m.secret_storage.key.[key ID]` + account-data. + +`AesHmacSha2KeyDescription` + +| Parameter | Type | Description | +|-------------|--------|-----------------------------------------------------------------------------------------------------------------------------------| +| name | string | **Required.** The name of the key. | +| algorithm | string | **Required.** The encryption algorithm to be used for this key. Currently, only `m.secret_storage.v1.aes-hmac-sha2` is supported. | +| passphrase | object | See [deriving keys from passphrases](#deriving-keys-from-passphrases) section for a description of this property. | +| iv | string | The 16-byte initialization vector, encoded as base64. | +| mac | string | The MAC of the result of encrypting 32 bytes of 0, encoded as base64. | + +For example, the `m.secret_storage.key.key_id` for a key using this +algorithm could look like: + +```json +{ + "name": "m.default", + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "random+data", + "mac": "mac+of+encrypted+zeros" +} +``` + +and data encrypted using this algorithm could look like this: + +```json +{ + "encrypted": { + "key_id": { + "iv": "16+bytes+base64", + "ciphertext": "base64+encoded+encrypted+data", + "mac": "base64+encoded+mac" + } + } +} +``` + +###### Key representation + +When a user is given a raw key for `m.secret_storage.v1.aes-hmac-sha2`, +it will be presented as a string constructed as follows: + +1. The key is prepended by the two bytes `0x8b` and `0x01` +2. All the bytes in the string above, including the two header bytes, + are XORed together to form a parity byte. This parity byte is + appended to the byte string. +3. The byte string is encoded using base58, using the same [mapping as + is used for Bitcoin + addresses](https://en.bitcoin.it/wiki/Base58Check_encoding#Base58_symbol_chart), + that is, using the alphabet + `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz`. +4. The string is formatted into groups of four characters separated by + spaces. + +When decoding a raw key, the process should be reversed, with the +exception that whitespace is insignificant in the user's input. + +###### Deriving keys from passphrases + +A user may wish to use a chosen passphrase rather than a randomly +generated key. In this case, information on how to generate the key from +a passphrase will be stored in the `passphrase` property of the +`m.secret_storage.key.[key ID]` account-data. The `passphrase` property +has an `algorithm` property that indicates how to generate the key from +the passphrase. Other properties of the `passphrase` property are +defined by the `algorithm` specified. + +####### `m.pbkdf2` + +For the `m.pbkdf2` algorithm, the `passphrase` property has the +following properties: + +| Parameter | Type | Description | +|------------|---------|------------------------------------------------------------------------| +| algorithm | string | **Required.** Must be `m.pbkdf2` | +| salt | string | **Required.** The salt used in PBKDF2. | +| iterations | integer | **Required.** The number of iterations to use in PBKDF2. | +| bits | integer | Optional. The number of bits to generate for the key. Defaults to 256. | + +The key is generated using PBKDF2 with SHA-512 as the hash, using the +salt given in the `salt` parameter, and the number of iterations given +in the `iterations` parameter. + +Example: + +``` +{ + "passphrase": { + "algorithm": "m.pbkdf2", + "salt": "MmMsAlty", + "iterations": 100000, + "bits": 256 + }, + ... +} +``` + +#### Sharing + +To request a secret from other devices, a client sends an +`m.secret.requests` device event with `action` set to `request` and +`name` set to the identifier of the secret. A device that wishes to +share the secret will reply with an `m.secret.send` event, encrypted +using olm. When the original client obtains the secret, it sends an +`m.secret.request` event with `action` set to `request_cancellation` to +all devices other than the one that it received the secret from. Clients +should ignore `m.secret.send` events received from devices that it did +not send an `m.secret.request` event to. + +Clients must ensure that they only share secrets with other devices that +are allowed to see them. For example, clients should only share secrets +with the user’s own devices that are verified and may prompt the user to +confirm sharing the secret. + +##### Event definitions + +###### `m.secret.request` + +Sent by a client to request a secret from another device or to cancel a +previous request. It is sent as an unencrypted to-device event. + +| Parameter | Type | Description | +|-----------------------|--------|----------------------------------------------------------------------------------------| +| name | string | Required if ``action`` is ``request``. The name of the secret that is being requested. | +| action | enum | **Required.** One of ["request", "request_cancellation"]. | +| requesting_device_id | string | **Required.** The ID of the device requesting the secret. | +| request_id | string | **Required.** A random string uniquely identifying (with respect to the requester and the target) the target for a secret. If the secret is requested from multiple devices at the same time, the same ID may be used for every target. The same ID is also used in order to cancel a previous request. | + +Example: + +```json +{ + "name": "org.example.some.secret", + "action": "request", + "requesting_device_id": "ABCDEFG", + "request_id": "randomly_generated_id_9573" +} +``` + +###### `m.secret.send` + +Sent by a client to share a secret with another device, in response to +an `m.secret.request` event. It must be encrypted as an +`m.room.encrypted` event, then sent as a to-device event. + +| Parameter | Type | Description | +|-------------|--------|--------------------------------------------------------------| +| request_id | string | **Required.** The ID of the request that this a response to. | +| secret | string | **Required.** The contents of the secret. | + +Example: + +```json +{ + "request_id": "randomly_generated_id_9573", + "secret": "ThisIsASecretDon'tTellAnyone" +} +``` diff --git a/content/client-server-api/modules/send_to_device.md b/content/client-server-api/modules/send_to_device.md new file mode 100644 index 00000000..5a879637 --- /dev/null +++ b/content/client-server-api/modules/send_to_device.md @@ -0,0 +1,98 @@ +--- +type: module +weight: 80 +--- + +### Send-to-Device messaging + +This module provides a means by which clients can exchange signalling +messages without them being stored permanently as part of a shared +communication history. A message is delivered exactly once to each +client device. + +The primary motivation for this API is exchanging data that is +meaningless or undesirable to persist in the room DAG - for example, +one-time authentication tokens or key data. It is not intended for +conversational data, which should be sent using the normal \_ API for +consistency throughout Matrix. + +#### Client behaviour + +To send a message to other devices, a client should call +`/sendToDevice`\_. Only one message can be sent to each device per +transaction, and they must all have the same event type. The device ID +in the request body can be set to `*` to request that the message be +sent to all known devices. + +If there are send-to-device messages waiting for a client, they will be +returned by \_, as detailed in Extensions to /sync\_. Clients should +inspect the `type` of each returned event, and ignore any they do not +understand. + +#### Server behaviour + +Servers should store pending messages for local users until they are +successfully delivered to the destination device. When a client calls \_ +with an access token which corresponds to a device with pending +messages, the server should list the pending messages, in order of +arrival, in the response body. + +When the client calls `/sync` again with the `next_batch` token from the +first response, the server should infer that any send-to-device messages +in that response have been delivered successfully, and delete them from +the store. + +If there is a large queue of send-to-device messages, the server should +limit the number sent in each `/sync` response. 100 messages is +recommended as a reasonable limit. + +If the client sends messages to users on remote domains, those messages +should be sent on to the remote servers via +[federation](/server-server-api#send-to-device-messaging). + +#### Protocol definitions + +{{to\_device\_cs\_http\_api}} + +##### Extensions to /sync + +This module adds the following properties to the \_ response: + +| Parameter | Type | Description | +|-----------|-----------|-----------------------------------------------------------------------------| +| to_device | ToDevice | Optional. Information on the send-to-device messages for the client device. | + +`ToDevice` + +| Parameter | Type | Description | +|-----------|-----------|----------------------------------| +| events | [Event] | List of send-to-device messages. | + +`Event` + +| Parameter | Type | Description | +|------------|--------------|-------------------------------------------------------------------------------------------------| +| content | EventContent | The content of this event. The fields in this object will vary depending on the type of event. | +| sender | string | The Matrix user ID of the user who sent this event. | +| type | string | The type of event. | + +Example response: + +```json +{ + "next_batch": "s72595_4483_1934", + "rooms": {"leave": {}, "join": {}, "invite": {}}, + "to_device": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.new_device", + "content": { + "device_id": "XYZABCDE", + "rooms": ["!726s6s6q:example.com"] + } + } + ] + } +} +``` diff --git a/content/client-server-api/modules/server_acls.md b/content/client-server-api/modules/server_acls.md new file mode 100644 index 00000000..5ee09501 --- /dev/null +++ b/content/client-server-api/modules/server_acls.md @@ -0,0 +1,62 @@ +--- +type: module +weight: 290 +--- + +### Server Access Control Lists (ACLs) for rooms + +In some scenarios room operators may wish to prevent a malicious or +untrusted server from participating in their room. Sending an +[m.room.server\_acl](#mroomserver_acl) state event into a room is an effective way to +prevent the server from participating in the room at the federation +level. + +Server ACLs can also be used to make rooms only federate with a limited +set of servers, or retroactively make the room no longer federate with +any other server, similar to setting the `m.federate` value on the +[m.room.create](#mroomcreate) event. + +{{m\_room\_server\_acl\_event}} + +{{% boxes/note %}} +Port numbers are not supported because it is unclear to parsers whether +a port number should be matched or an IP address literal. Additionally, +it is unlikely that one would trust a server running on a particular +domain's port but not a different port, especially considering the +server host can easily change ports. +{{% /boxes/note %}} + +{{% boxes/note %}} +CIDR notation is not supported for IP addresses because Matrix does not +encourage the use of IPs for identifying servers. Instead, a blanket +`allow_ip_literals` is provided to cover banning them. +{{% /boxes/note %}} + +#### Client behaviour + +Clients are not expected to perform any additional duties beyond sending +the event. Clients should describe changes to the server ACLs to the +user in the user interface, such as in the timeline. + +Clients may wish to kick affected users from the room prior to denying a +server access to the room to help prevent those servers from +participating and to provide feedback to the users that they have been +excluded from the room. + +#### Server behaviour + +Servers MUST prevent blacklisted servers from sending events or +participating in the room when an [m.room.server\_acl](#mroomserver_acl) event is +present in the room state. Which APIs are specifically affected are +described in the Server-Server API specification. + +Servers should still send events to denied servers if they are still +residents of the room. + +#### Security considerations + +Server ACLs are only effective if every server in the room honours them. +Servers that do not honour the ACLs may still permit events sent by +denied servers into the room, leaking them to other servers in the room. +To effectively enforce an ACL in a room, the servers that do not honour +the ACLs should be denied in the room as well. diff --git a/content/client-server-api/modules/server_notices.md b/content/client-server-api/modules/server_notices.md new file mode 100644 index 00000000..13bafc46 --- /dev/null +++ b/content/client-server-api/modules/server_notices.md @@ -0,0 +1,68 @@ +--- +type: module +weight: 320 +--- + +### Server Notices + +Homeserver hosts often want to send messages to users in an official +capacity, or have resource limits which affect a user's ability to use +the homeserver. For example, the homeserver may be limited to a certain +number of active users per month and has exceeded that limit. To +communicate this failure to users, the homeserver would use the Server +Notices room. + +The aesthetics of the room (name, topic, avatar, etc) are left as an +implementation detail. It is recommended that the homeserver decorate +the room such that it looks like an official room to users. + +#### Events + +Notices are sent to the client as normal `m.room.message` events with a +`msgtype` of `m.server_notice` in the server notices room. Events with a +`m.server_notice` `msgtype` outside of the server notice room must be +ignored by clients. + +The specified values for `server_notice_type` are: + +`m.server_notice.usage_limit_reached` +The server has exceeded some limit which requires the server +administrator to intervene. The `limit_type` describes the kind of limit +reached. The specified values for `limit_type` are: + +`monthly_active_user` +The server's number of active users in the last 30 days has exceeded the +maximum. New connections are being refused by the server. What defines +"active" is left as an implementation detail, however servers are +encouraged to treat syncing users as "active". + +{{m\_room\_message\_m\_server\_notice\_event}} + +#### Client behaviour + +Clients can identify the server notices room by the `m.server_notice` +tag on the room. Active notices are represented by the [pinned +events](#mroompinned_events) in the server notices room. Server notice +events pinned in that room should be shown to the user through special +UI and not through the normal pinned events interface in the client. For +example, clients may show warning banners or bring up dialogs to get the +user's attention. Events which are not server notice events and are +pinned in the server notices room should be shown just like any other +pinned event in a room. + +The client must not expect to be able to reject an invite to join the +server notices room. Attempting to reject the invite must result in a +`M_CANNOT_LEAVE_SERVER_NOTICE_ROOM` error. Servers should not prevent +the user leaving the room after joining the server notices room, however +the same error code must be used if the server will prevent leaving the +room. + +#### Server behaviour + +Servers should manage exactly 1 server notices room per user. Servers +must identify this room to clients with the `m.server_notice` tag. +Servers should invite the target user rather than automatically join +them to the server notice room. + +How servers send notices to clients, and which user they use to send the +events, is left as an implementation detail for the server. diff --git a/content/client-server-api/modules/sso_login.md b/content/client-server-api/modules/sso_login.md new file mode 100644 index 00000000..36615375 --- /dev/null +++ b/content/client-server-api/modules/sso_login.md @@ -0,0 +1,311 @@ +--- +type: module +weight: 220 +--- + +### SSO client login/authentication + +Single Sign-On (SSO) is a generic term which refers to protocols which +allow users to log into applications via a single web-based +authentication portal. Examples include OpenID Connect, "Central +Authentication Service" (CAS) and SAML. + +This module allows a Matrix homeserver to delegate user authentication +to an external authentication server supporting one of these protocols. +In this process, there are three systems involved: + +- A Matrix client, using the APIs defined this specification, which + is seeking to authenticate a user to a Matrix homeserver. +- A Matrix homeserver, implementing the APIs defined in this + specification, but which is delegating user authentication to the + authentication server. +- An "authentication server", which is responsible for + authenticating the user. + +This specification is concerned only with communication between the +Matrix client and the homeserver, and is independent of the SSO protocol +used to communicate with the authentication server. Different Matrix +homeserver implementations might support different SSO protocols. + +Clients and homeservers implementing the SSO flow will need to consider +both [login](#login) and [user-interactive authentication](#user-interactive-authentication-api). The flow is +similar in both cases, but there are slight differences. + +Typically, SSO systems require a single "callback" URI to be configured +at the authentication server. Once the user is authenticated, their +browser is redirected to that URI. It is up to the Matrix homeserver +implementation to implement a suitable endpoint. For example, for CAS +authentication the homeserver should provide a means for the +administrator to configure where the CAS server is and the REST +endpoints which consume the ticket. + +#### Client login via SSO + +An overview of the process is as follows: + +1. The Matrix client calls `GET /login`\_ to find the supported login + types, and the homeserver includes a flow with + `"type": "m.login.sso"` in the response. +2. To initiate the `m.login.sso` login type, the Matrix client + instructs the user's browser to navigate to the + `/login/sso/redirect`\_ endpoint on the user's homeserver. +3. The homeserver responds with an HTTP redirect to the SSO user + interface, which the browser follows. +4. The authentication server and the homeserver interact to verify the + user's identity and other authentication information, potentially + using a number of redirects. +5. The browser is directed to the `redirectUrl` provided by the client + with a `loginToken` query parameter for the client to log in with. +6. The client exchanges the login token for an access token by calling + the `/login`\_ endpoint with a `type` of `m.login.token`. + +For native applications, typically steps 1 to 4 are carried out by +opening an embedded web view. + +These steps are illustrated as follows: + +``` + Matrix Client Matrix Homeserver Auth Server + | | | + |-------------(0) GET /login----------->| | + |<-------------login types--------------| | + | | | + | Webview | | + | | | | + |----->| | | + | |--(1) GET /login/sso/redirect-->| | + | |<---------(2) 302---------------| | + | | | | + | |<========(3) Authentication process================>| + | | | | + | |<--(4) redirect to redirectUrl--| | + |<-----| | | + | | | + |---(5) POST /login with login token--->| | + |<-------------access token-------------| | +``` + +{{% boxes/note %}} +In the older [r0.4.0 +version](https://matrix.org/docs/spec/client_server/r0.4.0.html#cas-based-client-login) +of this specification it was possible to authenticate via CAS when the +homeserver provides a `m.login.cas` login flow. This specification +deprecates the use of `m.login.cas` to instead prefer `m.login.sso`, +which is the same process with the only change being which redirect +endpoint to use: for `m.login.cas`, use `/cas/redirect` and for +`m.login.sso` use `/sso/redirect` (described below). The endpoints are +otherwise the same. +{{% /boxes/note %}} + +##### Client behaviour + +The client starts the process by instructing the browser to navigate to +`/login/sso/redirect`\_ with an appropriate `redirectUrl`. Once +authentication is successful, the browser will be redirected to that +`redirectUrl`. + +{{sso\_login\_redirect\_cs\_http\_api}} + +###### Security considerations + +1. CSRF attacks via manipulation of parameters on the `redirectUrl` + + Clients should validate any requests to the `redirectUrl`. In + particular, it may be possible for attackers to falsify any query + parameters, leading to cross-site request forgery (CSRF) attacks. + + For example, consider a web-based client at + `https://client.example.com`, which wants to initiate SSO login on + the homeserver at `server.example.org`. It does this by storing the + homeserver name in a query parameter for the `redirectUrl`: it + redirects to + `https://server.example.org/login/sso/redirect?redirectUrl=https://client.example.com?hs=server.example.org`. + + An attacker could trick a victim into following a link to + `https://server.example.org/login/sso/redirect?redirectUrl=https://client.example.com?hs=evil.com`, + which would result in the client sending a login token for the + victim's account to the attacker-controlled site `evil.com`. + + To guard against this, clients MUST NOT store state (such as the + address of the homeserver being logged into) anywhere it can be + modified by external processes. + + Instead, the state could be stored in + [localStorage](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage) + or in a cookie. + +2. For added security, clients SHOULD include a unique identifier in + the `redirectUrl` and reject any callbacks that do not contain a + recognised identifier, to guard against unsolicited login attempts + and replay attacks. + +##### Server behaviour + +###### Redirecting to the Authentication server + +The server should handle +`/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect` as follows: + +1. It should build a suitable request for the SSO system. +2. It should store enough state that the flow can be securely resumed + after the SSO process completes. One way to do this is by storing a + cookie which is stored in the user's browser, by adding a + `Set-Cookie` header to the response. +3. It should redirect the user's browser to the SSO login page with the + appropriate parameters. + +See also the "Security considerations" below. + +###### Handling the callback from the Authentication server + +Note that there will normally be a single callback URI which is used for +both login and user-interactive authentication: it is up to the +homeserver implementation to distinguish which is taking place. + +The homeserver should validate the response from the SSO system: this +may require additional calls to the authentication server, and/or may +require checking a signature on the response. + +The homeserver then proceeds as follows: + +1. The homeserver MUST map the user details received from the + authentication server to a valid [Matrix user + identifier](/appendices#user-identifiers). The guidance in + [Mapping from other character + sets](/appendices#mapping-from-other-character-sets) may be + useful. +2. If the generated user identifier represents a new user, it should be + registered as a new user. +3. The homeserver should generate a short-term login token. This is an + opaque token, suitable for use with the `m.login.token` type of the + `/login`\_ API. The lifetime of this token SHOULD be limited to + around five seconds. +4. The homeserver adds a query parameter of `loginToken`, with the + value of the generated login token, to the `redirectUrl` given in + the `/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect` + request. (Note: `redirectURL` may or may not include existing query + parameters. If it already includes one or more `loginToken` + parameters, they should be removed before adding the new one.) +5. The homeserver redirects the user's browser to the URI thus built. + +##### Security considerations + +1. Homeservers should ensure that login tokens are not sent to + malicious clients. + + For example, consider a homeserver at `server.example.org`. An + attacker tricks a victim into following a link to + `https://server.example.org/login/sso/redirect?redirectUrl=https://evil.com`, + resulting in a login token being sent to the attacker-controlled + site `evil.com`. This is a form of cross-site request forgery + (CSRF). + + To mitigate this, Homeservers SHOULD confirm with the user that they + are happy to grant access to their matrix account to the site named + in the `redirectUrl`. This can be done either *before* redirecting + to the SSO login page when handling the + `/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect` + endpoint, or *after* login when handling the callback from the + authentication server. (If the check is performed before + redirecting, it is particularly important that the homeserver guards + against unsolicited authentication attempts as below). + + It may be appropriate to whitelist a set of known-trusted client + URLs in this process. In particular, the homeserver's own [login + fallback](#login-fallback) implementation could be excluded. + +2. For added security, homeservers SHOULD guard against unsolicited + authentication attempts by tracking pending requests. One way to do + this is to set a cookie when handling + `/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect`, which + is checked and cleared when handling the callback from the + authentication server. + +#### SSO during User-Interactive Authentication + +[User-interactive authentication](#user-interactive-authentication-api) is used by client-server endpoints +which require additional confirmation of the user's identity (beyond +holding an access token). Typically this means that the user must +re-enter their password, but for homeservers which delegate to an SSO +server, this means redirecting to the authentication server during +user-interactive auth. + +The implemementation of this is based on the [Fallback](#fallback) mechanism for +user-interactive auth. + +#### Client behaviour + +Clients do not need to take any particular additional steps beyond +ensuring that the fallback mechanism has been implemented, and treating +the `m.login.sso` authentication type the same as any other unknown type +(i.e. they should open a browser window for +`/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web?session=`. +Once the flow has completed, the client retries the request with the +session only.) + +#### Server behaviour + +##### Redirecting to the Authentication server + +The server should handle +`/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web` +in much the same way as +`/_matrix/client/%CLIENT_MAJOR_VERSION%/login/sso/redirect`, which is to +say: + +1. It should build a suitable request for the SSO system. +2. It should store enough state that the flow can be securely resumed + after the SSO process completes. One way to do this is by storing a + cookie which is stored in the user's browser, by adding a + `Set-Cookie` header to the response. +3. It should redirect the user's browser to the SSO login page with the + appropriate parameters. + +See also the "Security considerations" below. + +###### Handling the callback from the Authentication server + +Note that there will normally be a single callback URI which is used for +both login and user-interactive authentication: it is up to the +homeserver implementation to distinguish which is taking place. + +The homeserver should validate the response from the SSO system: this +may require additional calls to the authentication server, and/or may +require checking a signature on the response. + +The homeserver then returns the [user-interactive authentication +fallback completion](#fallback) page to the user's browser. + +###### Security considerations + +1. Confirming the operation + + The homeserver SHOULD confirm that the user is happy for the + operation to go ahead. The goal of the user-interactive + authentication operation is to guard against a compromised + `access_token` being used to take over the user's account. Simply + redirecting the user to the SSO system is insufficient, since they + may not realise what is being asked of them, or the SSO system may + even confirm the authentication automatically. + + For example, the homeserver might serve a page with words to the + effect of: + + > A client is trying to remove a device from your account. To + > confirm this action, re-authenticate with single sign-on. If you + > did not expect this, your account may be compromised! + + This confirmation could take place before redirecting to the SSO + authentication page (when handling the + `/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web` + endpoint), or *after* authentication when handling the callback from + the authentication server. (If the check is performed before + redirecting, it is particularly important that the homeserver guards + against unsolicited authentication attempts as below). + +2. For added security, homeservers SHOULD guard against unsolicited + authentication attempts by tracking pending requests. One way to do + this is to set a cookie when handling + `/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/m.login.sso/fallback/web`, + which is checked and cleared when handling the callback from the + authentication server. diff --git a/content/client-server-api/modules/stickers.md b/content/client-server-api/modules/stickers.md new file mode 100644 index 00000000..b705d9ac --- /dev/null +++ b/content/client-server-api/modules/stickers.md @@ -0,0 +1,40 @@ +--- +type: module +weight: 250 +--- + +### Sticker Messages + +This module allows users to send sticker messages in to rooms or direct +messaging sessions. + +Sticker messages are specialised image messages that are displayed +without controls (e.g. no "download" link, or light-box view on click, +as would be displayed for for [m.image](#mimage) events). + +Sticker messages are intended to provide simple "reaction" events in the +message timeline. The matrix client should provide some mechanism to +display the sticker "body" e.g. as a tooltip on hover, or in a modal +when the sticker image is clicked. + +#### Events + +Sticker events are received as a single `m.sticker` event in the +`timeline` section of a room, in a `/sync`. + +{{m\_sticker\_event}} + +#### Client behaviour + +Clients supporting this message type should display the image content +from the event URL directly in the timeline. + +A thumbnail image should be provided in the `info` object. This is +largely intended as a fallback for clients that do not fully support the +`m.sticker` event type. In most cases it is fine to set the thumbnail +URL to the same URL as the main event content. + +It is recommended that sticker image content should be 512x512 pixels in +size or smaller. The dimensions of the image file should be twice the +intended display size specified in the `info` object in order to assist +rendering sharp images on higher DPI screens. diff --git a/content/client-server-api/modules/tags.md b/content/client-server-api/modules/tags.md new file mode 100644 index 00000000..94d455d2 --- /dev/null +++ b/content/client-server-api/modules/tags.md @@ -0,0 +1,66 @@ +--- +type: module +weight: 180 +--- + +### Room Tagging + +Users can add tags to rooms. Tags are namespaced strings used to label +rooms. A room may have multiple tags. Tags are only visible to the user +that set them but are shared across all their devices. + +#### Events + +The tags on a room are received as single `m.tag` event in the +`account_data` section of a room. The content of the `m.tag` event is a +`tags` key whose value is an object mapping the name of each tag to +another object. + +The JSON object associated with each tag gives information about the +tag, e.g how to order the rooms with a given tag. + +Ordering information is given under the `order` key as a number between +0 and 1. The numbers are compared such that 0 is displayed first. +Therefore a room with an `order` of `0.2` would be displayed before a +room with an `order` of `0.7`. If a room has a tag without an `order` +key then it should appear after the rooms with that tag that have an +`order` key. + +The name of a tag MUST NOT exceed 255 bytes. + +The tag namespace is defined as follows: + +- The namespace `m.*` is reserved for tags defined in the Matrix + specification. Clients must ignore any tags in this namespace they + don't understand. +- The namespace `u.*` is reserved for user-defined tags. The portion + of the string after the `u.` is defined to be the display name of + this tag. No other semantics should be inferred from tags in this + namespace. +- A client or app willing to use special tags for advanced + functionality should namespace them similarly to state keys: + `tld.name.*` +- Any tag in the `tld.name.*` form but not matching the namespace of + the current client should be ignored +- Any tag not matching the above rules should be interpreted as a user + tag from the `u.*` namespace, as if the name had already had `u.` + stripped from the start (ie. the name of the tag is used as the + display name directly). These non-namespaced tags are supported for + historical reasons. New tags should use one of the defined + namespaces above. + +Several special names are listed in the specification: The following +tags are defined in the `m.*` namespace: + +- `m.favourite`: The user's favourite rooms. These should be shown + with higher precedence than other rooms. +- `m.lowpriority`: These should be shown with lower precedence than + others. +- `m.server_notice`: Used to identify [Server Notice + Rooms](#server-notices). + +{{m\_tag\_event}} + +#### Client Behaviour + +{{tags\_cs\_http\_api}} diff --git a/content/client-server-api/modules/third_party_invites.md b/content/client-server-api/modules/third_party_invites.md new file mode 100644 index 00000000..3b37dd97 --- /dev/null +++ b/content/client-server-api/modules/third_party_invites.md @@ -0,0 +1,241 @@ +--- +type: module +weight: 140 +--- + +### Third party invites + +This module adds in support for inviting new members to a room where +their Matrix user ID is not known, instead addressing them by a third +party identifier such as an email address. There are two flows here; one +if a Matrix user ID is known for the third party identifier, and one if +not. Either way, the client calls `/invite` with the details of the +third party identifier. + +The homeserver asks the identity server whether a Matrix user ID is +known for that identifier: + +- If it is, an invite is simply issued for that user. +- If it is not, the homeserver asks the identity server to record the + details of the invitation, and to notify the invitee's homeserver of + this pending invitation if it gets a binding for this identifier in + the future. The identity server returns a token and public key to + the inviting homeserver. + +When the invitee's homeserver receives the notification of the binding, +it should insert an `m.room.member` event into the room's graph for that +user, with `content.membership` = `invite`, as well as a +`content.third_party_invite` property which contains proof that the +invitee does indeed own that third party identifier. See the +[m.room.member](#mroommember) schema for more information. + +#### Events + +{{m\_room\_third\_party\_invite\_event}} + +#### Client behaviour + +A client asks a server to invite a user by their third party identifier. + +{{third\_party\_membership\_cs\_http\_api}} + +#### Server behaviour + +Upon receipt of an `/invite`, the server is expected to look up the +third party identifier with the provided identity server. If the lookup +yields a result for a Matrix User ID then the normal invite process can +be initiated. This process ends up looking like this: + +``` + +---------+ +-------------+ +-----------------+ + | Client | | Homeserver | | IdentityServer | + +---------+ +-------------+ +-----------------+ + | | | + | POST /invite | | + |------------------------------------>| | + | | | + | | GET /lookup | + | |--------------------------------------------------->| + | | | + | | User ID result | + | |<---------------------------------------------------| + | | | + | | Invite process for the discovered User ID | + | |------------------------------------------ | + | | | | + | |<----------------------------------------- | + | | | + | Complete the /invite request | | + |<------------------------------------| | + | | | +``` + +However, if the lookup does not yield a bound User ID, the homeserver +must store the invite on the identity server and emit a valid +`m.room.third_party_invite` event to the room. This process ends up +looking like this: + +``` + +---------+ +-------------+ +-----------------+ + | Client | | Homeserver | | IdentityServer | + +---------+ +-------------+ +-----------------+ + | | | + | POST /invite | | + |------------------------------------>| | + | | | + | | GET /lookup | + | |-------------------------------------------------------------->| + | | | + | | "no users" result | + | |<--------------------------------------------------------------| + | | | + | | POST /store-invite | + | |-------------------------------------------------------------->| + | | | + | | Information needed for the m.room.third_party_invite | + | |<--------------------------------------------------------------| + | | | + | | Emit m.room.third_party_invite to the room | + | |------------------------------------------- | + | | | | + | |<------------------------------------------ | + | | | + | Complete the /invite request | | + |<------------------------------------| | + | | | +``` + +All homeservers MUST verify the signature in the event's +`content.third_party_invite.signed` object. + +The third party user will then need to verify their identity, which +results in a call from the identity server to the homeserver that bound +the third party identifier to a user. The homeserver then exchanges the +`m.room.third_party_invite` event in the room for a complete +`m.room.member` event for `membership: invite` for the user that has +bound the third party identifier. + +If a homeserver is joining a room for the first time because of an +`m.room.third_party_invite`, the server which is already participating +in the room (which is chosen as per the standard server-server +specification) MUST validate that the public key used for signing is +still valid, by checking `key_validity_url` in the above described way. + +No other homeservers may reject the joining of the room on the basis of +`key_validity_url`, this is so that all homeservers have a consistent +view of the room. They may, however, indicate to their clients that a +member's membership is questionable. + +For example, given H1, H2, and H3 as homeservers, UserA as a user of H1, +and an identity server IS, the full sequence for a third party invite +would look like the following. This diagram assumes H1 and H2 are +residents of the room while H3 is attempting to join. + +``` + +-------+ +-----------------+ +-----+ +-----+ +-----+ +-----+ + | UserA | | ThirdPartyUser | | H1 | | H2 | | H3 | | IS | + +-------+ +-----------------+ +-----+ +-----+ +-----+ +-----+ + | | | | | | + | POST /invite for ThirdPartyUser | | | | + |----------------------------------->| | | | + | | | | | | + | | | GET /lookup | | | + | | |---------------------------------------------------------------------------------------------->| + | | | | | | + | | | | Lookup results (empty object) | + | | |<----------------------------------------------------------------------------------------------| + | | | | | | + | | | POST /store-invite | | | + | | |---------------------------------------------------------------------------------------------->| + | | | | | | + | | | | Token, keys, etc for third party invite | + | | |<----------------------------------------------------------------------------------------------| + | | | | | | + | | | (Federation) Emit m.room.third_party_invite | | | + | | |----------------------------------------------->| | | + | | | | | | + | Complete /invite request | | | | + |<-----------------------------------| | | | + | | | | | | + | | Verify identity | | | | + | |-------------------------------------------------------------------------------------------------------------------->| + | | | | | | + | | | | | POST /3pid/onbind | + | | | | |<---------------------------| + | | | | | | + | | | PUT /exchange_third_party_invite/:roomId | | + | | |<-----------------------------------------------------------------| | + | | | | | | + | | | Verify the request | | | + | | |------------------- | | | + | | | | | | | + | | |<------------------ | | | + | | | | | | + | | | (Federation) Emit m.room.member for invite | | | + | | |----------------------------------------------->| | | + | | | | | | + | | | | | | + | | | (Federation) Emit the m.room.member event sent to H2 | | + | | |----------------------------------------------------------------->| | + | | | | | | + | | | Complete /exchange_third_party_invite/:roomId request | | + | | |----------------------------------------------------------------->| | + | | | | | | + | | | | | Participate in the room | + | | | | |------------------------ | + | | | | | | | + | | | | |<----------------------- | + | | | | | | +``` + +Note that when H1 sends the `m.room.member` event to H2 and H3 it does +not have to block on either server's receipt of the event. Likewise, H1 +may complete the `/exchange_third_party_invite/:roomId` request at the +same time as sending the `m.room.member` event to H2 and H3. +Additionally, H3 may complete the `/3pid/onbind` request it got from IS +at any time - the completion is not shown in the diagram. + +H1 MUST verify the request from H3 to ensure the `signed` property is +correct as well as the `key_validity_url` as still being valid. This is +done by making a request to the [identity server +/isvalid](/identity-service-api/#get_matrixidentityv2pubkeyisvalid) +endpoint, using the provided URL rather than constructing a new one. The +query string and response for the provided URL must match the Identity +Service Specification. + +The reason that no other homeserver may reject the event based on +checking `key_validity_url` is that we must ensure event acceptance is +deterministic. If some other participating server doesn't have a network +path to the keyserver, or if the keyserver were to go offline, or revoke +its keys, that other server would reject the event and cause the +participating servers' graphs to diverge. This relies on participating +servers trusting each other, but that trust is already implied by the +server-server protocol. Also, the public key signature verification must +still be performed, so the attack surface here is minimized. + +#### Security considerations + +There are a number of privacy and trust implications to this module. + +It is important for user privacy that leaking the mapping between a +matrix user ID and a third party identifier is hard. In particular, +being able to look up all third party identifiers from a matrix user ID +(and accordingly, being able to link each third party identifier) should +be avoided wherever possible. To this end, the third party identifier is +not put in any event, rather an opaque display name provided by the +identity server is put into the events. Clients should not remember or +display third party identifiers from invites, other than for the use of +the inviter themself. + +Homeservers are not required to trust any particular identity server(s). +It is generally a client's responsibility to decide which identity +servers it trusts, not a homeserver's. Accordingly, this API takes +identity servers as input from end users, and doesn't have any specific +trusted set. It is possible some homeservers may want to supply +defaults, or reject some identity servers for *its* users, but no +homeserver is allowed to dictate which identity servers *other* +homeservers' users trust. + +There is some risk of denial of service attacks by flooding homeservers +or identity servers with many requests, or much state to store. +Defending against these is left to the implementer's discretion. diff --git a/content/client-server-api/modules/third_party_networks.md b/content/client-server-api/modules/third_party_networks.md new file mode 100644 index 00000000..13142470 --- /dev/null +++ b/content/client-server-api/modules/third_party_networks.md @@ -0,0 +1,22 @@ +--- +type: module +weight: 270 +--- + +### Third Party Networks + +Application services can provide access to third party networks via +bridging. This allows Matrix users to communicate with users on other +communication platforms, with messages ferried back and forth by the +application service. A single application service may bridge multiple +third party networks, and many individual locations within those +networks. A single third party network location may be bridged to +multiple Matrix rooms. + +#### Third Party Lookups + +A client may wish to provide a rich interface for joining third party +locations and connecting with third party users. Information necessary +for such an interface is provided by third party lookups. + +{{third\_party\_lookup\_cs\_http\_api}} diff --git a/content/client-server-api/modules/typing_notifications.md b/content/client-server-api/modules/typing_notifications.md new file mode 100644 index 00000000..d899b4e1 --- /dev/null +++ b/content/client-server-api/modules/typing_notifications.md @@ -0,0 +1,41 @@ +--- +type: module +weight: 30 +--- + +### Typing Notifications + +Users may wish to be informed when another user is typing in a room. +This can be achieved using typing notifications. These are ephemeral +events scoped to a `room_id`. This means they do not form part of the +[Event Graph](index.html#event-graphs) but still have a `room_id` key. + +#### Events + +{{m\_typing\_event}} + +#### Client behaviour + +When a client receives an `m.typing` event, it MUST use the user ID list +to **REPLACE** its knowledge of every user who is currently typing. The +reason for this is that the server *does not remember* users who are not +currently typing as that list gets big quickly. The client should mark +as not typing any user ID who is not in that list. + +It is recommended that clients store a `boolean` indicating whether the +user is typing or not. Whilst this value is `true` a timer should fire +periodically every N seconds to send a typing HTTP request. The value of +N is recommended to be no more than 20-30 seconds. This request should +be re-sent by the client to continue informing the server the user is +still typing. As subsequent requests will replace older requests, a +safety margin of 5 seconds before the expected timeout runs out is +recommended. When the user stops typing, the state change of the +`boolean` to `false` should trigger another HTTP request to inform the +server that the user has stopped typing. + +{{typing\_cs\_http\_api}} + +#### Security considerations + +Clients may not wish to inform everyone in a room that they are typing +and instead only specific users in the room. diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md new file mode 100644 index 00000000..31cd3fec --- /dev/null +++ b/content/client-server-api/modules/voip_events.md @@ -0,0 +1,91 @@ +--- +type: module +weight: 20 +--- + +### Voice over IP + +This module outlines how two users in a room can set up a Voice over IP +(VoIP) call to each other. Voice and video calls are built upon the +WebRTC 1.0 standard. Call signalling is achieved by sending [message +events](#events) to the room. In this version of the spec, only two-party +communication is supported (e.g. between two peers, or between a peer +and a multi-point conferencing unit). This means that clients MUST only +send call events to rooms with exactly two participants. + +#### Events + +{{voip\_events}} + +#### Client behaviour + +A call is set up with message events exchanged as follows: + +``` + Caller Callee + [Place Call] + m.call.invite -----------> + m.call.candidate --------> + [..candidates..] --------> + [Answers call] + <--------------- m.call.answer + [Call is active and ongoing] + <--------------- m.call.hangup +``` + +Or a rejected call: + +``` + Caller Callee + m.call.invite ------------> + m.call.candidate ---------> + [..candidates..] ---------> + [Rejects call] + <-------------- m.call.hangup +``` + +Calls are negotiated according to the WebRTC specification. + +##### Glare + +"Glare" is a problem which occurs when two users call each other at +roughly the same time. This results in the call failing to set up as +there already is an incoming/outgoing call. A glare resolution algorithm +can be used to determine which call to hangup and which call to answer. +If both clients implement the same algorithm then they will both select +the same call and the call will be successfully connected. + +As calls are "placed" to rooms rather than users, the glare resolution +algorithm outlined below is only considered for calls which are to the +same room. The algorithm is as follows: + +- If an `m.call.invite` to a room is received whilst the client is + **preparing to send** an `m.call.invite` to the same room: + - the client should cancel its outgoing call and instead + automatically accept the incoming call on behalf of the user. +- If an `m.call.invite` to a room is received **after the client has + sent** an `m.call.invite` to the same room and is waiting for a + response: + - the client should perform a lexicographical comparison of the + call IDs of the two calls and use the *lesser* of the two calls, + aborting the greater. If the incoming call is the lesser, the + client should accept this call on behalf of the user. + +The call setup should appear seamless to the user as if they had simply +placed a call and the other party had accepted. This means any media +stream that had been setup for use on a call should be transferred and +used for the call that replaces it. + +#### Server behaviour + +The homeserver MAY provide a TURN server which clients can use to +contact the remote party. The following HTTP API endpoints will be used +by clients in order to get information about the TURN server. + +{{voip\_cs\_http\_api}} + +#### Security considerations + +Calls should only be placed to rooms with one other user in them. If +they are placed to group chat rooms it is possible that another user +will intercept and answer the call. diff --git a/content/identity-service-api.md b/content/identity-service-api.md new file mode 100644 index 00000000..ef9d1bf0 --- /dev/null +++ b/content/identity-service-api.md @@ -0,0 +1,443 @@ +--- +title: "Identity Service API" +weight: 40 +type: docs +--- + +The Matrix client-server and server-server APIs are largely expressed in +Matrix user identifiers. From time to time, it is useful to refer to +users by other ("third-party") identifiers, or "3PID"s, e.g. their email +address or phone number. This Identity Service Specification describes +how mappings between third-party identifiers and Matrix user identifiers +can be established, validated, and used. This description technically +may apply to any 3PID, but in practice has only been applied +specifically to email addresses and phone numbers. + +## General principles + +The purpose of an identity server is to validate, store, and answer +questions about the identities of users. In particular, it stores +associations of the form "identifier X represents the same user as +identifier Y", where identities may exist on different systems (such as +email addresses, phone numbers, Matrix user IDs, etc). + +The identity server has some private-public keypairs. When asked about +an association, it will sign details of the association with its private +key. Clients may validate the assertions about associations by verifying +the signature with the public key of the identity server. + +In general, identity servers are treated as reliable oracles. They do +not necessarily provide evidence that they have validated associations, +but claim to have done so. Establishing the trustworthiness of an +individual identity server is left as an exercise for the client. + +3PID types are described in [3PID Types](/appendices#pid-types) +Appendix. + +## API standards + +The mandatory baseline for identity server communication in Matrix is +exchanging JSON objects over HTTP APIs. HTTPS is required for +communication, and all API calls use a Content-Type of +`application/json`. In addition, strings MUST be encoded as UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard +error response". This is a JSON object which looks like: + +```json +{ + "errcode": "", + "error": "" +} +``` + +The `error` string will be a human-readable error message, usually a +sentence explaining what went wrong. The `errcode` string will be a +unique string which can be used to handle an error message e.g. +`M_FORBIDDEN`. There may be additional keys depending on the error, but +the keys `error` and `errcode` MUST always be present. + +Some standard error codes are below: + +`M_NOT_FOUND` +The resource requested could not be located. + +`M_MISSING_PARAMS` +The request was missing one or more parameters. + +`M_INVALID_PARAM` +The request contained one or more invalid parameters. + +`M_SESSION_NOT_VALIDATED` +The session has not been validated. + +`M_NO_VALID_SESSION` +A session could not be located for the given parameters. + +`M_SESSION_EXPIRED` +The session has expired and must be renewed. + +`M_INVALID_EMAIL` +The email address provided was not valid. + +`M_EMAIL_SEND_ERROR` +There was an error sending an email. Typically seen when attempting to +verify ownership of a given email address. + +`M_INVALID_ADDRESS` +The provided third party address was not valid. + +`M_SEND_ERROR` +There was an error sending a notification. Typically seen when +attempting to verify ownership of a given third party address. + +`M_UNRECOGNIZED` +The request contained an unrecognised value, such as an unknown token or +medium. + +`M_THREEPID_IN_USE` +The third party identifier is already in use by another user. Typically +this error will have an additional `mxid` property to indicate who owns +the third party identifier. + +`M_UNKNOWN` +An unknown error has occurred. + +## Privacy + +Identity is a privacy-sensitive issue. While the identity server exists +to provide identity information, access should be restricted to avoid +leaking potentially sensitive data. In particular, being able to +construct large-scale connections between identities should be avoided. +To this end, in general APIs should allow a 3PID to be mapped to a +Matrix user identity, but not in the other direction (i.e. one should +not be able to get all 3PIDs associated with a Matrix user ID, or get +all 3PIDs associated with a 3PID). + +## Version 1 API deprecation + +As described on each of the version 1 endpoints, the v1 API is +deprecated in favour of the v2 API described here. The major difference, +with the exception of a few isolated cases, is that the v2 API requires +authentication to ensure the user has given permission for the identity +server to operate on their data. + +The v1 API is planned to be removed from the specification in a future +version. + +Clients SHOULD attempt the v2 endpoints first, and if they receive a +`404`, `400`, or similar error they should try the v1 endpoint or fail +the operation. Clients are strongly encouraged to warn the user of the +risks in using the v1 API, if they are planning on using it. + +## Web browser clients + +It is realistic to expect that some clients will be written to be run +within a web browser or similar environment. In these cases, the +identity server should respond to pre-flight requests and supply +Cross-Origin Resource Sharing (CORS) headers on all requests. + +When a client approaches the server with a pre-flight (OPTIONS) request, +the server should respond with the CORS headers for that route. The +recommended CORS headers to be returned by servers on all requests are: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + +## Authentication + +Most `v2` endpoints in the Identity Service API require authentication +in order to ensure that the requesting user has accepted all relevant +policies and is otherwise permitted to make the request. The `v1` API +(currently deprecated) does not require this authentication, however +using `v1` is strongly discouraged as it will be removed in a future +release. + +Identity Servers use a scheme similar to the Client-Server API's concept +of access tokens to authenticate users. The access tokens provided by an +Identity Server cannot be used to authenticate Client-Server API +requests. + +An access token is provided to an endpoint in one of two ways: + +1. Via a query string parameter, `access_token=TheTokenHere`. +2. Via a request header, `Authorization: Bearer TheTokenHere`. + +Clients are encouraged to the use the `Authorization` header where +possible to prevent the access token being leaked in access/HTTP logs. +The query string should only be used in cases where the `Authorization` +header is inaccessible for the client. + +When credentials are required but missing or invalid, the HTTP call will +return with a status of 401 and the error code `M_UNAUTHORIZED`. + +{{v2\_auth\_is\_http\_api}} + +## Terms of service + +Identity Servers are encouraged to have terms of service (or similar +policies) to ensure that users have agreed to their data being processed +by the server. To facilitate this, an identity server can respond to +almost any authenticated API endpoint with an HTTP 403 and the error +code `M_TERMS_NOT_SIGNED`. The error code is used to indicate that the +user must accept new terms of service before being able to continue. + +All endpoints which support authentication can return the +`M_TERMS_NOT_SIGNED` error. When clients receive the error, they are +expected to make a call to `GET /terms` to find out what terms the +server offers. The client compares this to the `m.accepted_terms` +account data for the user (described later) and presents the user with +option to accept the still-missing terms of service. After the user has +made their selection, if applicable, the client sends a request to +`POST /terms` to indicate the user's acceptance. The server cannot +expect that the client will send acceptance for all pending terms, and +the client should not expect that the server will not respond with +another `M_TERMS_NOT_SIGNED` on their next request. The terms the user +has just accepted are appended to `m.accepted_terms`. + +{{m\_accepted\_terms\_event}} + +{{v2\_terms\_is\_http\_api}} + +## Status check + +{{ping\_is\_http\_api}} + +{{v2\_ping\_is\_http\_api}} + +## Key management + +An identity server has some long-term public-private keypairs. These are +named in a scheme `algorithm:identifier`, e.g. `ed25519:0`. When signing +an association, the standard [Signing +JSON](/appendices#signing-json) algorithm applies. + +The identity server may also keep track of some short-term +public-private keypairs, which may have different usage and lifetime +characteristics than the service's long-term keys. + +{{pubkey\_is\_http\_api}} + +{{v2\_pubkey\_is\_http\_api}} + +## Association lookup + +{{lookup\_is\_http\_api}} + +{{v2\_lookup\_is\_http\_api}} + +### Client behaviour + +{{% boxes/note %}} +This section only covers the v2 lookup endpoint. The v1 endpoint is +described in isolation above. +{{% /boxes/note %}} + +Prior to performing a lookup clients SHOULD make a request to the +`/hash_details` endpoint to determine what algorithms the server +supports (described in more detail below). The client then uses this +information to form a `/lookup` request and receive known bindings from +the server. + +Clients MUST support at least the `sha256` algorithm. + +### Server behaviour + +{{% boxes/note %}} +This section only covers the v2 lookup endpoint. The v1 endpoint is +described in isolation above. +{{% /boxes/note %}} + +Servers, upon receipt of a `/lookup` request, will compare the query +against known bindings it has, hashing the identifiers it knows about as +needed to verify exact matches to the request. + +Servers MUST support at least the `sha256` algorithm. + +### Algorithms + +Some algorithms are defined as part of the specification, however other +formats can be negotiated between the client and server using +`/hash_details`. + +#### `sha256` + +This algorithm MUST be supported by clients and servers at a minimum. It +is additionally the preferred algorithm for lookups. + +When using this algorithm, the client converts the query first into +strings separated by spaces in the format `
`. +The `` is retrieved from `/hash_details`, the `` is +typically `email` or `msisdn` (both lowercase), and the `
` is +the 3PID to search for. For example, if the client wanted to know about +`alice@example.org`'s bindings, it would first format the query as +`alice@example.org email ThePepperGoesHere`. + +{{% boxes/rationale %}} +Mediums and peppers are appended to the address to prevent a common +prefix for each 3PID, helping prevent attackers from pre-computing the +internal state of the hash function. +{{% /boxes/rationale %}} + +After formatting each query, the string is run through SHA-256 as +defined by [RFC 4634](https://tools.ietf.org/html/rfc4634). The +resulting bytes are then encoded using URL-Safe [Unpadded +Base64](/appendices#unpadded-base64) (similar to [room version +4's event ID format](/rooms/v4#event-ids)). + +An example set of queries when using the pepper `matrixrocks` would be: + + "alice@example.com email matrixrocks" -> "4kenr7N9drpCJ4AfalmlGQVsOn3o2RHjkADUpXJWZUc" + "bob@example.com email matrixrocks" -> "LJwSazmv46n0hlMlsb_iYxI0_HXEqy_yj6Jm636cdT8" + "18005552067 msisdn matrixrocks" -> "nlo35_T5fzSGZzJApqu8lgIudJvmOQtDaHtr-I4rU7I" + +The set of hashes is then given as the `addresses` array in `/lookup`. +Note that the pepper used MUST be supplied as `pepper` in the `/lookup` +request. + +#### `none` + +This algorithm performs plaintext lookups on the identity server. +Typically this algorithm should not be used due to the security concerns +of unhashed identifiers, however some scenarios (such as LDAP-backed +identity servers) prevent the use of hashed identifiers. Identity +servers (and optionally clients) can use this algorithm to perform those +kinds of lookups. + +Similar to the `sha256` algorithm, the client converts the queries into +strings separated by spaces in the format `
` - note +the lack of ``. For example, if the client wanted to know about +`alice@example.org`'s bindings, it would format the query as +`alice@example.org email`. + +The formatted strings are then given as the `addresses` in `/lookup`. +Note that the `pepper` is still required, and must be provided to ensure +the client has made an appropriate request to `/hash_details` first. + +### Security considerations + +{{% boxes/note %}} +[MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) has much +more information about the security considerations made for this section +of the specification. This section covers the high-level details for why +the specification is the way it is. +{{% /boxes/note %}} + +Typically the lookup endpoint is used when a client has an unknown 3PID +it wants to find a Matrix User ID for. Clients normally do this kind of +lookup when inviting new users to a room or searching a user's address +book to find any Matrix users they may not have discovered yet. Rogue or +malicious identity servers could harvest this unknown information and do +nefarious things with it if it were sent in plain text. In order to +protect the privacy of users who might not have a Matrix identifier +bound to their 3PID addresses, the specification attempts to make it +difficult to harvest 3PIDs. + +{{% boxes/rationale %}} +Hashing identifiers, while not perfect, helps make the effort required +to harvest identifiers significantly higher. Phone numbers in particular +are still difficult to protect with hashing, however hashing is +objectively better than not. + +An alternative to hashing would be using bcrypt or similar with many +rounds, however by nature of needing to serve mobile clients and clients +on limited hardware the solution needs be kept relatively lightweight. +{{% /boxes/rationale %}} + +Clients should be cautious of servers not rotating their pepper very +often, and potentially of servers which use a weak pepper - these +servers may be attempting to brute force the identifiers or use rainbow +tables to mine the addresses. Similarly, clients which support the +`none` algorithm should consider at least warning the user of the risks +in sending identifiers in plain text to the identity server. + +Addresses are still potentially reversable using a calculated rainbow +table given some identifiers, such as phone numbers, common email +address domains, and leaked addresses are easily calculated. For +example, phone numbers can have roughly 12 digits to them, making them +an easier target for attack than email addresses. + +## Establishing associations + +The flow for creating an association is session-based. + +Within a session, one may prove that one has ownership of a 3PID. Once +this has been established, the user can form an association between that +3PID and a Matrix user ID. Note that this association is only proved one +way; a user can associate *any* Matrix user ID with a validated 3PID, +i.e. I can claim that any email address I own is associated with +@billg:microsoft.com. + +Sessions are time-limited; a session is considered to have been modified +when it was created, and then when a validation is performed within it. +A session can only be checked for validation, and validation can only be +performed within a session, within a 24-hour period since its most +recent modification. Any attempts to perform these actions after the +expiry will be rejected, and a new session should be created and used +instead. + +To start a session, the client makes a request to the appropriate +`/requestToken` endpoint. The identity server then sends a validation +token to the user, and the user provides the token to the client. The +client then provides the token to the appropriate `/submitToken` +endpoint, completing the session. At this point, the client should +`/bind` the third party identifier or leave it for another entity to +bind. + +### Format of a validation token + +The format of the validation token is left up to the identity server: it +should choose one appropriate to the 3PID type. (For example, it would +be inappropriate to expect a user to copy a long passphrase including +punctuation from an SMS message into a client.) + +Whatever format the identity server uses, the validation token must +consist of at most 255 Unicode codepoints. Clients must pass the token +through without modification. + +### Email associations + +{{email\_associations\_is\_http\_api}} + +{{v2\_email\_associations\_is\_http\_api}} + +### Phone number associations + +{{phone\_associations\_is\_http\_api}} + +{{v2\_phone\_associations\_is\_http\_api}} + +### General + +{{associations\_is\_http\_api}} + +{{v2\_associations\_is\_http\_api}} + +## Invitation storage + +An identity server can store pending invitations to a user's 3PID, which +will be retrieved and can be either notified on or look up when the 3PID +is associated with a Matrix user ID. + +At a later point, if the owner of that particular 3PID binds it with a +Matrix user ID, the identity server will attempt to make an HTTP POST to +the Matrix user's homeserver via the +[/3pid/onbind](/server-server-api#put_matrixfederationv13pidonbind) +endpoint. The request MUST be signed with a long-term private key for +the identity server. + +{{store\_invite\_is\_http\_api}} + +{{v2\_store\_invite\_is\_http\_api}} + +## Ephemeral invitation signing + +To aid clients who may not be able to perform crypto themselves, the +identity server offers some crypto functionality to help in accepting +invitations. This is less secure than the client doing it itself, but +may be useful where this isn't possible. + +{{invitation\_signing\_is\_http\_api}} + +{{v2\_invitation\_signing\_is\_http\_api}} diff --git a/content/proposals.md b/content/proposals.md new file mode 100644 index 00000000..985716f7 --- /dev/null +++ b/content/proposals.md @@ -0,0 +1,481 @@ +--- +title: "Spec Change Proposals" +weight: 60 +type: docs +--- + +If you are interested in submitting a change to the Matrix +Specification, please take note of the following guidelines. + +Most changes to the Specification require a formal proposal. Bug fixes, +typos, and clarifications to existing behaviour do not need proposals - +see the [contributing +guide](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst) +for more information on what does and does not need a proposal. + +The proposal process involves some technical writing, having it reviewed +by everyone, having the proposal being accepted, then actually having +your ideas implemented as committed changes to the [Specification +repository](https://github.com/matrix-org/matrix-doc). + +Meet the [members of the Core Team](https://matrix.org/foundation), a +group of individuals tasked with ensuring the spec process is as smooth +and painless as possible. Members of the Spec Core Team will do their +best to participate in discussion, summarise when things become +long-winded, and generally try to act towards the benefit of everyone. +As a majority, team members have the ability to change the state of a +proposal, and individually have the final say in proposal discussion. + +## Guiding Principles + +Proposals **must** act to the greater benefit of the entire Matrix +ecosystem, rather than benefiting or privileging any single player or +subset of players -and must not contain any patent encumbered +intellectual property. Members of the Core Team pledge to act as a +neutral custodian for Matrix on behalf of the whole ecosystem. + +For clarity: the Matrix ecosystem is anyone who uses the Matrix +protocol. That includes client users, server admins, client developers, +bot developers, bridge and application service developers, users and +admins who are indirectly using Matrix via 3rd party networks which +happen to be bridged, server developers, room moderators and admins, +companies/projects building products or services on Matrix, spec +contributors, translators, and those who created it in the first place. + +"Greater benefit" could include maximising: + +- the number of end-users reachable on the open Matrix network +- the number of regular users on the Matrix network (e.g. 30-day + retained federated users) +- the number of online servers in the open federation +- the number of developers building on Matrix +- the number of independent implementations which use Matrix +- the number of bridged end-users reachable on the open Matrix network +- the signal-to-noise ratio of the content on the open Matrix network + (i.e. minimising spam) +- the ability for users to discover content on their terms (empowering + them to select what to see and what not to see) +- the quality and utility of the Matrix spec (as defined by ease and + ability with which a developer can implement spec-compliant clients, + servers, bots, bridges, and other integrations without needing to + refer to any other external material) + +In addition, proposal authors are expected to uphold the following +values in their proposed changes to the Matrix protocol: + +- Supporting the whole long-term ecosystem rather than individual + stakeholder gain +- Openness rather than proprietary lock-in +- Interoperability rather than fragmentation +- Cross-platform rather than platform-specific +- Collaboration rather than competition +- Accessibility rather than elitism +- Transparency rather than stealth +- Empathy rather than contrariness +- Pragmatism rather than perfection +- Proof rather than conjecture + +Please [see +MSC1779](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1779-open-governance.md) +for full details of the project's Guiding Principles. + +## Technical notes + +Proposals **must** develop Matrix as a layered protocol: with new +features building on layers of shared abstractions rather than +introducing tight vertical coupling within the stack. This ensures that +new features can evolve rapidly by building on existing layers and +swapping out old features without impacting the rest of the stack or +requiring substantial upgrades to the whole ecosystem. This is critical +for Matrix to rapidly evolve and compete effectively with centralised +systems, despite being a federated protocol. + +For instance, new features should be implemented using the highest layer +abstractions possible (e.g. new event types, which layer on top of the +existing room semantics, and so don't even require any API changes). +Failing that, the next recourse would be backwards-compatible changes to +the next layer down (e.g. room APIs); failing that, considering changes +to the format of events or the DAG; etc. It would be a very unusual +feature which doesn't build on the existing infrastructure provided by +the spec and instead created new primitives or low level APIs. + +Backwards compatibility is very important for Matrix, but not at the +expense of hindering the protocol's evolution. Backwards incompatible +changes to endpoints are allowed when no other alternative exists, and +must be versioned under a new major release of the API. Backwards +incompatible changes to the room algorithm are also allowed when no +other alternative exists, and must be versioned under a new version of +the room algorithm. + +There is sometimes a dilemma over where to include higher level +features: for instance, should video conferencing be formalised in the +spec, or should it be implemented via widgets? Should reputation systems +be specified? Should search engine behaviour be specified? + +There is no universal answer to this, but the following guidelines +should be applied: + +1. If the feature would benefit the whole Matrix ecosystem and is + aligned with the guiding principles above, then it should be + supported by the spec. +2. If the spec already makes the feature possible without changing any + of the implementations and spec, then it may not need to be added to + the spec. +3. However, if the best user experience for a feature does require + custom implementation behaviour then the behaviour should be defined + in the spec such that all implementations may implement it. +4. However, the spec must never add dependencies on + unspecified/nonstandardised 3rd party behaviour. + +As a worked example: + +1. Video conferencing is clearly a feature which would benefit the + whole ecosystem, and so the spec should find a way to make it + happen. +2. Video conferencing can be achieved by widgets without requiring any + compulsory changes to clients nor servers to work, and so could be + omitted from the spec. +3. A better experience could be achieved by embedding Jitsi natively + into clients rather than using a widget... +4. ...except that would add a dependency on unspecified/nonstandardised + 3rd party behaviour, so must not be added to the spec. + +Therefore, our two options in the specific case of video conferencing +are either to spec SFU conferencing semantics for WebRTC (or refer to an +existing spec for doing so), or to keep it as a widget-based approach +(optionally with widget extensions specific for more deeply integrating +video conferencing use cases). + +As an alternative example: it's very unlikely that "how to visualise +Magnetic Resonance Imaging data over Matrix" would ever be added to the +Matrix spec (other than perhaps a custom event type in a wider +standardised Matrix event registry) given that the spec's existing +primitives of file transfer and extensible events (MSC1767) give +excellent tools for transferring and visualising arbitrary rich data. + +Supporting public search engines are likely to not require custom spec +features (other than possibly better bulk access APIs), given they can +be implemented as clients using the existing CS API. An exception could +be API features required by decentralised search infrastructure +(avoiding centralisation of power by a centralised search engine). + +Features such as reactions, threaded messages, editable messages, +spam/abuse/content filtering (and reputation systems), are all features +which would clearly benefit the whole Matrix ecosystem, and cannot be +implemented in an interoperable way using the current spec; so they +necessitate a spec change. + +## Process + +The process for submitting a Matrix Spec Change (MSC) Proposal in detail +is as follows: + +- Create a first draft of your proposal using [GitHub-flavored + Markdown](https://help.github.com/articles/basic-writing-and-formatting-syntax/) + - In the document, clearly state the problem being solved, and the + possible solutions being proposed for solving it and their + respective trade-offs. + - Proposal documents are intended to be as lightweight and + flexible as the author desires; there is no formal template; the + intention is to iterate as quickly as possible to get to a good + design. + - However, a [template with suggested + headers](https://github.com/matrix-org/matrix-doc/blob/master/proposals/0000-proposal-template.md) + is available to get you started if necessary. + - Take care in creating your proposal. Specify your intended + changes, and give reasoning to back them up. Changes without + justification will likely be poorly received by the community. +- Fork and make a PR to the + [matrix-doc](https://github.com/matrix-org/matrix-doc) repository. + The ID of your PR will become the MSC ID for the lifetime of your + proposal. + - The proposal must live in the `proposals/` directory with a + filename that follows the format `1234-my-new-proposal.md` where + `1234` is the MSC ID. + - Your PR description must include a link to the rendered Markdown + document and a summary of the proposal. + - It is often very helpful to link any related MSCs or [matrix-doc + issues](https://github.com/matrix-org/matrix-doc/issues) to give + context for the proposal. + - Additionally, please be sure to sign off your proposal PR as per + the guidelines listed on + [CONTRIBUTING.rst](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst). +- Gather feedback as widely as possible. + - The aim is to get maximum consensus towards an optimal solution. + Sometimes trade-offs are required to meet this goal. Decisions + should be made to the benefit of all major use cases. + - A good place to ask for feedback on a specific proposal is + [\#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). + If preferred, an alternative room can be created and advertised + in \#matrix-spec:matrix.org. Please also link to the room in + your PR description. + - For additional discussion areas, know that + \#matrix-dev:matrix.org is for developers using existing Matrix + APIs, \#matrix:matrix.org is for users trying to run Matrix apps + (clients & servers) and \#matrix-architecture:matrix.org is for + cross-cutting discussion of Matrix's architectural design. + - The point of the spec proposal process is to be collaborative + rather than competitive, and to try to solve the problem in + question with the optimal set of trade-offs. The author should + neutrally gather the various viewpoints and get consensus, but + this can sometimes be time-consuming (or the author may be + biased), in which case an impartial 'shepherd' can be assigned + to help guide the proposal through this process instead. A + shepherd is typically a neutral party from the Spec Core Team or + an experienced member of the community. There is no formal + process for assignment. Simply ask for a shepherd to help get + your proposal through and one will be assigned based on + availability. Having a shepherd is not a requirement for + proposal acceptance. +- Members of the Spec Core Team and community will review and discuss + the PR in the comments and in relevant rooms on Matrix. Discussion + outside of GitHub should be summarised in a comment on the PR. +- When a member of the Spec Core Team believes that no new discussion + points are being made, and the proposal has suitable evidence of + working (see [implementing a proposal](#implementing-a-proposal) + below), they will propose a motion for a final comment period (FCP), + along with a *disposition* of either merge, close or postpone. This + FCP is provided to allow a short period of time for any invested + party to provide a final objection before a major decision is made. + If sufficient reasoning is given, an FCP can be cancelled. It is + often preceded by a comment summarising the current state of the + discussion, along with reasoning for its occurrence. +- A concern can be raised by a Spec Core Team member at any time, + which will block an FCP from beginning. An FCP will only begin when + 75% of the members of the Spec Core Team agree on its outcome, and + all existing concerns have been resolved. +- The FCP will then begin and last for 5 days, giving anyone else some + time to speak up before it concludes. On its conclusion, the + disposition of the FCP will be carried out. If sufficient reasoning + against the disposition is raised, the FCP can be cancelled and the + MSC will continue to evolve accordingly. +- Once the proposal has been accepted and merged, it is time to submit + the actual change to the Specification that your proposal reasoned + about. This is known as a spec PR. However in order for the spec PR + to be accepted, an implementation **must** be shown to prove that it + works well in practice. A link to the implementation should be + included in the PR description. In addition, any significant + unforeseen changes to the original idea found during this process + will warrant another MSC. Any minor, non-fundamental changes are + allowed but **must** be documented in the original proposal + document. This ensures that someone reading a proposal in the future + doesn't assume old information wasn't merged into the spec. + - Similar to the proposal PR, please sign off the spec PR as per + the guidelines on + [CONTRIBUTING.rst](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst). +- Your PR will then be reviewed and hopefully merged on the grounds it + is implemented sufficiently. If so, then give yourself a pat on the + back knowing you've contributed to the Matrix protocol for the + benefit of users and developers alike :) + +The process for handling proposals is shown visually in the following +diagram. Note that the lifetime of a proposal is tracked through the +corresponding labels for each stage on the +[matrix-doc](https://github.com/matrix-org/matrix-doc) issue and pull +request trackers. + +``` + + + + Proposals | Spec PRs | Additional States + +-------+ | +------+ | +---------------+ + | | + +----------------------+ | +---------+ | +-----------+ + | | | | | | | | + | Proposal | | +------= Spec PR | | | Postponed | + | Drafting and Initial | | | | Missing | | | | + | Feedback Gathering | | | | | | +-----------+ + | | | | +----+----+ | + +----------+-----------+ | | | | +----------+ + | | | v | | | + v | | +-----------------+ | | Closed | + +-------------------+ | | | | | | | + | | | | | Spec PR Created | | +----------+ + | Proposal PR | | | | and In Review | | + | In Review | | | | | | + | | | | +--------+--------+ | + +---------+---------+ | | | | + | | | v | + v | | +-----------+ | + +----------------------+ | | | | | + | | | | | Spec PR | | + | Proposed Final | | | | Merged! | | + | Comment Period | | | | | | + | | | | +-----------+ | + +----------+-----------+ | | | + | | | | + v | | | + +----------------------+ | | | + | | | | | + | Final Comment Period | | | | + | | | | | + +----------+-----------+ | | | + | | | | + v | | | + +----------------------+ | | | + | | | | | + | Final Comment Period | | | | + | Complete | | | | + | | | | | + +----------+-----------+ | | | + | | | | + +-----------------+ | + | | + + + +``` + +## Lifetime States + +**Note:** All labels are to be placed on the proposal PR. + +| Name | GitHub Label | Description | +|---------------------------------|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Proposal Drafting and Feedback | [No label](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+-label%3Aabandoned+-label%3Afinal-comment-period+-label%3Afinished-final-comment-period+-label%3Amerged+-label%3Aobsolete+-label%3Aproposal-postponed+-label%3Aproposed-final-comment-period+-label%3Aproposal-in-review+-label%3Aspec-pr-in-review+-label%3Aspec-pr-missing) | A proposal document which is still work-in-progress but is being shared to incorporate feedback. Please prefix your proposal's title with `[WIP]` to make it easier for reviewers to skim their notifications list. | +| Proposal In Review | [proposal-in-review](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aproposal-in-review) | A proposal document which is now ready and waiting for review by the Spec Core Team and community | +| Proposed Final Comment Period | [proposed-final-comment-period](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aproposed-final-comment-period+) | Currently awaiting signoff of a 75% majority of team members in order to enter the final comment period | +| Final Comment Period | [final-comment-period](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Afinal-comment-period+) | A proposal document which has reached final comment period either for merge, closure or postponement | +| Final Comment Period Complete | [finished-final-comment-period](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Afinished-final-comment-period+) | The final comment period has been completed. Waiting for a demonstration implementation | +| Spec PR Missing | [spec-pr-missing](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aspec-pr-missing) | The proposal has been agreed, and proven with a demonstration implementation. Waiting for a PR against the Spec | +| Spec PR In Review | [spec-pr-in-review](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aspec-pr-in-review+) | The spec PR has been written, and is currently under review | +| Spec PR Merged | [merged](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Amerged) | A proposal with a sufficient working implementation and whose Spec PR has been merged! | +| Postponed | [proposal-postponed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aproposal-postponed+) | A proposal that is temporarily blocked or a feature that may not be useful currently but perhaps sometime in the future | +| Abandoned | [proposal-closed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aabandoned) | A proposal where the author/shepherd is not responsive | +| Obsolete | [obsolete](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aobsolete+) | A proposal which has been made obsolete by another proposal or decision elsewhere. | + +## Categories + +We use category labels on MSCs to place them into a track of work. The +Spec Core Team decides which of the tracks they are focusing on for the +next while and generally makes an effort to pull MSCs out of that +category when possible. + +The current categories are: + +| Name | GitHub Label | Description | +|-------------|------------------|---------------------------------------| +| Core | kind:core | Important for the protocol's success. | +| Feature | kind:feature | Nice to have additions to the spec. | +| Maintenance | kind:maintenance | Fixes or clarifies existing spec. | + +Some examples of core MSCs would be aggregations, cross-signing, and +groups/communities. These are the sorts of things that if not +implemented could cause the protocol to fail or become second-class. +Features would be areas like enhanced media APIs, new transports, and +bookmarks in comparison. Finally, maintenance MSCs would include +improving error codes, clarifying what is required of an API, and adding +properties to an API which makes it easier to use. + +The Spec Core Team assigns a category to each MSC based on the +descriptions above. This can mean that new MSCs get categorized into an +area the team isn't focused on, though that can always change as +priorities evolve. We still encourage that MSCs be opened, even if not +the focus for the time being, as they can still make progress and even +be merged without the Spec Core Team focusing on them specifically. + +## Implementing a proposal + +As part of the proposal process the spec core team will require evidence +of the MSC working in order for it to move into FCP. This can usually be +a branch/pull request to whichever implementation of choice that proves +the MSC works in practice, though in some cases the MSC itself will be +small enough to be considered proven. Where it's unclear if an MSC will +require an implementation proof, ask in +[\#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). + +### Early release of an MSC/idea + +To help facilitate early releases of software dependent on a spec +release, implementations are required to use the following process to +ensure that the official Matrix namespace is not cluttered with +development or testing data. + +Note + +Unreleased implementations (including proofs-of-concept demonstrating +that a particular MSC works) do not have to follow this process. + +1. Have an idea for a feature. +2. Implement the feature using unstable endpoints, vendor prefixes, and + unstable feature flags as appropriate. + - When using unstable endpoints, they MUST include a vendor + prefix. For example: + `/_matrix/client/unstable/com.example/login`. Vendor prefixes + throughout Matrix always use the Java package naming convention. + The MSC for the feature should identify which preferred vendor + prefix is to be used by early adopters. + - Note that unstable namespaces do not automatically inherit + endpoints from stable namespaces: for example, the fact that + `/_matrix/client/r0/sync` exists does not imply that + `/_matrix/client/unstable/com.example/sync` exists. + - If the client needs to be sure the server supports the feature, + an unstable feature flag that MUST be vendor prefixed is to be + used. This kind of flag shows up in the `unstable_features` + section of `/versions` as, for example, `com.example.new_login`. + The MSC for the feature should identify which preferred feature + flag is to be used by early adopters. + - When using this approach correctly, the implementation can + ship/release the feature at any time, so long as the + implementation is able to accept the technical debt that results + from needing to provide adequate backwards and forwards + compatibility. The implementation MUST support the flag (and + server-side implementation) disappearing and be generally safe + for users. Note that implementations early in the MSC review + process may also be required to provide backwards compatibility + with earlier editions of the proposal. + - If the implementation cannot support the technical debt (or if + it's impossible to provide forwards/backwards compatibility - + e.g. a user authentication change which can't be safely rolled + back), the implementation should not attempt to implement the + feature and should instead wait for a spec release. + - If at any point after early release, the idea changes in a + backwards-incompatible way, the feature flag should also change + so that implementations can adapt as needed. +3. In parallel, or ahead of implementation, open an MSC and solicit + review per above. +4. Before FCP can be called, the Spec Core Team will require evidence + of the MSC working as proposed. A typical example of this is an + implementation of the MSC, though the implementation does not need + to be shipped anywhere and can therefore avoid the + forwards/backwards compatibility concerns mentioned here. +5. The FCP process is completed, and assuming nothing is flagged the + MSC lands. +6. A spec PR is written to incorporate the changes into Matrix. +7. A spec release happens. +8. Implementations switch to using stable prefixes (e.g.: `/r0`) if the + server supports the specification version released. If the server + doesn't advertise the specification version, but does have the + feature flag, unstable prefixes should still be used. +9. A transition period of about 2 months starts immediately after the + spec release, before implementations start to encourage other + implementations to switch to stable endpoints. For example, a server + implementation should start asking client implementations to support + the stable endpoints 2 months after the spec release, if they + haven't already. The same applies in the reverse: if clients cannot + switch to stable prefixes because server implementations haven't + started supporting the new spec release, some noise should be raised + in the general direction of the implementation. + +{{% boxes/note %}} +MSCs MUST still describe what the stable endpoints/feature looks like +with a note towards the bottom for what the unstable feature +flag/prefixes are. For example, an MSC would propose `/_matrix/client/r0/new/endpoint`, not `/_matrix/client/unstable/ +com.example/new/endpoint`. +{{% /boxes/note %}} + +In summary: + +- Implementations MUST NOT use stable endpoints before the MSC is in + the spec. This includes NOT using stable endpoints in the period + between completion of FCP and release of the spec. passed. +- Implementations are able to ship features that are exposed to users + by default before an MSC has been merged to the spec, provided they + follow the process above. +- Implementations SHOULD be wary of the technical debt they are + incurring by moving faster than the spec. +- The vendor prefix is chosen by the developer of the feature, using + the Java package naming convention. The foundation's preferred + vendor prefix is `org.matrix`. +- The vendor prefixes, unstable feature flags, and unstable endpoints + should be included in the MSC, though the MSC MUST be written in a + way that proposes new stable endpoints. Typically this is solved by + a small table at the bottom mapping the various values from stable + to unstable. diff --git a/content/push-gateway-api.md b/content/push-gateway-api.md new file mode 100644 index 00000000..a6a3d1a3 --- /dev/null +++ b/content/push-gateway-api.md @@ -0,0 +1,57 @@ +--- +title: "Push Gateway API" +weight: 50 +type: docs +--- + +Clients may want to receive push notifications when events are received +at the homeserver. This is managed by a distinct entity called the Push +Gateway. + +## Overview + +A client's homeserver forwards information about received events to the +push gateway. The gateway then submits a push notification to the push +notification provider (e.g. APNS, GCM). + +``` + +--------------------+ +-------------------+ + Matrix HTTP | | | | + Notification Protocol | App Developer | | Device Vendor | + | | | | + +-------------------+ | +----------------+ | | +---------------+ | + | | | | | | | | | | + | Matrix homeserver +-----> Push Gateway +------> Push Provider | | + | | | | | | | | | | + +-^-----------------+ | +----------------+ | | +----+----------+ | + | | | | | | + Matrix | | | | | | + Client/Server API + | | | | | + | | +--------------------+ +-------------------+ + | +--+-+ | + | | <-------------------------------------------+ + +---+ | + | | Provider Push Protocol + +----+ + + Mobile Device or Client +``` + +## Homeserver behaviour + +This describes the format used by "HTTP" pushers to send notifications +of events to Push Gateways. If the endpoint returns an HTTP error code, +the homeserver SHOULD retry for a reasonable amount of time using +exponential backoff. + +When pushing notifications for events, the homeserver is expected to +include all of the event-related fields in the `/notify` request. When +the homeserver is performing a push where the `format` is +`"event_id_only"`, only the `event_id`, `room_id`, `counts`, and +`devices` are required to be populated. + +Note that most of the values and behaviour of this endpoint is described +by the Client-Server API's [Push +Module](/client-server-api#push-notifications). + +{{push\_notifier\_push\_http\_api}} diff --git a/content/rooms/_index.md b/content/rooms/_index.md new file mode 100644 index 00000000..92e54bfd --- /dev/null +++ b/content/rooms/_index.md @@ -0,0 +1,12 @@ +--- +title: Room Versions +type: docs +weight: 60 +--- + +* [Room Version 1](v1) +* [Room Version 2](v2) +* [Room Version 3](v3) +* [Room Version 4](v4) +* [Room Version 5](v5) +* [Room Version 6](v6) diff --git a/content/rooms/v1.md b/content/rooms/v1.md new file mode 100644 index 00000000..450eea6d --- /dev/null +++ b/content/rooms/v1.md @@ -0,0 +1,285 @@ +--- +title: Room Version 1 +type: docs +weight: 10 +--- + +This room version is the first ever version for rooms, and contains the +building blocks for other room versions. + +## Client considerations + +Clients may need to consider some algorithms performed by the server for +their own implementation. + +### Redactions + +Upon receipt of a redaction event, the server must strip off any keys +not in the following list: + +- `event_id` +- `type` +- `room_id` +- `sender` +- `state_key` +- `content` +- `hashes` +- `signatures` +- `depth` +- `prev_events` +- `prev_state` +- `auth_events` +- `origin` +- `origin_server_ts` +- `membership` + +The content object must also be stripped of all keys, unless it is one +of one of the following event types: + +- `m.room.member` allows key `membership`. +- `m.room.create` allows key `creator`. +- `m.room.join_rules` allows key `join_rule`. +- `m.room.power_levels` allows keys `ban`, `events`, `events_default`, + `kick`, `redact`, `state_default`, `users`, `users_default`. +- `m.room.aliases` allows key `aliases`. +- `m.room.history_visibility` allows key `history_visibility`. + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. +{{% /boxes/warning %}} + +The algorithms defined here should only apply to version 1 rooms. Other +algorithms may be used by other room versions, and as such servers +should be aware of which version room they are dealing with prior to +executing a given algorithm. + +{{% boxes/warning %}} +Although there are many rooms using room version 1, it is known to have +undesirable effects. Servers implementing support for room version 1 +should be aware that restrictions should be generally relaxed and that +inconsistencies may occur. +{{% /boxes/warning %}} + +### State resolution + +{{% boxes/warning %}} +Room version 1 is known to have bugs that can cause the state of rooms +to reset to older versions of the room's state. For example this could +mean that users who had joined the room may be removed from the room, +admins and moderators could lose their power level, and users who have +been banned from the room may be able to rejoin. Other state events such +as the the room's name or topic could also reset to a previous version. + +This is fixed in the state resolution algorithm introduced in room +version 2. +{{% /boxes/warning %}} + +The room state *S*′(*E*) after an event *E* is defined in terms of the +room state *S*(*E*) before *E*, and depends on whether *E* is a state +event or a message event: + +- If *E* is a message event, then *S*′(*E*) = *S*(*E*). +- If *E* is a state event, then *S*′(*E*) is *S*(*E*), except that its + entry corresponding to *E*'s `event_type` and `state_key` is + replaced by *E*'s `event_id`. + +The room state *S*(*E*) before *E* is the *resolution* of the set of +states {*S*′(*E*′), *S*′(*E*″), …} consisting of the states after each +of *E*'s `prev_event`s {*E*′, *E*″, …}. + +The *resolution* of a set of states is defined as follows. The resolved +state is built up in a number of passes; here we use *R* to refer to the +results of the resolution so far. + +- Start by setting *R* to the union of the states to be resolved, + excluding any *conflicting* events. +- First we resolve conflicts between `m.room.power_levels` events. If + there is no conflict, this step is skipped, otherwise: + - Assemble all the `m.room.power_levels` events from the states to + be resolved into a list. + - Sort the list by ascending `depth` then descending + `sha1(event_id)`. + - Add the first event in the list to *R*. + - For each subsequent event in the list, check that the event + would be allowed by the authorization rules for a room in state + *R*. If the event would be allowed, then update *R* with the + event and continue with the next event in the list. If it would + not be allowed, stop and continue below with `m.room.join_rules` + events. +- Repeat the above process for conflicts between `m.room.join_rules` + events. +- Repeat the above process for conflicts between `m.room.member` + events. +- No other events affect the authorization rules, so for all other + conflicts, just pick the event with the highest depth and lowest + `sha1(event_id)` that passes authentication in *R* and add it to + *R*. + +A *conflict* occurs between states where those states have different +`event_ids` for the same `(event_type, state_key)`. The events thus +affected are said to be *conflicting* events. + +### Authorization rules + +The types of state events that affect authorization are: + +- `m.room.create` +- `m.room.member` +- `m.room.join_rules` +- `m.room.power_levels` +- `m.room.third_party_invite` + +{{% boxes/note %}} +Power levels are inferred from defaults when not explicitly supplied. +For example, mentions of the `sender`'s power level can also refer to +the default power level for users in the room. +{{% /boxes/note %}} + +The rules are as follows: + +1. If type is `m.room.create`: + 1. If it has any previous events, reject. + 2. If the domain of the `room_id` does not match the domain of the + `sender`, reject. + 3. If `content.room_version` is present and is not a recognised + version, reject. + 4. If `content` has no `creator` field, reject. + 5. Otherwise, allow. +2. Reject if event has `auth_events` that: + 1. have duplicate entries for a given `type` and `state_key` pair + 2. have entries whose `type` and `state_key` don't match those + specified by the [auth events + selection](/server-server-api#auth-events-selection) + algorithm described in the server specification. +3. If event does not have a `m.room.create` in its `auth_events`, + reject. +4. If type is `m.room.aliases`: + 1. If event has no `state_key`, reject. + 2. If sender's domain doesn't matches `state_key`, reject. + 3. Otherwise, allow. +5. If type is `m.room.member`: + 1. If no `state_key` key or `membership` key in `content`, reject. + 2. If `membership` is `join`: + 1. If the only previous event is an `m.room.create` and the + `state_key` is the creator, allow. + 2. If the `sender` does not match `state_key`, reject. + 3. If the `sender` is banned, reject. + 4. If the `join_rule` is `invite` then allow if membership + state is `invite` or `join`. + 5. If the `join_rule` is `public`, allow. + 6. Otherwise, reject. + 3. If `membership` is `invite`: + 1. If `content` has `third_party_invite` key: + 1. If *target user* is banned, reject. + 2. If `content.third_party_invite` does not have a `signed` + key, reject. + 3. If `signed` does not have `mxid` and `token` keys, + reject. + 4. If `mxid` does not match `state_key`, reject. + 5. If there is no `m.room.third_party_invite` event in the + current room state with `state_key` matching `token`, + reject. + 6. If `sender` does not match `sender` of the + `m.room.third_party_invite`, reject. + 7. If any signature in `signed` matches any public key in + the `m.room.third_party_invite` event, allow. The public + keys are in `content` of `m.room.third_party_invite` as: + 1. A single public key in the `public_key` field. + 2. A list of public keys in the `public_keys` field. + 8. Otherwise, reject. + 2. If the `sender`'s current membership state is not `join`, + reject. + 3. If *target user*'s current membership state is `join` or + `ban`, reject. + 4. If the `sender`'s power level is greater than or equal to + the *invite level*, allow. + 5. Otherwise, reject. + 4. If `membership` is `leave`: + 1. If the `sender` matches `state_key`, allow if and only if + that user's current membership state is `invite` or `join`. + 2. If the `sender`'s current membership state is not `join`, + reject. + 3. If the *target user*'s current membership state is `ban`, + and the `sender`'s power level is less than the *ban level*, + reject. + 4. If the `sender`'s power level is greater than or equal to + the *kick level*, and the *target user*'s power level is + less than the `sender`'s power level, allow. + 5. Otherwise, reject. + 5. If `membership` is `ban`: + 1. If the `sender`'s current membership state is not `join`, + reject. + 2. If the `sender`'s power level is greater than or equal to + the *ban level*, and the *target user*'s power level is less + than the `sender`'s power level, allow. + 3. Otherwise, reject. + 6. Otherwise, the membership is unknown. Reject. +6. If the `sender`'s current membership state is not `join`, reject. +7. If type is `m.room.third_party_invite`: + 1. Allow if and only if `sender`'s current power level is greater + than or equal to the *invite level*. +8. If the event type's *required power level* is greater than the + `sender`'s power level, reject. +9. If the event has a `state_key` that starts with an `@` and does not + match the `sender`, reject. +10. If type is `m.room.power_levels`: + 1. If `users` key in `content` is not a dictionary with keys that + are valid user IDs with values that are integers (or a string + that is an integer), reject. + 2. If there is no previous `m.room.power_levels` event in the room, + allow. + 3. For the keys `users_default`, `events_default`, `state_default`, + `ban`, `redact`, `kick`, `invite` check if they were added, + changed or removed. For each found alteration: + 1. If the current value is higher than the `sender`'s current + power level, reject. + 2. If the new value is higher than the `sender`'s current power + level, reject. + 4. For each entry being added, changed or removed in both the + `events` and `users` keys: + 1. If the current value is higher than the `sender`'s current + power level, reject. + 2. If the new value is higher than the `sender`'s current power + level, reject. + 5. For each entry being changed under the `users` key, other than + the `sender`'s own entry: + 1. If the current value is equal to the `sender`'s current + power level, reject. + 6. Otherwise, allow. +11. If type is `m.room.redaction`: + 1. If the `sender`'s power level is greater than or equal to the + *redact level*, allow. + 2. If the domain of the `event_id` of the event being redacted is + the same as the domain of the `event_id` of the + `m.room.redaction`, allow. + 3. Otherwise, reject. +12. Otherwise, allow. + +{{% boxes/note %}} +Some consequences of these rules: + +- Unless you are a member of the room, the only permitted operations + (apart from the initial create/join) are: joining a public room; + accepting or rejecting an invitation to a room. +- To unban somebody, you must have power level greater than or equal + to both the kick *and* ban levels, *and* greater than the target + user's power level. +{{% /boxes/note %}} + +### Event format + +Events in version 1 rooms have the following structure: + +{{definition\_ss\_pdu}} + +### Canonical JSON + +Servers MUST NOT strictly enforce the JSON format specified in the +[appendices](/appendices#canonical-json) for the reasons +described there. diff --git a/content/rooms/v2.md b/content/rooms/v2.md new file mode 100644 index 00000000..f087f00e --- /dev/null +++ b/content/rooms/v2.md @@ -0,0 +1,188 @@ +--- +title: Room Version 2 +type: docs +weight: 20 +--- + +This room version builds off of [version 1](/rooms/v1) with an improved +state resolution algorithm. + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the details contained here, and can safely ignore their +presence. +{{% /boxes/warning %}} + +Room version 2 uses the base components of [room version 1](/rooms/v1), +changing only the state resolution algorithm. + +### State resolution + +The room state *S*′(*E*) after an event *E* is defined in terms of the +room state *S*(*E*) before *E*, and depends on whether *E* is a state +event or a message event: + +- If *E* is a message event, then *S*′(*E*) = *S*(*E*). +- If *E* is a state event, then *S*′(*E*) is *S*(*E*), except that its + entry corresponding to *E*'s `event_type` and `state_key` is + replaced by *E*'s `event_id`. + +The room state *S*(*E*) before *E* is the *resolution* of the set of +states {*S*′(*E*1), *S*′(*E*2), …} consisting of +the states after each of *E*'s `prev_event`s +{*E*1, *E*2, …}, where the resolution of a set of +states is given in the algorithm below. + +#### Definitions + +The state resolution algorithm for version 2 rooms uses the following +definitions, given the set of room states +{*S*1, *S*2, …}: + +Power events +A *power event* is a state event with type `m.room.power_levels` or +`m.room.join_rules`, or a state event with type `m.room.member` where +the `membership` is `leave` or `ban` and the `sender` does not match the +`state_key`. The idea behind this is that power events are events that +might remove someone's ability to do something in the room. + +Unconflicted state map and conflicted state set +The *unconflicted state map* is the state where the value of each key +exists and is the same in each state *S**i*. The *conflicted +state set* is the set of all other state events. Note that the +unconflicted state map only has one event per `(event_type, state_key)`, +whereas the conflicted state set may have multiple events. + +Auth difference +The *auth difference* is calculated by first calculating the full auth +chain for each state *S**i*, that is the union of the auth +chains for each event in *S**i*, and then taking every event +that doesn't appear in every auth chain. If *C**i* is the +full auth chain of *S**i*, then the auth difference is + ∪ *C**i* −  ∩ *C**i*. + +Full conflicted set +The *full conflicted set* is the union of the conflicted state set and +the auth difference. + +Reverse topological power ordering +The *reverse topological power ordering* of a set of events is the +lexicographically smallest topological ordering based on the DAG formed +by auth events. The reverse topological power ordering is ordered from +earliest event to latest. For comparing two topological orderings to +determine which is the lexicographically smallest, the following +comparison relation on events is used: for events *x* and *y*, +*x* < *y* if + +1. *x*'s sender has *greater* power level than *y*'s sender, when + looking at their respective `auth_event`s; or +2. the senders have the same power level, but *x*'s `origin_server_ts` + is *less* than *y*'s `origin_server_ts`; or +3. the senders have the same power level and the events have the same + `origin_server_ts`, but *x*'s `event_id` is *less* than *y*'s + `event_id`. + +The reverse topological power ordering can be found by sorting the +events using Kahn's algorithm for topological sorting, and at each step +selecting, among all the candidate vertices, the smallest vertex using +the above comparison relation. + +Mainline ordering +Given an `m.room.power_levels` event *P*, the *mainline of* *P* is the +list of events generated by starting with *P* and recursively taking the +`m.room.power_levels` events from the `auth_events`, ordered such that +*P* is last. Given another event *e*, the *closest mainline event to* +*e* is the first event encountered in the mainline when iteratively +descending through the `m.room.power_levels` events in the `auth_events` +starting at *e*. If no mainline event is encountered when iteratively +descending through the `m.room.power_levels` events, then the closest +mainline event to *e* can be considered to be a dummy event that is +before any other event in the mainline of *P* for the purposes of +condition 1 below. + +The *mainline ordering based on* *P* of a set of events is the ordering, +from smallest to largest, using the following comparison relation on +events: for events *x* and *y*, *x* < *y* if + +1. the closest mainline event to *x* appears *before* the closest + mainline event to *y*; or +2. the closest mainline events are the same, but *x*'s + `origin_server_ts` is *less* than *y*'s `origin_server_ts`; or +3. the closest mainline events are the same and the events have the + same `origin_server_ts`, but *x*'s `event_id` is *less* than *y*'s + `event_id`. + +Iterative auth checks +The *iterative auth checks algorithm* takes as input an initial room +state and a sorted list of state events, and constructs a new room state +by iterating through the event list and applying the state event to the +room state if the state event is allowed by the [authorization +rules](/server-server-api#authorization-rules). +If the state event is not allowed by the authorization rules, then the +event is ignored. If a `(event_type, state_key)` key that is required +for checking the authorization rules is not present in the state, then +the appropriate state event from the event's `auth_events` is used if +the auth event is not rejected. + +#### Algorithm + +The *resolution* of a set of states is obtained as follows: + +1. Take all *power events* and any events in their auth chains, + recursively, that appear in the *full conflicted set* and order them + by the *reverse topological power ordering*. +2. Apply the *iterative auth checks algorithm*, starting from the + *unconflicted state map*, to the list of events from the previous + step to get a partially resolved state. +3. Take all remaining events that weren't picked in step 1 and order + them by the mainline ordering based on the power level in the + partially resolved state obtained in step 2. +4. Apply the *iterative auth checks algorithm* on the partial resolved + state and the list of events from the previous step. +5. Update the result by replacing any event with the event with the + same key from the *unconflicted state map*, if such an event exists, + to get the final resolved state. + +#### Rejected events + +Events that have been rejected due to failing auth based on the state at +the event (rather than based on their auth chain) are handled as usual +by the algorithm, unless otherwise specified. + +Note that no events rejected due to failure to auth against their auth +chain should appear in the process, as they should not appear in state +(the algorithm only uses events that appear in either the state sets or +in the auth chain of the events in the state sets). + +{{% boxes/rationale %}} +This helps ensure that different servers' view of state is more likely +to converge, since rejection state of an event may be different. This +can happen if a third server gives an incorrect version of the state +when a server joins a room via it (either due to being faulty or +malicious). Convergence of state is a desirable property as it ensures +that all users in the room have a (mostly) consistent view of the state +of the room. If the view of the state on different servers diverges it +can lead to bifurcation of the room due to e.g. servers disagreeing on +who is in the room. + +Intuitively, using rejected events feels dangerous, however: + +1. Servers cannot arbitrarily make up state, since they still need to + pass the auth checks based on the event's auth chain (e.g. they + can't grant themselves power levels if they didn't have them + before). +2. For a previously rejected event to pass auth there must be a set of + state that allows said event. A malicious server could therefore + produce a fork where it claims the state is that particular set of + state, duplicate the rejected event to point to that fork, and send + the event. The duplicated event would then pass the auth checks. + Ignoring rejected events would therefore not eliminate any potential + attack vectors. +{{% /boxes/rationale %}} + +Rejected auth events are deliberately excluded from use in the iterative +auth checks, as auth events aren't re-authed (although non-auth events +are) during the iterative auth checks. diff --git a/content/rooms/v3.md b/content/rooms/v3.md new file mode 100644 index 00000000..71d64c1a --- /dev/null +++ b/content/rooms/v3.md @@ -0,0 +1,100 @@ +--- +title: Room Version 3 +type: docs +weight: 30 +--- + +This room version builds on [version 2](/rooms/v2) with an improved event +format. + +## Client considerations + +This room version changes the format for event IDs sent to clients. +Clients should be aware that these event IDs may contain slashes and +other potentially problematic characters. Clients should be treating +event IDs as opaque identifiers and should not be attempting to parse +them into a usable form, just like with other room versions. + +Clients should expect to see event IDs changed from the format of +`$randomstring:example.org` to something like +`$acR1l0raoZnm60CBwAVgqbZqoO/mYU81xysh1u7XcJk` (note the lack of domain +and the potentially problematic slash). + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. +{{% /boxes/warning %}} + +Room version 3 uses the state resolution algorithm defined in [room +version 2](/rooms/v2), and the event format defined here. + +### Event IDs + +{{% boxes/rationale %}} +In other room versions (namely version 1 and 2) the event ID is a +distinct field from the remainder of the event, which must be tracked as +such. This leads to complications where servers receive multiple events +with the same ID in either the same or different rooms where the server +cannot easily keep track of which event it should be using. By removing +the use of a dedicated event ID, servers are required to track the +hashes on an event to determine its ID. +{{% /boxes/rationale %}} + +The event ID is the [reference +hash](/server-server-api#calculating-the-reference-hash-for-an-event) of +the event encoded using [Unpadded +Base64](/appendices#unpadded-base64), prefixed with `$`. A +resulting event ID using this approach should look similar to +`$CD66HAED5npg6074c6pDtLKalHjVfYb2q4Q3LZgrW6o`. + +Event IDs should not be sent over federation to servers when the room +uses this room version. On the receiving end of an event, the server +should compute the relevant event ID for itself. + +Additionally, the `auth_events` and `prev_events` have had a format +change compared to other room versions to make it easier to handle. +Instead of a tuple of values, they are now plain lists of events. + +{{definition\_ss\_pdu\_v3}} + +### Changes to APIs + +Due to the event ID being removed from the event, some APIs need to +change. All APIs which currently accept an event ID must do so with the +new format. Servers must append the calculated event ID to all events +sent to clients where an event ID would normally be expected. + +Because the format of events has changed, servers must be aware of the +room version where the event resides so that the server may parse and +handle the event. The federation API has taken this concern into +consideration by ensuring that servers are aware of (or can find) the +room version during a request. + +### Authorization rules for events + +The authorization rules for a given event have changed in this room +version due to the change in event format: + +- The event no longer needs to be signed by the domain of the event ID + (as there is no domain in the event ID), but still needs to be + signed by the sender's domain. +- In past room versions, redactions were only permitted to enter the + DAG if the sender's domain matched the domain in the event ID being + redacted, or the sender had appropriate permissions per the power + levels. Due to servers now not being able to determine where an + event came from during event authorization, redaction events are + always accepted (provided the event is allowed by `events` and + `events_default` in the power levels). However, servers should not + apply or send redactions to clients until both the redaction event + and original event have been seen, and are valid. Servers should + only apply redactions to events where the sender's domains match, or + the sender of the redaction has the appropriate permissions per the + power levels. + +The remaining rules are the same as [room version +1](/rooms/v1#authorization-rules). diff --git a/content/rooms/v4.md b/content/rooms/v4.md new file mode 100644 index 00000000..d2aa8a23 --- /dev/null +++ b/content/rooms/v4.md @@ -0,0 +1,61 @@ +--- +title: Room Version 4 +type: docs +weight: 40 +--- + +This room version builds on [version 3](/rooms/v3) using a different +encoding for event IDs. + +## Client considerations + +This room version changes the format form event IDs sent to clients. +Clients should already be treating event IDs as opaque identifiers, and +should not be concerned with the format of them. Clients should still +encode the event ID when including it in a request path. + +Clients should expect to see event IDs changed from the format of +`$randomstring:example.org` to something like +`$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg` (note the lack of +domain). + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. +{{% /boxes/warning %}} + +Room version 4 uses the same algorithms defined in [room version +3](/rooms/v3), however using URL-safe base64 to generate the event ID. + +### Event IDs + +{{% boxes/rationale %}} +Room version 3 generated event IDs that were difficult for client +implementations which were not encoding the event ID to function in +those rooms. It additionally raised concern due to the `/` character +being interpretted differently by some reverse proxy software, and +generally made administration harder. +{{% /boxes/rationale %}} + +The event ID is the [reference +hash](/server-server-api#calculating-the-reference-hash-for-an-event) of +the event encoded using a variation of [Unpadded +Base64](/appendices#unpadded-base64) which replaces the 62nd and +63rd characters with `-` and `_` instead of using `+` and `/`. This +matches [RFC4648's definition of URL-safe +base64](https://tools.ietf.org/html/rfc4648#section-5). Event IDs are +still prefixed with `$` and may result in looking like +`$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg`. + +Just like in room version 3, event IDs should not be sent over +federation to servers when the room uses this room version. On the +receiving end of an event, the server should compute the relevant event +ID for itself. Room version 3 also changes the format of `auth_events` +and `prev_events` in a PDU. + +{{definition\_ss\_pdu\_v4}} diff --git a/content/rooms/v5.md b/content/rooms/v5.md new file mode 100644 index 00000000..9c5ade72 --- /dev/null +++ b/content/rooms/v5.md @@ -0,0 +1,45 @@ +--- +title: Room Version 5 +type: docs +weight: 50 +--- + +This room version builds on [version 4](/rooms/v4) while enforcing signing +key validity periods for events. + +## Client considerations + +There are no specific requirements for clients in this room version. +Clients should be aware of event ID changes in [room version +4](/rooms/v4), however. + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. +{{% /boxes/warning %}} + +Room version 5 uses the same algorithms defined in [room version +4](/rooms/v4), ensuring that signing key validity is respected. + +### Signing key validity period + +When validating event signatures, servers MUST enforce the +`valid_until_ts` property from a key request is at least as large as the +`origin_server_ts` for the event being validated. Servers missing a copy +of the signing key MUST try to obtain one via the [GET +/\_matrix/key/v2/server](/server-server-api#get_matrixkeyv2serverkeyid) +or [POST +/\_matrix/key/v2/query](/server-server-api#post_matrixkeyv2query) +APIs. When using the `/query` endpoint, servers MUST set the +`minimum_valid_until_ts` property to prompt the notary server to attempt +to refresh the key if appropriate. + +Servers MUST use the lesser of `valid_until_ts` and 7 days into the +future when determining if a key is valid. This is to avoid a situation +where an attacker publishes a key which is valid for a significant +amount of time without a way for the homeserver owner to revoke it. diff --git a/content/rooms/v6.md b/content/rooms/v6.md new file mode 100644 index 00000000..e9cea27a --- /dev/null +++ b/content/rooms/v6.md @@ -0,0 +1,84 @@ +--- +title: Room Version 6 +type: docs +weight: 60 +--- + +This room version builds on [version 5](/rooms/v5) while changing various +authorization rules performed on events. + +## Client considerations + +The redaction algorithm has changed from [room version 1](/rooms/v1) to +remove all rules against events of type `m.room.aliases`. Room versions +2, 3, 4, and 5 all use v1's redaction algorithm. The algorithm is +otherwise unchanged. + +## Server implementation components + +{{% boxes/warning %}} +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. +{{% /boxes/warning %}} + +Room version 6 makes the following alterations to algorithms described +in [room version 5](/rooms/v5). + +### Redactions + +As mentioned in the client considerations portion of this specification, +all special meaning has been removed for events of type +`m.room.aliases`. The algorithm is otherwise unchanged. + +### Authorization rules for events + +Like redactions, all rules relating specifically to events of type +`m.room.aliases` are removed. They must still pass authorization checks +relating to state events. + +Additionally, the authorization rules for events of type +`m.room.power_levels` now include the content key `notifications`. This +new rule takes the place of the rule which checks the `events` and +`users` keys. + +For completeness, the changes to the auth rules can be represented as +follows: + + ... + + -If type is `m.room.aliases`: + - + - a. If event has no `state_key`, reject. + - b. If sender's domain doesn't matches `state_key`, reject. + - c. Otherwise, allow. + + ... + + If type is `m.room.power_levels`: + + ... + + - * For each entry being added, changed or removed in both the `events` and `users` keys: + + * For each entry being added, changed or removed in the `events`, `users`, and `notifications` keys: + + i. If the current value is higher than the `sender`'s current power level, reject. + + ii. If the new value is higher than the `sender`'s current power level, reject. + + ... + +The remaining rules are the same as in [room version +3](/rooms/v3#authorization-rules-for-events) (the last inherited room +version to specify the authorization rules). + +### Canonical JSON + +Servers MUST strictly enforce the JSON format specified in the +[appendices](/appendices#canonical-json). This translates to a +400 `M_BAD_JSON` error on most endpoints, or discarding of events over +federation. For example, the Federation API's `/send` endpoint would +discard the event whereas the Client Server API's `/send/{eventType}` +endpoint would return a `M_BAD_JSON` error. diff --git a/content/server-server-api.md b/content/server-server-api.md new file mode 100644 index 00000000..21d6b495 --- /dev/null +++ b/content/server-server-api.md @@ -0,0 +1,1155 @@ +--- +title: "Server-Server API" +weight: 20 +type: docs +--- + +Matrix homeservers use the Federation APIs (also known as server-server +APIs) to communicate with each other. Homeservers use these APIs to push +messages to each other in real-time, to retrieve historic messages from +each other, and to query profile and presence information about users on +each other's servers. + +The APIs are implemented using HTTPS requests between each of the +servers. These HTTPS requests are strongly authenticated using public +key signatures at the TLS transport layer and using public key +signatures in HTTP Authorization headers at the HTTP layer. + +There are three main kinds of communication that occur between +homeservers: + +Persisted Data Units (PDUs): +These events are broadcast from one homeserver to any others that have +joined the same room (identified by Room ID). They are persisted in +long-term storage and record the history of messages and state for a +room. + +Like email, it is the responsibility of the originating server of a PDU +to deliver that event to its recipient servers. However PDUs are signed +using the originating server's private key so that it is possible to +deliver them through third-party servers. + +Ephemeral Data Units (EDUs): +These events are pushed between pairs of homeservers. They are not +persisted and are not part of the history of a room, nor does the +receiving homeserver have to reply to them. + +Queries: +These are single request/response interactions between a given pair of +servers, initiated by one side sending an HTTPS GET request to obtain +some information, and responded by the other. They are not persisted and +contain no long-term significant history. They simply request a snapshot +state at the instant the query is made. + +EDUs and PDUs are further wrapped in an envelope called a Transaction, +which is transferred from the origin to the destination homeserver using +an HTTPS PUT request. + +## API standards + +The mandatory baseline for client-server communication in Matrix is +exchanging JSON objects over HTTP APIs. More efficient optional +transports will in future be supported as optional extensions - e.g. a +packed binary encoding over stream-cipher encrypted TCP socket for +low-bandwidth/low-roundtrip mobile usage. For the default HTTP +transport, all API calls use a Content-Type of `application/json`. In +addition, all strings MUST be encoded as UTF-8. + +## Server discovery + +### Resolving server names + +Each Matrix homeserver is identified by a server name consisting of a +hostname and an optional port, as described by the +[grammar](/appendices#server-name). Where applicable, a delegated +server name uses the same grammar. + +Server names are resolved to an IP address and port to connect to, and +have various conditions affecting which certificates and `Host` headers +to send. The process overall is as follows: + +1. If the hostname is an IP literal, then that IP address should be + used, together with the given port number, or 8448 if no port is + given. The target server must present a valid certificate for the IP + address. The `Host` header in the request should be set to the + server name, including the port if the server name included one. +2. If the hostname is not an IP literal, and the server name includes + an explicit port, resolve the IP address using AAAA or A records. + Requests are made to the resolved IP address and given port with a + `Host` header of the original server name (with port). The target + server must present a valid certificate for the hostname. +3. If the hostname is not an IP literal, a regular HTTPS request is + made to `https:///.well-known/matrix/server`, expecting + the schema defined later in this section. 30x redirects should be + followed, however redirection loops should be avoided. Responses + (successful or otherwise) to the `/.well-known` endpoint should be + cached by the requesting server. Servers should respect the cache + control headers present on the response, or use a sensible default + when headers are not present. The recommended sensible default is 24 + hours. Servers should additionally impose a maximum cache time for + responses: 48 hours is recommended. Errors are recommended to be + cached for up to an hour, and servers are encouraged to + exponentially back off for repeated failures. The schema of the + `/.well-known` request is later in this section. If the response is + invalid (bad JSON, missing properties, non-200 response, etc), skip + to step 4. If the response is valid, the `m.server` property is + parsed as `[:]` and processed as + follows: + - If `` is an IP literal, then that IP address + should be used together with the `` or 8448 if + no port is provided. The target server must present a valid TLS + certificate for the IP address. Requests must be made with a + `Host` header containing the IP address, including the port if + one was provided. + - If `` is not an IP literal, and + `` is present, an IP address is discovered by + looking up an AAAA or A record for ``. The + resulting IP address is used, alongside the ``. + Requests must be made with a `Host` header of + `:`. The target server must + present a valid certificate for ``. + - If `` is not an IP literal and no + `` is present, an SRV record is looked up for + `_matrix._tcp.`. This may result in another + hostname (to be resolved using AAAA or A records) and port. + Requests should be made to the resolved IP address and port with + a `Host` header containing the ``. The + target server must present a valid certificate for + ``. + - If no SRV record is found, an IP address is resolved using AAAA + or A records. Requests are then made to the resolve IP address + and a port of 8448, using a `Host` header of + ``. The target server must present a valid + certificate for ``. +4. If the `/.well-known` request resulted in an error response, a + server is found by resolving an SRV record for + `_matrix._tcp.`. This may result in a hostname (to be + resolved using AAAA or A records) and port. Requests are made to the + resolved IP address and port, using 8448 as a default port, with a + `Host` header of ``. The target server must present a + valid certificate for ``. +5. If the `/.well-known` request returned an error response, and the + SRV record was not found, an IP address is resolved using AAAA and A + records. Requests are made to the resolved IP address using port + 8448 and a `Host` header containing the ``. The target + server must present a valid certificate for ``. + +The TLS certificate provided by the target server must be signed by a +known Certificate Authority. Servers are ultimately responsible for +determining the trusted Certificate Authorities, however are strongly +encouraged to rely on the operating system's judgement. Servers can +offer administrators a means to override the trusted authorities list. +Servers can additionally skip the certificate validation for a given +whitelist of domains or netmasks for the purposes of testing or in +networks where verification is done elsewhere, such as with `.onion` +addresses. Servers should respect SNI when making requests where +possible: a SNI should be sent for the certificate which is expected, +unless that certificate is expected to be an IP address in which case +SNI is not supported and should not be sent. + +Servers are encouraged to make use of the [Certificate +Transparency](https://www.certificate-transparency.org/) project. + +{{wellknown\_ss\_http\_api}} + +### Server implementation + +{{version\_ss\_http\_api}} + +### Retrieving server keys + +{{% boxes/note %}} +There was once a "version 1" of the key exchange. It has been removed +from the specification due to lack of significance. It may be reviewed +[from the historical +draft](https://github.com/matrix-org/matrix-doc/blob/51faf8ed2e4a63d4cfd6d23183698ed169956cc0/specification/server_server_api.rst#232version-1). +{{% /boxes/note %}} + +Each homeserver publishes its public keys under +`/_matrix/key/v2/server/{keyId}`. Homeservers query for keys by either +getting `/_matrix/key/v2/server/{keyId}` directly or by querying an +intermediate notary server using a +`/_matrix/key/v2/query/{serverName}/{keyId}` API. Intermediate notary +servers query the `/_matrix/key/v2/server/{keyId}` API on behalf of +another server and sign the response with their own key. A server may +query multiple notary servers to ensure that they all report the same +public keys. + +This approach is borrowed from the [Perspectives +Project](https://web.archive.org/web/20170702024706/https://perspectives-project.org/), +but modified to include the NACL keys and to use JSON instead of XML. It +has the advantage of avoiding a single trust-root since each server is +free to pick which notary servers they trust and can corroborate the +keys returned by a given notary server by querying other servers. + +#### Publishing Keys + +Homeservers publish their signing keys in a JSON object at +`/_matrix/key/v2/server/{key_id}`. The response contains a list of +`verify_keys` that are valid for signing federation requests made by the +homeserver and for signing events. It contains a list of +`old_verify_keys` which are only valid for signing events. + +{{keys\_server\_ss\_http\_api}} + +#### Querying Keys Through Another Server + +Servers may query another server's keys through a notary server. The +notary server may be another homeserver. The notary server will retrieve +keys from the queried servers through use of the +`/_matrix/key/v2/server/{keyId}` API. The notary server will +additionally sign the response from the queried server before returning +the results. + +Notary servers can return keys for servers that are offline or having +issues serving their own keys by using cached responses. Keys can be +queried from multiple servers to mitigate against DNS spoofing. + +{{keys\_query\_ss\_http\_api}} + +## Authentication + +### Request Authentication + +Every HTTP request made by a homeserver is authenticated using public +key digital signatures. The request method, target and body are signed +by wrapping them in a JSON object and signing it using the JSON signing +algorithm. The resulting signatures are added as an Authorization header +with an auth scheme of `X-Matrix`. Note that the target field should +include the full path starting with `/_matrix/...`, including the `?` +and any query parameters if present, but should not include the leading +`https:`, nor the destination server's hostname. + +Step 1 sign JSON: + +``` +{ + "method": "GET", + "uri": "/target", + "origin": "origin.hs.example.com", + "destination": "destination.hs.example.com", + "content": , + "signatures": { + "origin.hs.example.com": { + "ed25519:key1": "ABCDEF..." + } + } +} +``` + +The server names in the JSON above are the server names for each +homeserver involved. Delegation from the [server name resolution +section](#resolving-server-names) above do not affect these - the server +names from before delegation would take place are used. This same +condition applies throughout the request signing process. + +Step 2 add Authorization header: + + GET /target HTTP/1.1 + Authorization: X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..." + Content-Type: application/json + + + +Example python code: + +```py +def authorization_headers(origin_name, origin_signing_key, + destination_name, request_method, request_target, + content=None): + request_json = { + "method": request_method, + "uri": request_target, + "origin": origin_name, + "destination": destination_name, + } + + if content is not None: + request_json["content"] = content + + signed_json = sign_json(request_json, origin_name, origin_signing_key) + + authorization_headers = [] + + for key, sig in signed_json["signatures"][origin_name].items(): + authorization_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + origin_name, key, sig, + ) + )) + + return ("Authorization", authorization_headers) +``` + +### Response Authentication + +Responses are authenticated by the TLS server certificate. A homeserver +should not send a request until it has authenticated the connected +server to avoid leaking messages to eavesdroppers. + +### Client TLS Certificates + +Requests are authenticated at the HTTP layer rather than at the TLS +layer because HTTP services like Matrix are often deployed behind load +balancers that handle the TLS and these load balancers make it difficult +to check TLS client certificates. + +A homeserver may provide a TLS client certificate and the receiving +homeserver may check that the client certificate matches the certificate +of the origin homeserver. + +## Transactions + +The transfer of EDUs and PDUs between homeservers is performed by an +exchange of Transaction messages, which are encoded as JSON objects, +passed over an HTTP PUT request. A Transaction is meaningful only to the +pair of homeservers that exchanged it; they are not globally-meaningful. + +Transactions are limited in size; they can have at most 50 PDUs and 100 +EDUs. + +{{transactions\_ss\_http\_api}} + +## PDUs + +Each PDU contains a single Room Event which the origin server wants to +send to the destination. + +The `prev_events` field of a PDU identifies the "parents" of the event, +and thus establishes a partial ordering on events within the room by +linking them into a Directed Acyclic Graph (DAG). The sending server +should populate this field with all of the events in the room for which +it has not yet seen a child - thus demonstrating that the event comes +after all other known events. + +For example, consider a room whose events form the DAG shown below. A +server creating a new event in this room should populate the new event's +`prev_events` field with `E4` and `E5`, since neither event yet has a +child: + + E1 + ^ + | + +-> E2 <-+ + | | + E3 E5 + ^ + | + E4 + +For a full schema of what a PDU looks like, see the [room version +specification](../index.html#room-versions). + +### Checks performed on receipt of a PDU + +Whenever a server receives an event from a remote server, the receiving +server must ensure that the event: + +1. Is a valid event, otherwise it is dropped. +2. Passes signature checks, otherwise it is dropped. +3. Passes hash checks, otherwise it is redacted before being processed + further. +4. Passes authorization rules based on the event's auth events, + otherwise it is rejected. +5. Passes authorization rules based on the state at the event, + otherwise it is rejected. +6. Passes authorization rules based on the current state of the room, + otherwise it is "soft failed". + +Further details of these checks, and how to handle failures, are +described below. + +The [Signing Events](#signing-events) section has more information on +which hashes and signatures are expected on events, and how to calculate +them. + +#### Definitions + +Required Power Level +A given event type has an associated *required power level*. This is +given by the current `m.room.power_levels` event. The event type is +either listed explicitly in the `events` section or given by either +`state_default` or `events_default` depending on if the event is a state +event or not. + +Invite Level, Kick Level, Ban Level, Redact Level +The levels given by the `invite`, `kick`, `ban`, and `redact` properties +in the current `m.room.power_levels` state. Each defaults to 50 if +unspecified. + +Target User +For an `m.room.member` state event, the user given by the `state_key` of +the event. + +#### Authorization rules + +The rules governing whether an event is authorized depends on a set of +state. A given event is checked multiple times against different sets of +state, as specified above. Each room version can have a different +algorithm for how the rules work, and which rules are applied. For more +detailed information, please see the [room version +specification](../index.html#room-versions). + +##### Auth events selection + +The `auth_events` field of a PDU identifies the set of events which give +the sender permission to send the event. The `auth_events` for the +`m.room.create` event in a room is empty; for other events, it should be +the following subset of the room state: + +- The `m.room.create` event. + +- The current `m.room.power_levels` event, if any. + +- The sender's current `m.room.member` event, if any. + +- If type is `m.room.member`: + + - The target's current `m.room.member` event, if any. + - If `membership` is `join` or `invite`, the current + `m.room.join_rules` event, if any. + - If membership is `invite` and `content` contains a + `third_party_invite` property, the current + `m.room.third_party_invite` event with `state_key` matching + `content.third_party_invite.signed.token`, if any. + +#### Rejection + +If an event is rejected it should neither be relayed to clients nor be +included as a prev event in any new events generated by the server. +Subsequent events from other servers that reference rejected events +should be allowed if they still pass the auth rules. The state used in +the checks should be calculated as normal, except not updating with the +rejected event where it is a state event. + +If an event in an incoming transaction is rejected, this should not +cause the transaction request to be responded to with an error response. + +{{% boxes/note %}} +This means that events may be included in the room DAG even though they +should be rejected. +{{% /boxes/note %}} + +{{% boxes/note %}} +This is in contrast to redacted events which can still affect the state +of the room. For example, a redacted `join` event will still result in +the user being considered joined. +{{% /boxes/note %}} + +#### Soft failure + +{{% boxes/rationale %}} +It is important that we prevent users from evading bans (or other power +restrictions) by creating events which reference old parts of the DAG. +For example, a banned user could continue to send messages to a room by +having their server send events which reference the event before they +were banned. Note that such events are entirely valid, and we cannot +simply reject them, as it is impossible to distinguish such an event +from a legitimate one which has been delayed. We must therefore accept +such events and let them participate in state resolution and the +federation protocol as normal. However, servers may choose not to send +such events on to their clients, so that end users won't actually see +the events. + +When this happens it is often fairly obvious to servers, as they can see +that the new event doesn't actually pass auth based on the "current +state" (i.e. the resolved state across all forward extremities). While +the event is technically valid, the server can choose to not notify +clients about the new event. + +This discourages servers from sending events that evade bans etc. in +this way, as end users won't actually see the events. +{{% /boxes/rationale %}} + +When the homeserver receives a new event over federation it should also +check whether the event passes auth checks based on the current state of +the room (as well as based on the state at the event). If the event does +not pass the auth checks based on the *current state* of the room (but +does pass the auth checks based on the state at that event) it should be +"soft failed". + +When an event is "soft failed" it should not be relayed to the client +nor be referenced by new events created by the homeserver (i.e. they +should not be added to the server's list of forward extremities of the +room). Soft failed events are otherwise handled as usual. + +{{% boxes/note %}} +Soft failed events participate in state resolution as normal if further +events are received which reference it. It is the job of the state +resolution algorithm to ensure that malicious events cannot be injected +into the room state via this mechanism. +{{% /boxes/note %}} + +{{% boxes/note %}} +Because soft failed state events participate in state resolution as +normal, it is possible for such events to appear in the current state of +the room. In that case the client should be told about the soft failed +event in the usual way (e.g. by sending it down in the `state` section +of a sync response). +{{% /boxes/note %}} + +{{% boxes/note %}} +A soft failed event should be returned in response to federation +requests where appropriate (e.g. in `/event/`). Note that soft +failed events are returned in `/backfill` and `/get_missing_events` +responses only if the requests include events referencing the soft +failed events. +{{% /boxes/note %}} + +Example + +As an example consider the event graph: + + A + / + B + +where `B` is a ban of a user `X`. If the user `X` tries to set the topic +by sending an event `C` while evading the ban: + + A + / \ + B C + +servers that receive `C` after `B` should soft fail event `C`, and so +will neither relay `C` to its clients nor send any events referencing +`C`. + +If later another server sends an event `D` that references both `B` and +`C` (this can happen if it received `C` before `B`): + + A + / \ + B C + \ / + D + +then servers will handle `D` as normal. `D` is sent to the servers' +clients (assuming `D` passes auth checks). The state at `D` may resolve +to a state that includes `C`, in which case clients should also to be +told that the state has changed to include `C`. (*Note*: This depends on +the exact state resolution algorithm used. In the original version of +the algorithm `C` would be in the resolved state, whereas in latter +versions the algorithm tries to prioritise the ban over the topic +change.) + +Note that this is essentially equivalent to the situation where one +server doesn't receive `C` at all, and so asks another server for the +state of the `C` branch. + +Let's go back to the graph before `D` was sent: + + A + / \ + B C + +If all the servers in the room saw `B` before `C` and so soft fail `C`, +then any new event `D'` will not reference `C`: + + A + / \ + B C + | + D + +#### Retrieving event authorization information + +The homeserver may be missing event authorization information, or wish +to check with other servers to ensure it is receiving the correct auth +chain. These APIs give the homeserver an avenue for getting the +information it needs. + +{{event\_auth\_ss\_http\_api}} + +## EDUs + +EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of +"previous" IDs. They are intended to be non-persistent data such as user +presence, typing notifications, etc. + +{{definition\_ss\_edu}} + +## Room State Resolution + +The *state* of a room is a map of `(event_type, state_key)` to +`event_id`. Each room starts with an empty state, and each state event +which is accepted into the room updates the state of that room. + +Where each event has a single `prev_event`, it is clear what the state +of the room after each event should be. However, when two branches in +the event graph merge, the state of those branches might differ, so a +*state resolution* algorithm must be used to determine the resultant +state. + +For example, consider the following event graph (where the oldest event, +E0, is at the top): + + E0 + | + E1 + / \ + E2 E4 + | | + E3 | + \ / + E5 + +Suppose E3 and E4 are both `m.room.name` events which set the name of +the room. What should the name of the room be at E5? + +The algorithm to be used for state resolution depends on the room +version. For a description of each room version's algorithm, please see +the [room version specification](/#room-versions). + +## Backfilling and retrieving missing events + +Once a homeserver has joined a room, it receives all the events emitted +by other homeservers in that room, and is thus aware of the entire +history of the room from that moment onwards. Since users in that room +are able to request the history by the `/messages` client API endpoint, +it's possible that they might step backwards far enough into history +before the homeserver itself was a member of that room. + +To cover this case, the federation API provides a server-to-server +analog of the `/messages` client API, allowing one homeserver to fetch +history from another. This is the `/backfill` API. + +To request more history, the requesting homeserver picks another +homeserver that it thinks may have more (most likely this should be a +homeserver for some of the existing users in the room at the earliest +point in history it has currently), and makes a `/backfill` request. + +Similar to backfilling a room's history, a server may not have all the +events in the graph. That server may use the `/get_missing_events` API +to acquire the events it is missing. + +{{backfill\_ss\_http\_api}} + +## Retrieving events + +In some circumstances, a homeserver may be missing a particular event or +information about the room which cannot be easily determined from +backfilling. These APIs provide homeservers with the option of getting +events and the state of the room at a given point in the timeline. + +{{events\_ss\_http\_api}} + +## Joining Rooms + +When a new user wishes to join a room that the user's homeserver already +knows about, the homeserver can immediately determine if this is +allowable by inspecting the state of the room. If it is acceptable, it +can generate, sign, and emit a new `m.room.member` state event adding +the user into that room. When the homeserver does not yet know about the +room it cannot do this directly. Instead, it must take a longer +multi-stage handshaking process by which it first selects a remote +homeserver which is already participating in that room, and use it to +assist in the joining process. This is the remote join handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting +the room alias the user is requesting to join with, and a homeserver +where existing room members are already present (referred to as the +"resident" server). + +In summary, the remote join handshake consists of the joining server +querying the directory server for information about the room alias; +receiving a room ID and a list of join candidates. The joining server +then requests information about the room from one of the residents. It +uses this information to construct an `m.room.member` event which it +finally sends to a resident server. + +Conceptually these are three different roles of homeserver. In practice +the directory server is likely to be resident in the room, and so may be +selected by the joining server to be the assisting resident. Likewise, +it is likely that the joining server picks the same candidate resident +for both phases of event construction, though in principle any valid +candidate may be used at each time. Thus, any join handshake can +potentially involve anywhere from two to four homeservers, though most +in practice will use just two. + +``` + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response +``` + +The first part of the handshake usually involves using the directory +server to request the room ID and join candidates through the +`/query/directory`\_ API endpoint. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver +could optimise this step away by picking the origin server of that +invite message as the join candidate. However, the joining server should +be aware that the origin server of the invite might since have left the +room, so should be prepared to fall back on the regular join flow if +this optimisation fails. + +Once the joining server has the room ID and the join candidates, it then +needs to obtain enough information about the room to fill in the +required fields of the `m.room.member` event. It obtains this by +selecting a resident from the candidate list, and using the +`GET /make_join` endpoint. The resident server will then reply with +enough information for the joining server to fill in the event. + +The joining server is expected to add or replace the `origin`, +`origin_server_ts`, and `event_id` on the templated event received by +the resident server. This event is then signed by the joining server. + +To complete the join handshake, the joining server must now submit this +new event to a resident homeserver, by using the `PUT /send_join` +endpoint. + +The resident homeserver then accepts this event into the room's event +graph, and responds to the joining server with the full set of state for +the newly-joined room. The resident server must also send the event to +other servers participating in the room. + +{{joins\_v1\_ss\_http\_api}} + +{{joins\_v2\_ss\_http\_api}} + +## Inviting to a room + +When a user on a given homeserver invites another user on the same +homeserver, the homeserver may sign the membership event itself and skip +the process defined here. However, when a user invites another user on a +different homeserver, a request to that homeserver to have the event +signed and verified must be made. + +{{invites\_v1\_ss\_http\_api}} + +{{invites\_v2\_ss\_http\_api}} + +## Leaving Rooms (Rejecting Invites) + +Normally homeservers can send appropriate `m.room.member` events to have +users leave the room, or to reject local invites. Remote invites from +other homeservers do not involve the server in the graph and therefore +need another approach to reject the invite. Joining the room and +promptly leaving is not recommended as clients and servers will +interpret that as accepting the invite, then leaving the room rather +than rejecting the invite. + +Similar to the [Joining Rooms](#joining-rooms) handshake, the server +which wishes to leave the room starts with sending a `/make_leave` +request to a resident server. In the case of rejecting invites, the +resident server may be the server which sent the invite. After receiving +a template event from `/make_leave`, the leaving server signs the event +and replaces the `event_id` with its own. This is then sent to the +resident server via `/send_leave`. The resident server will then send +the event to other servers in the room. + +{{leaving\_v1\_ss\_http\_api}} + +{{leaving\_v2\_ss\_http\_api}} + +## Third-party invites + +{{% boxes/note %}} +More information about third party invites is available in the +[Client-Server API](/client-server-api) under +the Third Party Invites module. +{{% /boxes/note %}} + +When a user wants to invite another user in a room but doesn't know the +Matrix ID to invite, they can do so using a third-party identifier (e.g. +an e-mail or a phone number). + +This identifier and its bindings to Matrix IDs are verified by an +identity server implementing the [Identity Service +API](/identity-service-api). + +### Cases where an association exists for a third-party identifier + +If the third-party identifier is already bound to a Matrix ID, a lookup +request on the identity server will return it. The invite is then +processed by the inviting homeserver as a standard `m.room.member` +invite event. This is the simplest case. + +### Cases where an association doesn't exist for a third-party identifier + +If the third-party identifier isn't bound to any Matrix ID, the inviting +homeserver will request the identity server to store an invite for this +identifier and to deliver it to whoever binds it to its Matrix ID. It +will also send an `m.room.third_party_invite` event in the room to +specify a display name, a token and public keys the identity server +provided as a response to the invite storage request. + +When a third-party identifier with pending invites gets bound to a +Matrix ID, the identity server will send a POST request to the ID's +homeserver as described in the [Invitation +Storage](/identity-service-api#invitation-storage) +section of the Identity Service API. + +The following process applies for each invite sent by the identity +server: + +The invited homeserver will create an `m.room.member` invite event +containing a special `third_party_invite` section containing the token +and a signed object, both provided by the identity server. + +If the invited homeserver is in the room the invite came from, it can +auth the event and send it. + +However, if the invited homeserver isn't in the room the invite came +from, it will need to request the room's homeserver to auth the event. + +{{third\_party\_invite\_ss\_http\_api}} + +#### Verifying the invite + +When a homeserver receives an `m.room.member` invite event for a room +it's in with a `third_party_invite` object, it must verify that the +association between the third-party identifier initially invited to the +room and the Matrix ID that claims to be bound to it has been verified +without having to rely on a third-party server. + +To do so, it will fetch from the room's state events the +`m.room.third_party_invite` event for which the state key matches with +the value for the `token` key in the `third_party_invite` object from +the `m.room.member` event's content to fetch the public keys initially +delivered by the identity server that stored the invite. + +It will then use these keys to verify that the `signed` object (in the +`third_party_invite` object from the `m.room.member` event's content) +was signed by the same identity server. + +Since this `signed` object can only be delivered once in the POST +request emitted by the identity server upon binding between the +third-party identifier and the Matrix ID, and contains the invited +user's Matrix ID and the token delivered when the invite was stored, +this verification will prove that the `m.room.member` invite event comes +from the user owning the invited third-party identifier. + +## Public Room Directory + +To complement the [Client-Server +API](/client-server-api)'s room directory, +homeservers need a way to query the public rooms for another server. +This can be done by making a request to the `/publicRooms` endpoint for +the server the room directory should be retrieved for. + +{{public\_rooms\_ss\_http\_api}} + +## Typing Notifications + +When a server's users send typing notifications, those notifications +need to be sent to other servers in the room so their users are aware of +the same state. Receiving servers should verify that the user is in the +room, and is a user belonging to the sending server. + +{{definition\_ss\_event\_schemas\_m\_typing}} + +## Presence + +The server API for presence is based entirely on exchange of the +following EDUs. There are no PDUs or Federation Queries involved. + +Servers should only send presence updates for users that the receiving +server would be interested in. Such as the receiving server sharing a +room with a given user. + +{{definition\_ss\_event\_schemas\_m\_presence}} + +## Receipts + +Receipts are EDUs used to communicate a marker for a given event. +Currently the only kind of receipt supported is a "read receipt", or +where in the event graph the user has read up to. + +Read receipts for events that a user sent do not need to be sent. It is +implied that by sending the event the user has read up to the event. + +{{definition\_ss\_event\_schemas\_m\_receipt}} + +## Querying for information + +Queries are a way to retrieve information from a homeserver about a +resource, such as a user or room. The endpoints here are often called in +conjunction with a request from a client on the client-server API in +order to complete the call. + +There are several types of queries that can be made. The generic +endpoint to represent all queries is described first, followed by the +more specific queries that can be made. + +{{query\_ss\_http\_api}} + +## OpenID + +Third party services can exchange an access token previously generated +by the Client-Server API for information +about a user. This can help verify that a user is who they say they are +without granting full access to the user's account. + +Access tokens generated by the OpenID API are only good for the OpenID +API and nothing else. + +{{openid\_ss\_http\_api}} + +## Device Management + +Details of a user's devices must be efficiently published to other users +and kept up-to-date. This is critical for reliable end-to-end +encryption, in order for users to know which devices are participating +in a room. It's also required for to-device messaging to work. This +section is intended to complement the [Device Management +module](/client-server-api#device-management) +of the Client-Server API. + +Matrix currently uses a custom pubsub system for synchronising +information about the list of devices for a given user over federation. +When a server wishes to determine a remote user's device list for the +first time, it should populate a local cache from the result of a +`/user/keys/query` API on the remote server. However, subsequent updates +to the cache should be applied by consuming `m.device_list_update` EDUs. +Each new `m.device_list_update` EDU describes an incremental change to +one device for a given user which should replace any existing entry in +the local server's cache of that device list. Servers must send +`m.device_list_update` EDUs to all the servers who share a room with a +given local user, and must be sent whenever that user's device list +changes (i.e. for new or deleted devices, when that user joins a room +which contains servers which are not already receiving updates for that +user's device list, or changes in device information such as the +device's human-readable name). + +Servers send `m.device_list_update` EDUs in a sequence per origin user, +each with a unique `stream_id`. They also include a pointer to the most +recent previous EDU(s) that this update is relative to in the `prev_id` +field. To simplify implementation for clustered servers which could send +multiple EDUs at the same time, the `prev_id` field should include all +`m.device_list_update` EDUs which have not been yet been referenced in a +EDU. If EDUs are emitted in series by a server, there should only ever +be one `prev_id` in the EDU. + +This forms a simple directed acyclic graph of `m.device_list_update` +EDUs, showing which EDUs a server needs to have received in order to +apply an update to its local copy of the remote user's device list. If a +server receives an EDU which refers to a `prev_id` it does not +recognise, it must resynchronise its list by calling the +`/user/keys/query API` and resume the process. The response contains a +`stream_id` which should be used to correlate with subsequent +`m.device_list_update` EDUs. + +{{user\_devices\_ss\_http\_api}} + +{{definition\_ss\_event\_schemas\_m\_device\_list\_update}} + +## End-to-End Encryption + +This section complements the [End-to-End Encryption +module](/client-server-api#end-to-end-encryption) +of the Client-Server API. For detailed information about end-to-end +encryption, please see that module. + +The APIs defined here are designed to be able to proxy much of the +client's request through to federation, and have the response also be +proxied through to the client. + +{{user\_keys\_ss\_http\_api}} + +{{definition\_ss\_event\_schemas\_m\_signing\_key\_update}} + +## Send-to-device messaging + +The server API for send-to-device messaging is based on the +`m.direct_to_device` EDU. There are no PDUs or Federation Queries +involved. + +Each send-to-device message should be sent to the destination server +using the following EDU: + +{{definition\_ss\_event\_schemas\_m\_direct\_to\_device}} + +## Content Repository + +Attachments to events (images, files, etc) are uploaded to a homeserver +via the Content Repository described in the [Client-Server +API](/client-server-api). When a server wishes +to serve content originating from a remote server, it needs to ask the +remote server for the media. + +Servers should use the server described in the Matrix Content URI, which +has the format `mxc://{ServerName}/{MediaID}`. Servers should use the +download endpoint described in the [Client-Server +API](/client-server-api), being sure to use +the `allow_remote` parameter (set to `false`). + +## Server Access Control Lists (ACLs) + +Server ACLs and their purpose are described in the [Server +ACLs](/client-server-api#server-access-control-lists-acls-for-rooms) +section of the Client-Server API. + +When a remote server makes a request, it MUST be verified to be allowed +by the server ACLs. If the server is denied access to a room, the +receiving server MUST reply with a 403 HTTP status code and an `errcode` +of `M_FORBIDDEN`. + +The following endpoint prefixes MUST be protected: + +- `/_matrix/federation/v1/send` (on a per-PDU basis) +- `/_matrix/federation/v1/make_join` +- `/_matrix/federation/v1/make_leave` +- `/_matrix/federation/v1/send_join` +- `/_matrix/federation/v2/send_join` +- `/_matrix/federation/v1/send_leave` +- `/_matrix/federation/v2/send_leave` +- `/_matrix/federation/v1/invite` +- `/_matrix/federation/v2/invite` +- `/_matrix/federation/v1/state` +- `/_matrix/federation/v1/state_ids` +- `/_matrix/federation/v1/backfill` +- `/_matrix/federation/v1/event_auth` +- `/_matrix/federation/v1/get_missing_events` + +## Signing Events + +Signing events is complicated by the fact that servers can choose to +redact non-essential parts of an event. + +### Adding hashes and signatures to outgoing events + +Before signing the event, the *content hash* of the event is calculated +as described below. The hash is encoded using [Unpadded +Base64](/appendices#unpadded-base64) and stored in the event +object, in a `hashes` object, under a `sha256` key. + +The event object is then *redacted*, following the [redaction +algorithm](/client-server-api#redactions). +Finally it is signed as described in [Signing +JSON](/appendices#signing-json), using the server's signing key +(see also [Retrieving server keys](#retrieving-server-keys)). + +The signature is then copied back to the original event object. + +See [Persistent Data Unit schema](#Persistent Data Unit schema) for an +example of a signed event. + +### Validating hashes and signatures on received events + +When a server receives an event over federation from another server, the +receiving server should check the hashes and signatures on that event. + +First the signature is checked. The event is redacted following the +[redaction +algorithm](/client-server-api#redactions), and +the resultant object is checked for a signature from the originating +server, following the algorithm described in [Checking for a +signature](/appendices#checking-for-a-signature). Note that this +step should succeed whether we have been sent the full event or a +redacted copy. + +The signatures expected on an event are: + +- The `sender`'s server, unless the invite was created as a result of + 3rd party invite. The sender must already match the 3rd party + invite, and the server which actually sends the event may be a + different server. +- For room versions 1 and 2, the server which created the `event_id`. + Other room versions do not track the `event_id` over federation and + therefore do not need a signature from those servers. + +If the signature is found to be valid, the expected content hash is +calculated as described below. The content hash in the `hashes` property +of the received event is base64-decoded, and the two are compared for +equality. + +If the hash check fails, then it is assumed that this is because we have +only been given a redacted version of the event. To enforce this, the +receiving server should use the redacted copy it calculated rather than +the full copy it received. + +### Calculating the reference hash for an event + +The *reference hash* of an event covers the essential fields of an +event, including content hashes. It is used for event identifiers in +some room versions. See the [room version +specification](/#room-versions) for more information. It is +calculated as follows. + +1. The event is put through the redaction algorithm. +2. The `signatures`, `age_ts`, and `unsigned` properties are removed + from the event, if present. +3. The event is converted into [Canonical + JSON](/appendices#canonical-json). +4. A sha256 hash is calculated on the resulting JSON object. + +### Calculating the content hash for an event + +The *content hash* of an event covers the complete event including the +*unredacted* contents. It is calculated as follows. + +First, any existing `unsigned`, `signature`, and `hashes` members are +removed. The resulting object is then encoded as [Canonical +JSON](/appendices#canonical-json), and the JSON is hashed using +SHA-256. + +### Example code + +```py +def hash_and_sign_event(event_object, signing_key, signing_name): + # First we need to hash the event object. + content_hash = compute_content_hash(event_object) + event_object["hashes"] = {"sha256": encode_unpadded_base64(content_hash)} + + # Strip all the keys that would be removed if the event was redacted. + # The hashes are not stripped and cover all the keys in the event. + # This means that we can tell if any of the non-essential keys are + # modified or removed. + stripped_object = strip_non_essential_keys(event_object) + + # Sign the stripped JSON object. The signature only covers the + # essential keys and the hashes. This means that we can check the + # signature even if the event is redacted. + signed_object = sign_json(stripped_object, signing_key, signing_name) + + # Copy the signatures from the stripped event to the original event. + event_object["signatures"] = signed_object["signatures"] + +def compute_content_hash(event_object): + # take a copy of the event before we remove any keys. + event_object = dict(event_object) + + # Keys under "unsigned" can be modified by other servers. + # They are useful for conveying information like the age of an + # event that will change in transit. + # Since they can be modified we need to exclude them from the hash. + event_object.pop("unsigned", None) + + # Signatures will depend on the current value of the "hashes" key. + # We cannot add new hashes without invalidating existing signatures. + event_object.pop("signatures", None) + + # The "hashes" key might contain multiple algorithms if we decide to + # migrate away from SHA-2. We don't want to include an existing hash + # output in our hash so we exclude the "hashes" dict from the hash. + event_object.pop("hashes", None) + + # Encode the JSON using a canonical encoding so that we get the same + # bytes on every server for the same JSON object. + event_json_bytes = encode_canonical_json(event_object) + + return hashlib.sha256(event_json_bytes) +``` + +## Security considerations + +When a domain's ownership changes, the new controller of the domain can +masquerade as the previous owner, receiving messages (similarly to +email) and request past messages from other servers. In the future, +proposals like +[MSC1228](https://github.com/matrix-org/matrix-doc/issues/1228) will +address this issue. diff --git a/layouts/partials/alert.html b/layouts/partials/alert.html new file mode 100644 index 00000000..9096769f --- /dev/null +++ b/layouts/partials/alert.html @@ -0,0 +1,20 @@ +{{/* + + Display an alert box of the given type, containing the given content. + + Supported types are "warning", "info", and "rationale". + The type determines the colors used in the box and, by default, + the title for the box: WARNING, INFO, RATIONALE. + + By passing "omit_title", a caller can suppress the title, and just get + a box using the colors and styles for the given type. + +*/}} + +{{ $type := .type}} +{{ $content := .content}} +{{ $omit_title := .omit_title }} + + diff --git a/layouts/shortcodes/boxes/note.html b/layouts/shortcodes/boxes/note.html new file mode 100644 index 00000000..48c9d3c9 --- /dev/null +++ b/layouts/shortcodes/boxes/note.html @@ -0,0 +1 @@ +{{ partial "alert" (dict "type" "note" "content" .Inner) }} diff --git a/layouts/shortcodes/boxes/rationale.html b/layouts/shortcodes/boxes/rationale.html new file mode 100644 index 00000000..385bd4b6 --- /dev/null +++ b/layouts/shortcodes/boxes/rationale.html @@ -0,0 +1 @@ +{{ partial "alert" (dict "type" "rationale" "content" .Inner) }} diff --git a/layouts/shortcodes/boxes/warning.html b/layouts/shortcodes/boxes/warning.html new file mode 100644 index 00000000..6e884026 --- /dev/null +++ b/layouts/shortcodes/boxes/warning.html @@ -0,0 +1 @@ +{{ partial "alert" (dict "type" "warning" "content" .Inner) }} diff --git a/layouts/shortcodes/cs-modules.html b/layouts/shortcodes/cs-modules.html new file mode 100644 index 00000000..eb294166 --- /dev/null +++ b/layouts/shortcodes/cs-modules.html @@ -0,0 +1,14 @@ +{{/* + + This template is used to embed module documentation in the client-server API spec. + + It searches the site for pages of type "module", sorts them by weight, and + emits the page's rendered content. + +*/}} + +{{ $modules := where site.Pages "Type" "module" }} + +{{ range $modules.ByWeight }} +{{ .Content }} +{{ end }} diff --git a/layouts/shortcodes/sas-emojis.html b/layouts/shortcodes/sas-emojis.html new file mode 100644 index 00000000..5ce6a009 --- /dev/null +++ b/layouts/shortcodes/sas-emojis.html @@ -0,0 +1,26 @@ +{{/* + + This template is used to render the table of SAS emoji table. + + It replaces the old {{sas_emoji_table}} template. + +*/}} + +{{ $emoji_json := readFile "data-definitions/sas-emoji.json" | transform.Unmarshal }} + + + + + + + + +{{ range $emoji_json }} + + + + + + +{{ end }} +
NumberEmojiUnicodeDescription
{{ .number }}{{ .emoji }}{{ .unicode }}{{ .description }}