From c924b3246fb315299430d5d90e92bf5e452b568e Mon Sep 17 00:00:00 2001 From: Will Date: Tue, 19 Jan 2021 15:14:52 -0800 Subject: [PATCH] Add page content as raw Pandoc output --- content/appendices.md | 988 ++++++++++++++ content/application-service-api.md | 470 +++++++ content/client-server-api/_index.md | 1941 +++++++++++++++++++++++++++ content/identity-service-api.md | 467 +++++++ content/proposals.md | 594 ++++++++ content/push-gateway-api.md | 78 ++ content/rooms/v1.md | 284 ++++ content/rooms/v2.md | 187 +++ content/rooms/v3.md | 99 ++ content/rooms/v4.md | 60 + content/rooms/v5.md | 44 + content/rooms/v6.md | 83 ++ content/server-server-api.md | 1174 ++++++++++++++++ 13 files changed, 6469 insertions(+) diff --git a/content/appendices.md b/content/appendices.md index 27c27774..12470873 100644 --- a/content/appendices.md +++ b/content/appendices.md @@ -3,3 +3,991 @@ title: "Appendices" weight: 70 type: docs --- + +# Unpadded Base64 + +*Unpadded* Base64 refers to 'standard' Base64 encoding as defined in +[RFC 4648](https://tools.ietf.org/html/rfc4648), without "=" padding. +Specifically, where RFC 4648 requires that encoded data be padded to a +multiple of four characters using `=` characters, unpadded Base64 omits +this padding. + +For reference, RFC 4648 uses the following alphabet for Base 64: + + Value Encoding Value Encoding Value Encoding Value Encoding + 0 A 17 R 34 i 51 z + 1 B 18 S 35 j 52 0 + 2 C 19 T 36 k 53 1 + 3 D 20 U 37 l 54 2 + 4 E 21 V 38 m 55 3 + 5 F 22 W 39 n 56 4 + 6 G 23 X 40 o 57 5 + 7 H 24 Y 41 p 58 6 + 8 I 25 Z 42 q 59 7 + 9 J 26 a 43 r 60 8 + 10 K 27 b 44 s 61 9 + 11 L 28 c 45 t 62 + + 12 M 29 d 46 u 63 / + 13 N 30 e 47 v + 14 O 31 f 48 w + 15 P 32 g 49 x + 16 Q 33 h 50 y + +Examples of strings encoded using unpadded Base64: + + UNPADDED_BASE64("") = "" + UNPADDED_BASE64("f") = "Zg" + UNPADDED_BASE64("fo") = "Zm8" + UNPADDED_BASE64("foo") = "Zm9v" + UNPADDED_BASE64("foob") = "Zm9vYg" + UNPADDED_BASE64("fooba") = "Zm9vYmE" + UNPADDED_BASE64("foobar") = "Zm9vYmFy" + +When decoding Base64, implementations SHOULD accept input with or +without padding characters wherever possible, to ensure maximum +interoperability. + +# Signing JSON + +Various points in the Matrix specification require JSON objects to be +cryptographically signed. This requires us to encode the JSON as a +binary string. Unfortunately the same JSON can be encoded in different +ways by changing how much white space is used or by changing the order +of keys within objects. + +Signing an object therefore requires it to be encoded as a sequence of +bytes using [Canonical JSON](#canonical-json), computing the signature +for that sequence and then adding the signature to the original JSON +object. + +## Canonical JSON + +We define the canonical JSON encoding for a value to be the shortest +UTF-8 JSON encoding with dictionary keys lexicographically sorted by +Unicode codepoint. Numbers in the JSON must be integers in the range +`[-(2**53)+1, (2**53)-1]`. + +We pick UTF-8 as the encoding as it should be available to all platforms +and JSON received from the network is likely to be already encoded using +UTF-8. We sort the keys to give a consistent ordering. We force integers +to be in the range where they can be accurately represented using IEEE +double precision floating point numbers since a number of JSON libraries +represent all numbers using this representation. + +Warning + +Events in room versions 1, 2, 3, 4, and 5 might not be fully compliant +with these restrictions. Servers SHOULD be capable of handling JSON +which is considered invalid by these restrictions where possible. + +The most notable consideration is that integers might not be in the +range specified above. + +Note + +Float values are not permitted by this encoding. + + import json + + def canonical_json(value): + return json.dumps( + value, + # Encode code-points outside of ASCII as UTF-8 rather than \u escapes + ensure_ascii=False, + # Remove unnecessary white space. + separators=(',',':'), + # Sort the keys of dictionaries. + sort_keys=True, + # Encode the resulting Unicode as UTF-8 bytes. + ).encode("UTF-8") + +### Grammar + +Adapted from the grammar in +removing insignificant whitespace, fractions, exponents and redundant +character escapes. + + value = false / null / true / object / array / number / string + false = %x66.61.6c.73.65 + null = %x6e.75.6c.6c + true = %x74.72.75.65 + object = %x7B [ member *( %x2C member ) ] %7D + member = string %x3A value + array = %x5B [ value *( %x2C value ) ] %5B + number = [ %x2D ] int + int = %x30 / ( %x31-39 *digit ) + digit = %x30-39 + string = %x22 *char %x22 + char = unescaped / %x5C escaped + unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + escaped = %x22 ; " quotation mark U+0022 + / %x5C ; \ reverse solidus U+005C + / %x62 ; b backspace U+0008 + / %x66 ; f form feed U+000C + / %x6E ; n line feed U+000A + / %x72 ; r carriage return U+000D + / %x74 ; t tab U+0009 + / %x75.30.30.30 (%x30-37 / %x62 / %x65-66) ; u000X + / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X + +### Examples + +To assist in the development of compatible implementations, the +following test values may be useful for verifying the canonical +transformation code. + +Given the following JSON object: + + {} + +The following canonical JSON should be produced: + + {} + +Given the following JSON object: + + { + "one": 1, + "two": "Two" + } + +The following canonical JSON should be produced: + + {"one":1,"two":"Two"} + +Given the following JSON object: + + { + "b": "2", + "a": "1" + } + +The following canonical JSON should be produced: + + {"a":"1","b":"2"} + +Given the following JSON object: + + {"b":"2","a":"1"} + +The following canonical JSON should be produced: + + {"a":"1","b":"2"} + +Given the following JSON object: + + { + "auth": { + "success": true, + "mxid": "@john.doe:example.com", + "profile": { + "display_name": "John Doe", + "three_pids": [ + { + "medium": "email", + "address": "john.doe@example.org" + }, + { + "medium": "msisdn", + "address": "123456789" + } + ] + } + } + } + +The following canonical JSON should be produced: + + {"auth":{"mxid":"@john.doe:example.com","profile":{"display_name":"John Doe","three_pids":[{"address":"john.doe@example.org","medium":"email"},{"address":"123456789","medium":"msisdn"}]},"success":true}} + +Given the following JSON object: + + { + "a": "日本語" + } + +The following canonical JSON should be produced: + + {"a":"日本語"} + +Given the following JSON object: + + { + "本": 2, + "日": 1 + } + +The following canonical JSON should be produced: + + {"日":1,"本":2} + +Given the following JSON object: + + { + "a": "\u65E5" + } + +The following canonical JSON should be produced: + + {"a":"日"} + +Given the following JSON object: + + { + "a": null + } + +The following canonical JSON should be produced: + + {"a":null} + +## Signing Details + +JSON is signed by encoding the JSON object without `signatures` or keys +grouped as `unsigned`, using the canonical encoding described above. The +JSON bytes are then signed using the signature algorithm and the +signature is encoded using [unpadded Base64](). The resulting base64 +signature is added to an object under the *signing key identifier* which +is added to the `signatures` object under the name of the entity signing +it which is added back to the original JSON object along with the +`unsigned` object. + +The *signing key identifier* is the concatenation of the *signing +algorithm* and a *key identifier*. The *signing algorithm* identifies +the algorithm used to sign the JSON. The currently supported value for +*signing algorithm* is `ed25519` as implemented by NACL +(). The *key identifier* is used to distinguish +between different signing keys used by the same entity. + +The `unsigned` object and the `signatures` object are not covered by the +signature. Therefore intermediate entities can add unsigned data such as +timestamps and additional signatures. + + { + "name": "example.org", + "signing_keys": { + "ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ" + }, + "unsigned": { + "age_ts": 922834800000 + }, + "signatures": { + "example.org": { + "ed25519:1": "s76RUgajp8w172am0zQb/iPTHsRnb4SkrzGoeCOSFfcBY2V/1c8QfrmdXHpvnc2jK5BD1WiJIxiMW95fMjK7Bw" + } + } + } + + def sign_json(json_object, signing_key, signing_name): + signatures = json_object.pop("signatures", {}) + unsigned = json_object.pop("unsigned", None) + + signed = signing_key.sign(encode_canonical_json(json_object)) + signature_base64 = encode_base64(signed.signature) + + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + signatures.setdefault(signing_name, {})[key_id] = signature_base64 + + json_object["signatures"] = signatures + if unsigned is not None: + json_object["unsigned"] = unsigned + + return json_object + +## Checking for a Signature + +To check if an entity has signed a JSON object an implementation does +the following: + +1. Checks if the `signatures` member of the object contains an entry + with the name of the entity. If the entry is missing then the check + fails. +2. Removes any *signing key identifiers* from the entry with algorithms + it doesn't understand. If there are no *signing key identifiers* + left then the check fails. +3. Looks up *verification keys* for the remaining *signing key + identifiers* either from a local cache or by consulting a trusted + key server. If it cannot find a *verification key* then the check + fails. +4. Decodes the base64 encoded signature bytes. If base64 decoding fails + then the check fails. +5. Removes the `signatures` and `unsigned` members of the object. +6. Encodes the remainder of the JSON object using the [Canonical + JSON](#canonical-json) encoding. +7. Checks the signature bytes against the encoded object using the + *verification key*. If this fails then the check fails. Otherwise + the check succeeds. + +# Identifier Grammar + +Some identifiers are specific to given room versions, please refer to +the [room versions specification](index.html#room-versions) for more +information. + +## Server Name + +A homeserver is uniquely identified by its server name. This value is +used in a number of identifiers, as described below. + +The server name represents the address at which the homeserver in +question can be reached by other homeservers. All valid server names are +included by the following grammar: + + server_name = hostname [ ":" port ] + + port = 1*5DIGIT + + hostname = IPv4address / "[" IPv6address "]" / dns-name + + IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + + IPv6address = 2*45IPv6char + + IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." + ; 0-9, A-F, a-f, :, . + + dns-name = 1*255dns-char + + dns-char = DIGIT / ALPHA / "-" / "." + +— in other words, the server name is the hostname, followed by an +optional numeric port specifier. The hostname may be a dotted-quad IPv4 +address literal, an IPv6 address literal surrounded with square +brackets, or a DNS name. + +IPv4 literals must be a sequence of four decimal numbers in the range 0 +to 255, separated by `.`. IPv6 literals must be as specified by +[RFC3513, section 2.2](https://tools.ietf.org/html/rfc3513#section-2.2). + +DNS names for use with Matrix should follow the conventional +restrictions for internet hostnames: they should consist of a series of +labels separated by `.`, where each label consists of the alphanumeric +characters or hyphens. + +Examples of valid server names are: + +- `matrix.org` +- `matrix.org:8888` +- `1.2.3.4` (IPv4 literal) +- `1.2.3.4:1234` (IPv4 literal with explicit port) +- `[1234:5678::abcd]` (IPv6 literal) +- `[1234:5678::abcd]:5678` (IPv6 literal with explicit port) + +Note + +This grammar is based on the standard for internet host names, as +specified by [RFC1123, section +2.1](https://tools.ietf.org/html/rfc1123#page-13), with an extension for +IPv6 literals. + +Server names must be treated case-sensitively: in other words, +`@user:matrix.org` is a different person from `@user:MATRIX.ORG`. + +Some recommendations for a choice of server name follow: + +- The length of the complete server name should not exceed 230 + characters. +- Server names should not use upper-case characters. + +## Common Identifier Format + +The Matrix protocol uses a common format to assign unique identifiers to +a number of entities, including users, events and rooms. Each identifier +takes the form: + + &string + +where `&` represents a 'sigil' character; `string` is the string which +makes up the identifier. + +The sigil characters are as follows: + +- `@`: User ID +- `!`: Room ID +- `$`: Event ID +- `+`: Group ID +- `#`: Room alias + +User IDs, group IDs, room IDs, room aliases, and sometimes event IDs +take the form: + + &localpart:domain + +where `domain` is the [server name](#server-name) of the homeserver +which allocated the identifier, and `localpart` is an identifier +allocated by that homeserver. + +The precise grammar defining the allowable format of an identifier +depends on the type of identifier. For example, event IDs can sometimes +be represented with a `domain` component under some conditions - see the +[Event IDs](#room-ids-and-event-ids) section below for more information. + +### User Identifiers + +Users within Matrix are uniquely identified by their Matrix user ID. The +user ID is namespaced to the homeserver which allocated the account and +has the form: + + @localpart:domain + +The `localpart` of a user ID is an opaque identifier for that user. It +MUST NOT be empty, and MUST contain only the characters `a-z`, `0-9`, +`.`, `_`, `=`, `-`, and `/`. + +The `domain` of a user ID is the [server name](#server-name) of the +homeserver which allocated the account. + +The length of a user ID, including the `@` sigil and the domain, MUST +NOT exceed 255 characters. + +The complete grammar for a legal user ID is: + + user_id = "@" user_id_localpart ":" server_name + user_id_localpart = 1*user_id_char + user_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + +Rationale + +A number of factors were considered when defining the allowable +characters for a user ID. + +Firstly, we chose to exclude characters outside the basic US-ASCII +character set. User IDs are primarily intended for use as an identifier +at the protocol level, and their use as a human-readable handle is of +secondary benefit. Furthermore, they are useful as a last-resort +differentiator between users with similar display names. Allowing the +full Unicode character set would make very difficult for a human to +distinguish two similar user IDs. The limited character set used has the +advantage that even a user unfamiliar with the Latin alphabet should be +able to distinguish similar user IDs manually, if somewhat laboriously. + +We chose to disallow upper-case characters because we do not consider it +valid to have two user IDs which differ only in case: indeed it should +be possible to reach `@user:matrix.org` as `@USER:matrix.org`. However, +user IDs are necessarily used in a number of situations which are +inherently case-sensitive (notably in the `state_key` of `m.room.member` +events). Forbidding upper-case characters (and requiring homeservers to +downcase usernames when creating user IDs for new users) is a relatively +simple way to ensure that `@USER:matrix.org` cannot refer to a different +user to `@user:matrix.org`. + +Finally, we decided to restrict the allowable punctuation to a very +basic set to reduce the possibility of conflicts with special characters +in various situations. For example, "\*" is used as a wildcard in some +APIs (notably the filter API), so it cannot be a legal user ID +character. + +The length restriction is derived from the limit on the length of the +`sender` key on events; since the user ID appears in every event sent by +the user, it is limited to ensure that the user ID does not dominate +over the actual content of the events. + +Matrix user IDs are sometimes informally referred to as MXIDs. + +#### Historical User IDs + +Older versions of this specification were more tolerant of the +characters permitted in user ID localparts. There are currently active +users whose user IDs do not conform to the permitted character set, and +a number of rooms whose history includes events with a `sender` which +does not conform. In order to handle these rooms successfully, clients +and servers MUST accept user IDs with localparts from the expanded +character set: + + extended_user_id_char = %x21-39 / %x3B-7E ; all ASCII printing chars except : + +#### Mapping from other character sets + +In certain circumstances it will be desirable to map from a wider +character set onto the limited character set allowed in a user ID +localpart. Examples include a homeserver creating a user ID for a new +user based on the username passed to `/register`, or a bridge mapping +user ids from another protocol. + +Implementations are free to do this mapping however they choose. Since +the user ID is opaque except to the implementation which created it, the +only requirement is that the implementation can perform the mapping +consistently. However, we suggest the following algorithm: + +1. Encode character strings as UTF-8. +2. Convert the bytes `A-Z` to lower-case. + - In the case where a bridge must be able to distinguish two + different users with ids which differ only by case, escape + upper-case characters by prefixing with `_` before downcasing. + For example, `A` becomes `_a`. Escape a real `_` with a second + `_`. +3. Encode any remaining bytes outside the allowed character set, as + well as `=`, as their hexadecimal value, prefixed with `=`. For + example, `#` becomes `=23`; `á` becomes `=c3=a1`. + +Rationale + +The suggested mapping is an attempt to preserve human-readability of +simple ASCII identifiers (unlike, for example, base-32), whilst still +allowing representation of *any* character (unlike punycode, which +provides no way to encode ASCII punctuation). + +### Room IDs and Event IDs + +A room has exactly one room ID. A room ID has the format: + + !opaque_id:domain + +An event has exactly one event ID. The format of an event ID depends +upon the [room version specification](index.html#room-versions). + +The `domain` of a room ID is the [server name](#server-name) of the +homeserver which created the room/event. The domain is used only for +namespacing to avoid the risk of clashes of identifiers between +different homeservers. There is no implication that the room or event in +question is still available at the corresponding homeserver. + +Event IDs and Room IDs are case-sensitive. They are not meant to be +human-readable. They are intended to be treated as fully opaque strings +by clients. + +### Group Identifiers + +Groups within Matrix are uniquely identified by their group ID. The +group ID is namespaced to the group server which hosts this group and +has the form: + + +localpart:domain + +The `localpart` of a group ID is an opaque identifier for that group. It +MUST NOT be empty, and MUST contain only the characters `a-z`, `0-9`, +`.`, `_`, `=`, `-`, and `/`. + +The `domain` of a group ID is the [server name](#server-name) of the +group server which hosts this group. + +The length of a group ID, including the `+` sigil and the domain, MUST +NOT exceed 255 characters. + +The complete grammar for a legal group ID is: + + group_id = "+" group_id_localpart ":" server_name + group_id_localpart = 1*group_id_char + group_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + +### Room Aliases + +A room may have zero or more aliases. A room alias has the format: + + #room_alias:domain + +The `domain` of a room alias is the [server name](#server-name) of the +homeserver which created the alias. Other servers may contact this +homeserver to look up the alias. + +Room aliases MUST NOT exceed 255 bytes (including the `#` sigil and the +domain). + +### matrix.to navigation + +Note + +This namespacing is in place pending a `matrix://` (or similar) URI +scheme. This is **not** meant to be interpreted as an available web +service - see below for more details. + +Rooms, users, aliases, and groups may be represented as a "matrix.to" +URI. This URI can be used to reference particular objects in a given +context, such as mentioning a user in a message or linking someone to a +particular point in the room's history (a permalink). + +A matrix.to URI has the following format, based upon the specification +defined in RFC 3986: + +> <identifier>/<extra +> parameter>?<additional arguments> + +The identifier may be a room ID, room alias, user ID, or group ID. The +extra parameter is only used in the case of permalinks where an event ID +is referenced. The matrix.to URI, when referenced, must always start +with `https://matrix.to/#/` followed by the identifier. + +The `` and the preceding question mark are +optional and only apply in certain circumstances, documented below. + +Clients should not rely on matrix.to URIs falling back to a web server +if accessed and instead should perform some sort of action within the +client. For example, if the user were to click on a matrix.to URI for a +room alias, the client may open a view for the user to participate in +the room. + +The components of the matrix.to URI (`` and +``) are to be percent-encoded as per RFC 3986. + +Examples of matrix.to URIs are: + +- Room alias: `https://matrix.to/#/%23somewhere%3Aexample.org` +- Room: `https://matrix.to/#/!somewhere%3Aexample.org` +- Permalink by room: + `https://matrix.to/#/!somewhere%3Aexample.org/%24event%3Aexample.org` +- Permalink by room alias: + `https://matrix.to/#/%23somewhere:example.org/%24event%3Aexample.org` +- User: `https://matrix.to/#/%40alice%3Aexample.org` +- Group: `https://matrix.to/#/%2Bexample%3Aexample.org` + +Note + +Historically, clients have not produced URIs which are fully encoded. +Clients should try to interpret these cases to the best of their +ability. For example, an unencoded room alias should still work within +the client if possible. + +Note + +Clients should be aware that decoding a matrix.to URI may result in +extra slashes appearing due to some [room +versions](index.html#room-versions). These slashes should normally be +encoded when producing matrix.to URIs, however. + +#### Routing + +Room IDs are not routable on their own as there is no reliable domain to +send requests to. This is partially mitigated with the addition of a +`via` argument on a matrix.to URI, however the problem of routability is +still present. Clients should do their best to route Room IDs to where +they need to go, however they should also be aware of [issue +\#1579](https://github.com/matrix-org/matrix-doc/issues/1579). + +A room (or room permalink) which isn't using a room alias should supply +at least one server using `via` in the ``, like +so: +`https://matrix.to/!somewhere%3Aexample.org?via=example.org&via=alt.example.org`. +The parameter can be supplied multiple times to specify multiple servers +to try. + +The values of `via` are intended to be passed along as the `server_name` +parameters on the Client Server `/join` API. + +When generating room links and permalinks, the application should pick +servers which have a high probability of being in the room in the +distant future. How these servers are picked is left as an +implementation detail, however the current recommendation is to pick 3 +unique servers based on the following criteria: + +- The first server should be the server of the highest power level + user in the room, provided they are at least power level 50. If no + user meets this criterion, pick the most popular server in the room + (most joined users). The rationale for not picking users with power + levels under 50 is that they are unlikely to be around into the + distant future while higher ranking users (and therefore servers) + are less likely to give up their power and move somewhere else. Most + rooms in the public federation have a power level 100 user and have + not deviated from the default structure where power level 50 users + have moderator-style privileges. +- The second server should be the next highest server by population, + or the first highest by population if the first server was based on + a user's power level. The rationale for picking popular servers is + that the server is unlikely to be removed as the room naturally + grows in membership due to that server joining users. The server + could be refused participation in the future due to server ACLs or + similar, however the chance of that happening to a server which is + organically joining the room is unlikely. +- The third server should be the next highest server by population. +- Servers which are blocked due to server ACLs should never be chosen. +- Servers which are IP addresses should never be chosen. Servers which + use a domain name are less likely to be unroutable in the future + whereas IP addresses cannot be pointed to a different location and + therefore higher risk options. +- All 3 servers should be unique from each other. If the room does not + have enough users to supply 3 servers, the application should only + specify the servers it can. For example, a room with only 2 users in + it would result in maximum 2 `via` parameters. + +# 3PID Types + +Third Party Identifiers (3PIDs) represent identifiers on other +namespaces that might be associated with a particular person. They +comprise a tuple of `medium` which is a string that identifies the +namespace in which the identifier exists, and an `address`: a string +representing the identifier in that namespace. This must be a canonical +form of the identifier, *i.e.* if multiple strings could represent the +same identifier, only one of these strings must be used in a 3PID +address, in a well-defined manner. + +For example, for e-mail, the `medium` is 'email' and the `address` would +be the email address, *e.g.* the string `bob@example.com`. Since domain +resolution is case-insensitive, the email address `bob@Example.com` is +also has the 3PID address of `bob@example.com` (without the capital 'e') +rather than `bob@Example.com`. + +The namespaces defined by this specification are listed below. More +namespaces may be defined in future versions of this specification. + +## E-Mail + +Medium: `email` + +Represents E-Mail addresses. The `address` is the raw email address in +`user@domain` form with the domain in lowercase. It must not contain +other text such as real name, angle brackets or a mailto: prefix. + +## PSTN Phone numbers + +Medium: `msisdn` + +Represents telephone numbers on the public switched telephone network. +The `address` is the telephone number represented as a MSISDN (Mobile +Station International Subscriber Directory Number) as defined by the +E.164 numbering plan. Note that MSISDNs do not include a leading '+'. + +# Security Threat Model + +## Denial of Service + +The attacker could attempt to prevent delivery of messages to or from +the victim in order to: + +- Disrupt service or marketing campaign of a commercial competitor. +- Censor a discussion or censor a participant in a discussion. +- Perform general vandalism. + +### Threat: Resource Exhaustion + +An attacker could cause the victim's server to exhaust a particular +resource (e.g. open TCP connections, CPU, memory, disk storage) + +### Threat: Unrecoverable Consistency Violations + +An attacker could send messages which created an unrecoverable +"split-brain" state in the cluster such that the victim's servers could +no longer derive a consistent view of the chatroom state. + +### Threat: Bad History + +An attacker could convince the victim to accept invalid messages which +the victim would then include in their view of the chatroom history. +Other servers in the chatroom would reject the invalid messages and +potentially reject the victims messages as well since they depended on +the invalid messages. + +### Threat: Block Network Traffic + +An attacker could try to firewall traffic between the victim's server +and some or all of the other servers in the chatroom. + +### Threat: High Volume of Messages + +An attacker could send large volumes of messages to a chatroom with the +victim making the chatroom unusable. + +### Threat: Banning users without necessary authorisation + +An attacker could attempt to ban a user from a chatroom without the +necessary authorisation. + +## Spoofing + +An attacker could try to send a message claiming to be from the victim +without the victim having sent the message in order to: + +- Impersonate the victim while performing illicit activity. +- Obtain privileges of the victim. + +### Threat: Altering Message Contents + +An attacker could try to alter the contents of an existing message from +the victim. + +### Threat: Fake Message "origin" Field + +An attacker could try to send a new message purporting to be from the +victim with a phony "origin" field. + +## Spamming + +The attacker could try to send a high volume of solicited or unsolicited +messages to the victim in order to: + +- Find victims for scams. +- Market unwanted products. + +### Threat: Unsolicited Messages + +An attacker could try to send messages to victims who do not wish to +receive them. + +### Threat: Abusive Messages + +An attacker could send abusive or threatening messages to the victim + +## Spying + +The attacker could try to access message contents or metadata for +messages sent by the victim or to the victim that were not intended to +reach the attacker in order to: + +- Gain sensitive personal or commercial information. +- Impersonate the victim using credentials contained in the messages. + (e.g. password reset messages) +- Discover who the victim was talking to and when. + +### Threat: Disclosure during Transmission + +An attacker could try to expose the message contents or metadata during +transmission between the servers. + +### Threat: Disclosure to Servers Outside Chatroom + +An attacker could try to convince servers within a chatroom to send +messages to a server it controls that was not authorised to be within +the chatroom. + +### Threat: Disclosure to Servers Within Chatroom + +An attacker could take control of a server within a chatroom to expose +message contents or metadata for messages in that room. + +# Cryptographic Test Vectors + +To assist in the development of compatible implementations, the +following test values may be useful for verifying the cryptographic +event signing code. + +## Signing Key + +The following test vectors all use the 32-byte value given by the +following Base64-encoded string as the seed for generating the `ed25519` +signing key: + + SIGNING_KEY_SEED = decode_base64( + "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" + ) + +In each case, the server name and key ID are as follows: + + SERVER_NAME = "domain" + + KEY_ID = "ed25519:1" + +## JSON Signing + +Given an empty JSON object: + + {} + +The JSON signing algorithm should emit the following signed data: + + { + "signatures": { + "domain": { + "ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ" + } + } + } + +Given the following JSON object with data values in it: + + { + "one": 1, + "two": "Two" + } + +The JSON signing algorithm should emit the following signed JSON: + + { + "one": 1, + "signatures": { + "domain": { + "ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw" + } + }, + "two": "Two" + } + +## Event Signing + +Given the following minimally-sized event: + + { + "room_id": "!x:domain", + "sender": "@a:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "signatures": {}, + "hashes": {}, + "type": "X", + "content": {}, + "prev_events": [], + "auth_events": [], + "depth": 3, + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + + { + "auth_events": [], + "content": {}, + "depth": 3, + "hashes": { + "sha256": "5jM4wQpv6lnBo7CLIghJuHdW+s2CMBJPUOGOC89ncos" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "prev_events": [], + "room_id": "!x:domain", + "sender": "@a:domain", + "signatures": { + "domain": { + "ed25519:1": "KxwGjPSDEtvnFgU00fwFz+l6d2pJM6XBIaMEn81SXPTRl16AqLAYqfIReFGZlHi5KLjAWbOoMszkwsQma+lYAg" + } + }, + "type": "X", + "unsigned": { + "age_ts": 1000000 + } + } + +Given the following event containing redactable content: + + { + "content": { + "body": "Here is the message content" + }, + "event_id": "$0:domain", + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": {}, + "unsigned": { + "age_ts": 1000000 + } + } + +The event signing algorithm should emit the following signed event: + + { + "content": { + "body": "Here is the message content" + }, + "event_id": "$0:domain", + "hashes": { + "sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g" + }, + "origin": "domain", + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:domain", + "sender": "@u:domain", + "signatures": { + "domain": { + "ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" + } + }, + "unsigned": { + "age_ts": 1000000 + } + } diff --git a/content/application-service-api.md b/content/application-service-api.md index d529cdf4..9d30346a 100644 --- a/content/application-service-api.md +++ b/content/application-service-api.md @@ -3,3 +3,473 @@ title: "Application Service API" weight: 30 type: docs --- + +# Application Service API + +{{unstable\_warning\_block\_APPSERVICE\_RELEASE\_LABEL}} + +The Matrix client-server API and server-server APIs provide the means to +implement a consistent self-contained federated messaging fabric. +However, they provide limited means of implementing custom server-side +behaviour in Matrix (e.g. gateways, filters, extensible hooks etc). The +Application Service API (AS API) defines a standard API to allow such +extensible functionality to be implemented irrespective of the +underlying homeserver implementation. + +Table of Contents + +## Changelog + +**Version: %APPSERVICE\_RELEASE\_LABEL%** + +{{application\_service\_changelog}} + +This version of the specification is generated from +[matrix-doc](https://github.com/matrix-org/matrix-doc) as of Git commit +[{{git\_version}}](https://github.com/matrix-org/matrix-doc/tree/%7B%7Bgit_rev%7D%7D). + +For the full historical changelog, see + + +### Other versions of this specification + +The following other versions are also available, in reverse +chronological order: + +- [HEAD](https://matrix.org/docs/spec/application_service/unstable.html): + Includes all changes since the latest versioned release. +- [r0.1.1](https://matrix.org/docs/spec/application_service/r0.1.1.html) +- [r0.1.0](https://matrix.org/docs/spec/application_service/r0.1.0.html) + +## Application Services + +Application services are passive and can only observe events from +homeserver. They can inject events into rooms they are participating in. +They cannot prevent events from being sent, nor can they modify the +content of the event being sent. In order to observe events from a +homeserver, the homeserver needs to be configured to pass certain types +of traffic to the application service. This is achieved by manually +configuring the homeserver with information about the application +service. + +### Registration + +Note + +Previously, application services could register with a homeserver via +HTTP APIs. This was removed as it was seen as a security risk. A +compromised application service could re-register for a global `*` regex +and sniff *all* traffic on the homeserver. To protect against this, +application services now have to register via configuration files which +are linked to the homeserver configuration file. The addition of +configuration files allows homeserver admins to sanity check the +registration for suspicious regex strings. + +Application services register "namespaces" of user IDs, room aliases and +room IDs. These namespaces are represented as regular expressions. An +application service is said to be "interested" in a given event if one +of the IDs in the event match the regular expression provided by the +application service, such as the room having an alias or ID in the +relevant namespaces. Similarly, the application service is said to be +interested in a given event if one of the application service's +namespaced users is the target of the event, or is a joined member of +the room where the event occurred. + +An application service can also state whether they should be the only +ones who can manage a specified namespace. This is referred to as an +"exclusive" namespace. An exclusive namespace prevents humans and other +application services from creating/deleting entities in that namespace. +Typically, exclusive namespaces are used when the rooms represent real +rooms on another service (e.g. IRC). Non-exclusive namespaces are used +when the application service is merely augmenting the room itself (e.g. +providing logging or searching facilities). Namespaces are represented +by POSIX extended regular expressions and look like: + + users: + - exclusive: true + regex: "@_irc_bridge_.*" + +Application services may define the following namespaces (with none +being explicitly required): + + ++++ + + + + + + + + + + + + + + + + + + + + +
NameDescription
usersEvents which are sent from certain users.
aliasesEvents which are sent in rooms with certain room aliases.
roomsEvents which are sent in rooms with certain room IDs.
+ +Each individual namespace MUST declare the following fields: + + ++++ + + + + + + + + + + + + + + + + +
NameDescription
exclusiveRequired A true or false value stating whether this application service has exclusive access to events within this namespace.
regexRequired A regular expression defining which values this namespace includes.
+ +Exclusive user and alias namespaces should begin with an underscore +after the sigil to avoid collisions with other users on the homeserver. +Application services should additionally attempt to identify the service +they represent in the reserved namespace. For example, `@_irc_.*` would +be a good namespace to register for an application service which deals +with IRC. + +The registration is represented by a series of key-value pairs, which +this specification will present as YAML. See below for the possible +options along with their explanation: + + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
idRequired. A unique, user-defined ID of the application service which will never change.
urlRequired. The URL for the application service. May include a path after the domain name. Optionally set to null if no traffic is required.
as_tokenRequired. A unique token for application services to use to authenticate requests to Homeservers.
hs_tokenRequired. A unique token for Homeservers to use to authenticate requests to application services.
sender_localpartRequired. The localpart of the user associated with the application service.
namespacesRequired. A list of users, aliases and rooms namespaces that the application service controls.
rate_limitedWhether requests from masqueraded users are rate-limited. The sender is excluded.
protocolsThe external protocols which the application service provides (e.g. IRC).
+ +An example registration file for an IRC-bridging application service is +below: + + id: "IRC Bridge" + url: "http://127.0.0.1:1234" + as_token: "30c05ae90a248a4188e620216fa72e349803310ec83e2a77b34fe90be6081f46" + hs_token: "312df522183efd404ec1cd22d2ffa4bbc76a8c1ccf541dd692eef281356bb74e" + sender_localpart: "_irc_bot" # Will result in @_irc_bot:example.org + namespaces: + users: + - exclusive: true + regex: "@_irc_bridge_.*" + aliases: + - exclusive: false + regex: "#_irc_bridge_.*" + rooms: [] + +Warning + +If the homeserver in question has multiple application services, each +`as_token` and `id` MUST be unique per application service as these are +used to identify the application service. The homeserver MUST enforce +this. + +### Homeserver -> Application Service API + +#### Authorization + +Homeservers MUST include a query parameter named `access_token` +containing the `hs_token` from the application service's registration +when making requests to the application service. Application services +MUST verify the provided `access_token` matches their known `hs_token`, +failing the request with an `M_FORBIDDEN` error if it does not match. + +#### Legacy routes + +Previous drafts of the application service specification had a mix of +endpoints that have been used in the wild for a significant amount of +time. The application service specification now defines a version on all +endpoints to be more compatible with the rest of the Matrix +specification and the future. + +Homeservers should attempt to use the specified endpoints first when +communicating with application services. However, if the application +service receives an HTTP status code that does not indicate success +(i.e.: 404, 500, 501, etc) then the homeserver should fall back to the +older endpoints for the application service. + +The older endpoints have the exact same request body and response +format, they just belong at a different path. The equivalent path for +each is as follows: + +- `/_matrix/app/v1/transactions/{txnId}` should fall back to + `/transactions/{txnId}` +- `/_matrix/app/v1/users/{userId}` should fall back to + `/users/{userId}` +- `/_matrix/app/v1/rooms/{roomAlias}` should fall back to + `/rooms/{roomAlias}` +- `/_matrix/app/v1/thirdparty/protocol/{protocol}` should fall back to + `/_matrix/app/unstable/thirdparty/protocol/{protocol}` +- `/_matrix/app/v1/thirdparty/user/{user}` should fall back to + `/_matrix/app/unstable/thirdparty/user/{user}` +- `/_matrix/app/v1/thirdparty/location/{location}` should fall back to + `/_matrix/app/unstable/thirdparty/location/{location}` +- `/_matrix/app/v1/thirdparty/user` should fall back to + `/_matrix/app/unstable/thirdparty/user` +- `/_matrix/app/v1/thirdparty/location` should fall back to + `/_matrix/app/unstable/thirdparty/location` + +Homeservers should periodically try again for the newer endpoints +because the application service may have been updated. + +#### Pushing events + +The application service API provides a transaction API for sending a +list of events. Each list of events includes a transaction ID, which +works as follows: + + Typical + HS ---> AS : Homeserver sends events with transaction ID T. + <--- : Application Service sends back 200 OK. + + AS ACK Lost + HS ---> AS : Homeserver sends events with transaction ID T. + <-/- : AS 200 OK is lost. + HS ---> AS : Homeserver retries with the same transaction ID of T. + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). + +The events sent to the application service should be linearised, as if +they were from the event stream. The homeserver MUST maintain a queue of +transactions to send to the application service. If the application +service cannot be reached, the homeserver SHOULD backoff exponentially +until the application service is reachable again. As application +services cannot *modify* the events in any way, these requests can be +made without blocking other aspects of the homeserver. Homeservers MUST +NOT alter (e.g. add more) events they were going to send within that +transaction ID on retries, as the application service may have already +processed the events. + +{{transactions\_as\_http\_api}} + +#### Querying + +The application service API includes two querying APIs: for room aliases +and for user IDs. The application service SHOULD create the queried +entity if it desires. During this process, the application service is +blocking the homeserver until the entity is created and configured. If +the homeserver does not receive a response to this request, the +homeserver should retry several times before timing out. This should +result in an HTTP status 408 "Request Timeout" on the client which +initiated this request (e.g. to join a room alias). + +Rationale + +Blocking the homeserver and expecting the application service to create +the entity using the client-server API is simpler and more flexible than +alternative methods such as returning an initial sync style JSON blob +and get the HS to provision the room/user. This also meant that there +didn't need to be a "backchannel" to inform the application service +about information about the entity such as room ID to room alias +mappings. + +{{query\_user\_as\_http\_api}} + +{{query\_room\_as\_http\_api}} + +#### Third party networks + +Application services may declare which protocols they support via their +registration configuration for the homeserver. These networks are +generally for third party services such as IRC that the application +service is managing. Application services may populate a Matrix room +directory for their registered protocols, as defined in the +Client-Server API Extensions. + +Each protocol may have several "locations" (also known as "third party +locations" or "3PLs"). A location within a protocol is a place in the +third party network, such as an IRC channel. Users of the third party +network may also be represented by the application service. + +Locations and users can be searched by fields defined by the application +service, such as by display name or other attribute. When clients +request the homeserver to search in a particular "network" (protocol), +the search fields will be passed along to the application service for +filtering. + +{{protocols\_as\_http\_api}} + +### Client-Server API Extensions + +Application services can use a more powerful version of the +client-server API by identifying itself as an application service to the +homeserver. + +Endpoints defined in this section MUST be supported by homeservers in +the client-server API as accessible only by application services. + +#### Identity assertion + +The client-server API infers the user ID from the `access_token` +provided in every request. To avoid the application service from having +to keep track of each user's access token, the application service +should identify itself to the Client-Server API by providing its +`as_token` for the `access_token` alongside the user the application +service would like to masquerade as. + +Inputs: +- Application service token (`as_token`) +- User ID in the AS namespace to act as. + +Notes: +- This applies to all aspects of the Client-Server API, except for + Account Management. +- The `as_token` is inserted into `access_token` which is usually + where the client token is, such as via the query string or + `Authorization` header. This is done on purpose to allow application + services to reuse client SDKs. +- The `access_token` should be supplied through the `Authorization` + header where possible to prevent the token appearing in HTTP request + logs by accident. + +The application service may specify the virtual user to act as through +use of a `user_id` query string parameter on the request. The user +specified in the query string must be covered by one of the application +service's `user` namespaces. If the parameter is missing, the homeserver +is to assume the application service intends to act as the user implied +by the `sender_localpart` property of the registration. + +An example request would be: + + GET /_matrix/client/%CLIENT_MAJOR_VERSION%/account/whoami?user_id=@_irc_user:example.org + Authorization: Bearer YourApplicationServiceTokenHere + +#### Timestamp massaging + +Previous drafts of the Application Service API permitted application +services to alter the timestamp of their sent events by providing a `ts` +query parameter when sending an event. This API has been excluded from +the first release due to design concerns, however some servers may still +support the feature. Please visit [issue +\#1585](https://github.com/matrix-org/matrix-doc/issues/1585) for more +information. + +#### Server admin style permissions + +The homeserver needs to give the application service *full control* over +its namespace, both for users and for room aliases. This means that the +AS should be able to create/edit/delete any room alias in its namespace, +as well as create/delete any user in its namespace. No additional API +changes need to be made in order for control of room aliases to be +granted to the AS. Creation of users needs API changes in order to: + +- Work around captchas. +- Have a 'passwordless' user. + +This involves bypassing the registration flows entirely. This is +achieved by including the `as_token` on a `/register` request, along +with a login type of `m.login.application_service` to set the desired +user ID without a password. + + POST /_matrix/client/%CLIENT_MAJOR_VERSION%/register + Authorization: Bearer YourApplicationServiceTokenHere + + Content: + { + type: "m.login.application_service", + username: "_irc_example" + } + +Application services which attempt to create users or aliases *outside* +of their defined namespaces will receive an error code `M_EXCLUSIVE`. +Similarly, normal users who attempt to create users or aliases *inside* +an application service-defined namespace will receive the same +`M_EXCLUSIVE` error code, but only if the application service has +defined the namespace as `exclusive`. + +#### Using `/sync` and `/events` + +Application services wishing to use `/sync` or `/events` from the +Client-Server API MUST do so with a virtual user (provide a `user_id` +via the query string). It is expected that the application service use +the transactions pushed to it to handle events rather than syncing with +the user implied by `sender_localpart`. + +#### Application service room directories + +Application services can maintain their own room directories for their +defined third party protocols. These room directories may be accessed by +clients through additional parameters on the `/publicRooms` +client-server endpoint. + +{{appservice\_room\_directory\_cs\_http\_api}} + +### Referencing messages from a third party network + +Application services should include an `external_url` in the `content` +of events it emits to indicate where the message came from. This +typically applies to application services that bridge other networks +into Matrix, such as IRC, where an HTTP URL may be available to +reference. + +Clients should provide users with a way to access the `external_url` if +it is present. Clients should additionally ensure the URL has a scheme +of `https` or `http` before making use of it. + +The presence of an `external_url` on an event does not necessarily mean +the event was sent from an application service. Clients should be wary +of the URL contained within, as it may not be a legitimate reference to +the event's source. diff --git a/content/client-server-api/_index.md b/content/client-server-api/_index.md index 06a44212..bb77b424 100644 --- a/content/client-server-api/_index.md +++ b/content/client-server-api/_index.md @@ -3,3 +3,1944 @@ title: "Client-Server API" weight: 10 type: docs --- + +# Client-Server API + +{{unstable\_warning\_block\_CLIENT\_RELEASE\_LABEL}} + +The client-server API provides a simple lightweight API to let clients +send messages, control rooms and synchronise conversation history. It is +designed to support both lightweight clients which store no state and +lazy-load data from the server as required - as well as heavyweight +clients which maintain a full local persistent copy of server state. + +Table of Contents + +## Changelog + +**Version: %CLIENT\_RELEASE\_LABEL%** + +{{client\_server\_changelog}} + +This version of the specification is generated from +[matrix-doc](https://github.com/matrix-org/matrix-doc) as of Git commit +[{{git\_version}}](https://github.com/matrix-org/matrix-doc/tree/%7B%7Bgit_rev%7D%7D). + +For the full historical changelog, see + + +### Other versions of this specification + +The following other versions are also available, in reverse +chronological order: + +- [HEAD](https://matrix.org/docs/spec/client_server/unstable.html): + Includes all changes since the latest versioned release. +- [r0.6.1](https://matrix.org/docs/spec/client_server/r0.6.1.html) +- [r0.6.0](https://matrix.org/docs/spec/client_server/r0.6.0.html) +- [r0.5.0](https://matrix.org/docs/spec/client_server/r0.5.0.html) +- [r0.4.0](https://matrix.org/docs/spec/client_server/r0.4.0.html) +- [r0.3.0](https://matrix.org/docs/spec/client_server/r0.3.0.html) +- [r0.2.0](https://matrix.org/docs/spec/client_server/r0.2.0.html) +- [r0.1.0](https://matrix.org/docs/spec/client_server/r0.1.0.html) +- [r0.0.1](https://matrix.org/docs/spec/r0.0.1/client_server.html) +- [r0.0.0](https://matrix.org/docs/spec/r0.0.0/client_server.html) +- [Legacy](https://matrix.org/docs/spec/legacy/#client-server-api): + The last draft before the spec was formally released in version + r0.0.0. + +## API Standards + +The mandatory baseline for client-server communication in Matrix is +exchanging JSON objects over HTTP APIs. HTTPS is recommended for +communication, although HTTP may be supported as a fallback to support +basic HTTP clients. More efficient optional transports will in future be +supported as optional extensions - e.g. a packed binary encoding over +stream-cipher encrypted TCP socket for low-bandwidth/low-roundtrip +mobile usage. For the default HTTP transport, all API calls use a +Content-Type of `application/json`. In addition, all strings MUST be +encoded as UTF-8. Clients are authenticated using opaque `access_token` +strings (see [Client Authentication](#client-authentication) for +details), passed as a query string parameter on all requests. + +The names of the API endpoints for the HTTP transport follow a +convention of using underscores to separate words (for example +`/delete_devices`). The key names in JSON objects passed over the API +also follow this convention. + +Note + +There are a few historical exceptions to this rule, such as +`/createRoom`. A future version of this specification will address the +inconsistency. + +Any errors which occur at the Matrix API level MUST return a "standard +error response". This is a JSON object which looks like: + + { + "errcode": "", + "error": "" + } + +The `error` string will be a human-readable error message, usually a +sentence explaining what went wrong. The `errcode` string will be a +unique string which can be used to handle an error message e.g. +`M_FORBIDDEN`. These error codes should have their namespace first in +ALL CAPS, followed by a single \_ to ease separating the namespace from +the error code. For example, if there was a custom namespace +`com.mydomain.here`, and a `FORBIDDEN` code, the error code should look +like `COM.MYDOMAIN.HERE_FORBIDDEN`. There may be additional keys +depending on the error, but the keys `error` and `errcode` MUST always +be present. + +Errors are generally best expressed by their error code rather than the +HTTP status code returned. When encountering the error code `M_UNKNOWN`, +clients should prefer the HTTP status code as a more reliable reference +for what the issue was. For example, if the client receives an error +code of `M_NOT_FOUND` but the request gave a 400 Bad Request status +code, the client should treat the error as if the resource was not +found. However, if the client were to receive an error code of +`M_UNKNOWN` with a 400 Bad Request, the client should assume that the +request being made was invalid. + +The common error codes are: + +`M_FORBIDDEN` +Forbidden access, e.g. joining a room without permission, failed login. + +`M_UNKNOWN_TOKEN` +The access token specified was not recognised. + +An additional response parameter, `soft_logout`, might be present on the +response for 401 HTTP status codes. See [the soft logout +section](#soft-logout) for more information. + +`M_MISSING_TOKEN` +No access token was specified for the request. + +`M_BAD_JSON` +Request contained valid JSON, but it was malformed in some way, e.g. +missing required keys, invalid values for keys. + +`M_NOT_JSON` +Request did not contain valid JSON. + +`M_NOT_FOUND` +No resource was found for this request. + +`M_LIMIT_EXCEEDED` +Too many requests have been sent in a short period of time. Wait a while +then try again. + +`M_UNKNOWN` +An unknown error has occurred. + +Other error codes the client might encounter are: + +`M_UNRECOGNIZED` +The server did not understand the request. + +`M_UNAUTHORIZED` +The request was not correctly authorized. Usually due to login failures. + +`M_USER_DEACTIVATED` +The user ID associated with the request has been deactivated. Typically +for endpoints that prove authentication, such as `/login`. + +`M_USER_IN_USE` +Encountered when trying to register a user ID which has been taken. + +`M_INVALID_USERNAME` +Encountered when trying to register a user ID which is not valid. + +`M_ROOM_IN_USE` +Sent when the room alias given to the `createRoom` API is already in +use. + +`M_INVALID_ROOM_STATE` +Sent when the initial state given to the `createRoom` API is invalid. + +`M_THREEPID_IN_USE` +Sent when a threepid given to an API cannot be used because the same +threepid is already in use. + +`M_THREEPID_NOT_FOUND` +Sent when a threepid given to an API cannot be used because no record +matching the threepid was found. + +`M_THREEPID_AUTH_FAILED` +Authentication could not be performed on the third party identifier. + +`M_THREEPID_DENIED` +The server does not permit this third party identifier. This may happen +if the server only permits, for example, email addresses from a +particular domain. + +`M_SERVER_NOT_TRUSTED` +The client's request used a third party server, e.g. identity server, +that this server does not trust. + +`M_UNSUPPORTED_ROOM_VERSION` +The client's request to create a room used a room version that the +server does not support. + +`M_INCOMPATIBLE_ROOM_VERSION` +The client attempted to join a room that has a version the server does +not support. Inspect the `room_version` property of the error response +for the room's version. + +`M_BAD_STATE` +The state change requested cannot be performed, such as attempting to +unban a user who is not banned. + +`M_GUEST_ACCESS_FORBIDDEN` +The room or resource does not permit guests to access it. + +`M_CAPTCHA_NEEDED` +A Captcha is required to complete the request. + +`M_CAPTCHA_INVALID` +The Captcha provided did not match what was expected. + +`M_MISSING_PARAM` +A required parameter was missing from the request. + +`M_INVALID_PARAM` +A parameter that was specified has the wrong value. For example, the +server expected an integer and instead received a string. + +`M_TOO_LARGE` +The request or entity was too large. + +`M_EXCLUSIVE` +The resource being requested is reserved by an application service, or +the application service making the request has not created the resource. + +`M_RESOURCE_LIMIT_EXCEEDED` +The request cannot be completed because the homeserver has reached a +resource limit imposed on it. For example, a homeserver held in a shared +hosting environment may reach a resource limit if it starts using too +much memory or disk space. The error MUST have an `admin_contact` field +to provide the user receiving the error a place to reach out to. +Typically, this error will appear on routes which attempt to modify +state (e.g.: sending messages, account data, etc) and not routes which +only read state (e.g.: `/sync`, get account data, etc). + +`M_CANNOT_LEAVE_SERVER_NOTICE_ROOM` +The user is unable to reject an invite to join the server notices room. +See the [Server Notices](#server-notices) module for more information. + +The client-server API typically uses `HTTP PUT` to submit requests with +a client-generated transaction identifier. This means that these +requests are idempotent. The scope of a transaction identifier is a +particular access token. It **only** serves to identify new requests +from retransmits. After the request has finished, the `{txnId}` value +should be changed (how is not specified; a monotonically increasing +integer is recommended). + +Some API endpoints may allow or require the use of `POST` requests +without a transaction ID. Where this is optional, the use of a `PUT` +request is strongly recommended. + +{{versions\_cs\_http\_api}} + +## Web Browser Clients + +It is realistic to expect that some clients will be written to be run +within a web browser or similar environment. In these cases, the +homeserver should respond to pre-flight requests and supply Cross-Origin +Resource Sharing (CORS) headers on all requests. + +Servers MUST expect that clients will approach them with `OPTIONS` +requests, allowing clients to discover the CORS headers. All endpoints +in this specification support the `OPTIONS` method, however the server +MUST NOT perform any logic defined for the endpoints when approached +with an `OPTIONS` request. + +When a client approaches the server with a request, the server should +respond with the CORS headers for that route. The recommended CORS +headers to be returned by servers on all requests are: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + +## Server Discovery + +In order to allow users to connect to a Matrix server without needing to +explicitly specify the homeserver's URL or other parameters, clients +SHOULD use an auto-discovery mechanism to determine the server's URL +based on a user's Matrix ID. Auto-discovery should only be done at login +time. + +In this section, the following terms are used with specific meanings: + +`PROMPT` +Retrieve the specific piece of information from the user in a way which +fits within the existing client user experience, if the client is +inclined to do so. Failure can take place instead if no good user +experience for this is possible at this point. + +`IGNORE` +Stop the current auto-discovery mechanism. If no more auto-discovery +mechanisms are available, then the client may use other methods of +determining the required parameters, such as prompting the user, or +using default values. + +`FAIL_PROMPT` +Inform the user that auto-discovery failed due to invalid/empty data and +`PROMPT` for the parameter. + +`FAIL_ERROR` +Inform the user that auto-discovery did not return any usable URLs. Do +not continue further with the current login process. At this point, +valid data was obtained, but no server is available to serve the client. +No further guess should be attempted and the user should make a +conscientious decision what to do next. + +### Well-known URI + +Note + +Servers hosting the `.well-known` JSON file SHOULD offer CORS headers, +as per the [CORS](#CORS) section in this specification. + +The `.well-known` method uses a JSON file at a predetermined location to +specify parameter values. The flow for this method is as follows: + +1. Extract the server name from the user's Matrix ID by splitting the + Matrix ID at the first colon. +2. Extract the hostname from the server name. +3. Make a GET request to `https://hostname/.well-known/matrix/client`. + 1. If the returned status code is 404, then `IGNORE`. + 2. If the returned status code is not 200, or the response body is + empty, then `FAIL_PROMPT`. + 3. Parse the response body as a JSON object + 1. If the content cannot be parsed, then `FAIL_PROMPT`. + 4. Extract the `base_url` value from the `m.homeserver` property. + This value is to be used as the base URL of the homeserver. + 1. If this value is not provided, then `FAIL_PROMPT`. + 5. Validate the homeserver base URL: + 1. Parse it as a URL. If it is not a URL, then `FAIL_ERROR`. + 2. Clients SHOULD validate that the URL points to a valid + homeserver before accepting it by connecting to the + `/_matrix/client/versions`\_ endpoint, ensuring that it does + not return an error, and parsing and validating that the + data conforms with the expected response format. If any step + in the validation fails, then `FAIL_ERROR`. Validation is + done as a simple check against configuration errors, in + order to ensure that the discovered address points to a + valid homeserver. + 6. If the `m.identity_server` property is present, extract the + `base_url` value for use as the base URL of the identity server. + Validation for this URL is done as in the step above, but using + `/_matrix/identity/api/v1` as the endpoint to connect to. If the + `m.identity_server` property is present, but does not have a + `base_url` value, then `FAIL_ERROR`. + +{{wellknown\_cs\_http\_api}} + +## Client Authentication + +Most API endpoints require the user to identify themselves by presenting +previously obtained credentials in the form of an `access_token` query +parameter or through an Authorization Header of `Bearer $access_token`. +An access token is typically obtained via the [Login](#login) or +[Registration](#Registration) processes. + +Note + +This specification does not mandate a particular format for the access +token. Clients should treat it as an opaque byte sequence. Servers are +free to choose an appropriate format. Server implementors may like to +investigate [macaroons](http://research.google.com/pubs/pub41892.html). + +### Using access tokens + +Access tokens may be provided in two ways, both of which the homeserver +MUST support: + +1. Via a query string parameter, `access_token=TheTokenHere`. +2. Via a request header, `Authorization: Bearer TheTokenHere`. + +Clients are encouraged to use the `Authorization` header where possible +to prevent the access token being leaked in access/HTTP logs. The query +string should only be used in cases where the `Authorization` header is +inaccessible for the client. + +When credentials are required but missing or invalid, the HTTP call will +return with a status of 401 and the error code, `M_MISSING_TOKEN` or +`M_UNKNOWN_TOKEN` respectively. + +### Relationship between access tokens and devices + +Client [devices](../index.html#devices) are closely related to access +tokens. Matrix servers should record which device each access token is +assigned to, so that subsequent requests can be handled correctly. + +By default, the [Login](#login) and [Registration](#Registration) +processes auto-generate a new `device_id`. A client is also free to +generate its own `device_id` or, provided the user remains the same, +reuse a device: in either case the client should pass the `device_id` in +the request body. If the client sets the `device_id`, the server will +invalidate any access token previously assigned to that device. There is +therefore at most one active access token assigned to each device at any +one time. + +### Soft logout + +When a request fails due to a 401 status code per above, the server can +include an extra response parameter, `soft_logout`, to indicate if the +client's persisted information can be retained. This defaults to +`false`, indicating that the server has destroyed the session. Any +persisted state held by the client, such as encryption keys and device +information, must not be reused and must be discarded. + +When `soft_logout` is true, the client can acquire a new access token by +specifying the device ID it is already using to the login API. In most +cases a `soft_logout: true` response indicates that the user's session +has expired on the server-side and the user simply needs to provide +their credentials again. + +In either case, the client's previously known access token will no +longer function. + +### User-Interactive Authentication API + +#### Overview + +Some API endpoints require authentication that interacts with the user. +The homeserver may provide many different ways of authenticating, such +as user/password auth, login via a single-sign-on server (SSO), etc. +This specification does not define how homeservers should authorise +their users but instead defines the standard interface which +implementations should follow so that ANY client can log in to ANY +homeserver. + +The process takes the form of one or more 'stages'. At each stage the +client submits a set of data for a given authentication type and awaits +a response from the server, which will either be a final success or a +request to perform an additional stage. This exchange continues until +the final success. + +For each endpoint, a server offers one or more 'flows' that the client +can use to authenticate itself. Each flow comprises a series of stages, +as described above. The client is free to choose which flow it follows, +however the flow's stages must be completed in order. Failing to follow +the flows in order must result in an HTTP 401 response, as defined +below. When all stages in a flow are complete, authentication is +complete and the API call succeeds. + +#### User-interactive API in the REST API + +In the REST API described in this specification, authentication works by +the client and server exchanging JSON dictionaries. The server indicates +what authentication data it requires via the body of an HTTP 401 +response, and the client submits that authentication data via the `auth` +request parameter. + +A client should first make a request with no `auth` parameter[1]. The +homeserver returns an HTTP 401 response, with a JSON body, as follows: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + + { + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" + } + +In addition to the `flows`, this object contains some extra information: + +params +This section contains any information that the client will need to know +in order to use a given type of authentication. For each authentication +type presented, that type may be present as a key in this dictionary. +For example, the public part of an OAuth client ID could be given here. + +session +This is a session identifier that the client must pass back to the +homeserver, if one is provided, in subsequent attempts to authenticate +in the same API call. + +The client then chooses a flow and attempts to complete the first stage. +It does this by resubmitting the same request with the addition of an +`auth` key in the object that it submits. This dictionary contains a +`type` key whose value is the name of the authentication type that the +client is attempting to complete. It must also contain a `session` key +with the value of the session key given by the homeserver, if one was +given. It also contains other keys dependent on the auth type being +attempted. For example, if the client is attempting to complete auth +type `example.type.foo`, it might submit something like this: + + POST /_matrix/client/r0/endpoint HTTP/1.1 + Content-Type: application/json + + { + "a_request_parameter": "something", + "another_request_parameter": "something else", + "auth": { + "type": "example.type.foo", + "session": "xxxxxx", + "example_credential": "verypoorsharedsecret" + } + } + +If the homeserver deems the authentication attempt to be successful but +still requires more stages to be completed, it returns HTTP status 401 +along with the same object as when no authentication was attempted, with +the addition of the `completed` key which is an array of auth types the +client has completed successfully: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + + { + "completed": [ "example.type.foo" ], + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" + } + +Individual stages may require more than one request to complete, in +which case the response will be as if the request was unauthenticated +with the addition of any other keys as defined by the auth type. + +If the homeserver decides that an attempt on a stage was unsuccessful, +but the client may make a second attempt, it returns the same HTTP +status 401 response as above, with the addition of the standard +`errcode` and `error` fields describing the error. For example: + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + + { + "errcode": "M_FORBIDDEN", + "error": "Invalid password", + "completed": [ "example.type.foo" ], + "flows": [ + { + "stages": [ "example.type.foo", "example.type.bar" ] + }, + { + "stages": [ "example.type.foo", "example.type.baz" ] + } + ], + "params": { + "example.type.baz": { + "example_key": "foobar" + } + }, + "session": "xxxxxx" + } + +If the request fails for a reason other than authentication, the server +returns an error message in the standard format. For example: + + HTTP/1.1 400 Bad request + Content-Type: application/json + + { + "errcode": "M_EXAMPLE_ERROR", + "error": "Something was wrong" + } + +If the client has completed all stages of a flow, the homeserver +performs the API call and returns the result as normal. Completed stages +cannot be retried by clients, therefore servers must return either a 401 +response with the completed stages, or the result of the API call if all +stages were completed when a client retries a stage. + +Some authentication types may be completed by means other than through +the Matrix client, for example, an email confirmation may be completed +when the user clicks on the link in the email. In this case, the client +retries the request with an auth dict containing only the session key. +The response to this will be the same as if the client were attempting +to complete an auth state normally, i.e. the request will either +complete or request auth, with the presence or absence of that auth type +in the 'completed' array indicating whether that stage is complete. + +##### Example + +At a high level, the requests made for an API call completing an auth +flow with three stages will resemble the following diagram: + + _______________________ + | Stage 0 | + | No auth | + | ___________________ | + | |_Request_1_________| | <-- Returns "session" key which is used throughout. + |_______________________| + | + | + _________V_____________ + | Stage 1 | + | type: "" | + | ___________________ | + | |_Request_1_________| | + |_______________________| + | + | + _________V_____________ + | Stage 2 | + | type: "" | + | ___________________ | + | |_Request_1_________| | + | ___________________ | + | |_Request_2_________| | + | ___________________ | + | |_Request_3_________| | + |_______________________| + | + | + _________V_____________ + | Stage 3 | + | type: "" | + | ___________________ | + | |_Request_1_________| | <-- Returns API response + |_______________________| + +##### Authentication types + +This specification defines the following auth types: +- `m.login.password` +- `m.login.recaptcha` +- `m.login.sso` +- `m.login.email.identity` +- `m.login.msisdn` +- `m.login.dummy` + +#### Password-based + +Type +`m.login.password` + +Description +The client submits an identifier and secret password, both sent in +plain-text. + +To use this authentication type, clients should submit an auth dict as +follows: + + { + "type": "m.login.password", + "identifier": { + ... + }, + "password": "", + "session": "" + } + +where the `identifier` property is a user identifier object, as +described in [Identifier types](#identifier-types). + +For example, to authenticate using the user's Matrix ID, clients would +submit: + + { + "type": "m.login.password", + "identifier": { + "type": "m.id.user", + "user": "" + }, + "password": "", + "session": "" + } + +Alternatively reply using a 3PID bound to the user's account on the +homeserver using the `/account/3pid`\_ API rather than giving the `user` +explicitly as follows: + + { + "type": "m.login.password", + "identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" + }, + "password": "", + "session": "" + } + +In the case that the homeserver does not know about the supplied 3PID, +the homeserver must respond with 403 Forbidden. + +#### Google ReCaptcha + +Type +`m.login.recaptcha` + +Description +The user completes a Google ReCaptcha 2.0 challenge + +To use this authentication type, clients should submit an auth dict as +follows: + + { + "type": "m.login.recaptcha", + "response": "", + "session": "" + } + +#### Single Sign-On + +Type +`m.login.sso` + +Description +Authentication is supported by authorising with an external single +sign-on provider. + +A client wanting to complete authentication using SSO should use the +[Fallback](#fallback) mechanism. See [SSO during User-Interactive +Authentication]() for more information. + +#### Email-based (identity / homeserver) + +Type +`m.login.email.identity` + +Description +Authentication is supported by authorising an email address with an +identity server, or homeserver if supported. + +Prior to submitting this, the client should authenticate with an +identity server (or homeserver). After authenticating, the session +information should be submitted to the homeserver. + +To use this authentication type, clients should submit an auth dict as +follows: + + { + "type": "m.login.email.identity", + "threepidCreds": [ + { + "sid": "", + "client_secret": "", + "id_server": "", + "id_access_token": "" + } + ], + "session": "" + } + +Note that `id_server` (and therefore `id_access_token`) is optional if +the `/requestToken` request did not include them. + +#### Phone number/MSISDN-based (identity / homeserver) + +Type +`m.login.msisdn` + +Description +Authentication is supported by authorising a phone number with an +identity server, or homeserver if supported. + +Prior to submitting this, the client should authenticate with an +identity server (or homeserver). After authenticating, the session +information should be submitted to the homeserver. + +To use this authentication type, clients should submit an auth dict as +follows: + + { + "type": "m.login.msisdn", + "threepidCreds": [ + { + "sid": "", + "client_secret": "", + "id_server": "", + "id_access_token": "" + } + ], + "session": "" + } + +Note that `id_server` (and therefore `id_access_token`) is optional if +the `/requestToken` request did not include them. + +#### Dummy Auth + +Type +`m.login.dummy` + +Description +Dummy authentication always succeeds and requires no extra parameters. +Its purpose is to allow servers to not require any form of +User-Interactive Authentication to perform a request. It can also be +used to differentiate flows where otherwise one flow would be a subset +of another flow. e.g. if a server offers flows `m.login.recaptcha` and +`m.login.recaptcha, m.login.email.identity` and the client completes the +recaptcha stage first, the auth would succeed with the former flow, even +if the client was intending to then complete the email auth stage. A +server can instead send flows `m.login.recaptcha, m.login.dummy` and +`m.login.recaptcha, m.login.email.identity` to fix the ambiguity. + +To use this authentication type, clients should submit an auth dict with +just the type and session, if provided: + + { + "type": "m.login.dummy", + "session": "" + } + +##### Fallback + +Clients cannot be expected to be able to know how to process every +single login type. If a client does not know how to handle a given login +type, it can direct the user to a web browser with the URL of a fallback +page which will allow the user to complete that login step out-of-band +in their web browser. The URL it should open is: + + /_matrix/client/%CLIENT_MAJOR_VERSION%/auth//fallback/web?session= + +Where `auth type` is the type name of the stage it is attempting and +`session ID` is the ID of the session given by the homeserver. + +This MUST return an HTML page which can perform this authentication +stage. This page must use the following JavaScript when the +authentication has been completed: + + if (window.onAuthDone) { + window.onAuthDone(); + } else if (window.opener && window.opener.postMessage) { + window.opener.postMessage("authDone", "*"); + } + +This allows the client to either arrange for the global function +`onAuthDone` to be defined in an embedded browser, or to use the HTML5 +[cross-document +messaging](https://www.w3.org/TR/webmessaging/#web-messaging) API, to +receive a notification that the authentication stage has been completed. + +Once a client receives the notification that the authentication stage +has been completed, it should resubmit the request with an auth dict +with just the session ID: + + { + "session": "" + } + +#### Example + +A client webapp might use the following JavaScript to open a popup +window which will handle unknown login types: + + /** + * Arguments: + * homeserverUrl: the base url of the homeserver (e.g. "https://matrix.org") + * + * apiEndpoint: the API endpoint being used (e.g. + * "/_matrix/client/%CLIENT_MAJOR_VERSION%/account/password") + * + * loginType: the loginType being attempted (e.g. "m.login.recaptcha") + * + * sessionID: the session ID given by the homeserver in earlier requests + * + * onComplete: a callback which will be called with the results of the request + */ + function unknownLoginType(homeserverUrl, apiEndpoint, loginType, sessionID, onComplete) { + var popupWindow; + + var eventListener = function(ev) { + // check it's the right message from the right place. + if (ev.data !== "authDone" || ev.origin !== homeserverUrl) { + return; + } + + // close the popup + popupWindow.close(); + window.removeEventListener("message", eventListener); + + // repeat the request + var requestBody = { + auth: { + session: sessionID, + }, + }; + + request({ + method:'POST', url:apiEndpoint, json:requestBody, + }, onComplete); + }; + + window.addEventListener("message", eventListener); + + var url = homeserverUrl + + "/_matrix/client/%CLIENT_MAJOR_VERSION%/auth/" + + encodeURIComponent(loginType) + + "/fallback/web?session=" + + encodeURIComponent(sessionID); + + + popupWindow = window.open(url); + } + +##### Identifier types + +Some authentication mechanisms use a user identifier object to identify +a user. The user identifier object has a `type` field to indicate the +type of identifier being used, and depending on the type, has other +fields giving the information required to identify the user as described +below. + +This specification defines the following identifier types: +- `m.id.user` +- `m.id.thirdparty` +- `m.id.phone` + +#### Matrix User ID + +Type +`m.id.user` + +Description +The user is identified by their Matrix ID. + +A client can identify a user using their Matrix ID. This can either be +the fully qualified Matrix user ID, or just the localpart of the user +ID. + + "identifier": { + "type": "m.id.user", + "user": "" + } + +#### Third-party ID + +Type +`m.id.thirdparty` + +Description +The user is identified by a third-party identifier in canonicalised +form. + +A client can identify a user using a 3PID associated with the user's +account on the homeserver, where the 3PID was previously associated +using the `/account/3pid`\_ API. See the [3PID +Types](../appendices.html#pid-types) Appendix for a list of Third-party +ID media. + + "identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" + } + +#### Phone number + +Type +`m.id.phone` + +Description +The user is identified by a phone number. + +A client can identify a user using a phone number associated with the +user's account, where the phone number was previously associated using +the `/account/3pid`\_ API. The phone number can be passed in as entered +by the user; the homeserver will be responsible for canonicalising it. +If the client wishes to canonicalise the phone number, then it can use +the `m.id.thirdparty` identifier type with a `medium` of `msisdn` +instead. + + "identifier": { + "type": "m.id.phone", + "country": "", + "phone": "" + } + +The `country` is the two-letter uppercase ISO-3166-1 alpha-2 country +code that the number in `phone` should be parsed as if it were dialled +from. + +### Login + +A client can obtain access tokens using the `/login` API. + +Note that this endpoint does not +currently use the [User-Interactive Authentication +API](#user-interactive-authentication-api). + +For a simple username/password login, clients should submit a `/login` +request as follows: + + { + "type": "m.login.password", + "identifier": { + "type": "m.id.user", + "user": "" + }, + "password": "" + } + +Alternatively, a client can use a 3PID bound to the user's account on +the homeserver using the `/account/3pid`\_ API rather than giving the +`user` explicitly, as follows: + + { + "type": "m.login.password", + "identifier": { + "medium": "", + "address": "" + }, + "password": "" + } + +In the case that the homeserver does not know about the supplied 3PID, +the homeserver must respond with `403 Forbidden`. + +To log in using a login token, clients should submit a `/login` request +as follows: + + { + "type": "m.login.token", + "token": "" + } + +As with [token-based]() interactive login, the `token` must encode the +user ID. In the case that the token is not valid, the homeserver must +respond with `403 Forbidden` and an error code of `M_FORBIDDEN`. + +If the homeserver advertises `m.login.sso` as a viable flow, and the +client supports it, the client should redirect the user to the +`/redirect` endpoint for [client login via SSO](). After authentication +is complete, the client will need to submit a `/login` request matching +`m.login.token`. + +{{login\_cs\_http\_api}} + +{{logout\_cs\_http\_api}} + +#### Login Fallback + +If a client does not recognize any or all login flows it can use the +fallback login API: + + GET /_matrix/static/client/login/ + +This returns an HTML and JavaScript page which can perform the entire +login process. The page will attempt to call the JavaScript function +`window.onLogin` when login has been successfully completed. + +Non-credential parameters valid for the `/login` endpoint can be +provided as query string parameters here. These are to be forwarded to +the login endpoint during the login process. For example: + + GET /_matrix/static/client/login/?device_id=GHTYAJCE + +### Account registration and management + +{{registration\_cs\_http\_api}} + +##### Notes on password management + +Warning + +Clients SHOULD enforce that the password provided is suitably complex. +The password SHOULD include a lower-case letter, an upper-case letter, a +number and a symbol and be at a minimum 8 characters in length. Servers +MAY reject weak passwords with an error code `M_WEAK_PASSWORD`. + +### Adding Account Administrative Contact Information + +A homeserver may keep some contact information for administrative use. +This is independent of any information kept by any identity servers, +though can be proxied (bound) to the identity server in many cases. + +Note + +This section deals with two terms: "add" and "bind". Where "add" (or +"remove") is used, it is speaking about an identifier that was not bound +to an identity server. As a result, "bind" (or "unbind") references an +identifier that is found in an identity server. Note that an identifier +can be added and bound at the same time, depending on context. + +{{administrative\_contact\_cs\_http\_api}} + +### Current account information + +{{whoami\_cs\_http\_api}} + +##### Notes on identity servers + +Identity servers in Matrix store bindings (relationships) between a +user's third party identifier, typically email or phone number, and +their user ID. Once a user has chosen an identity server, that identity +server should be used by all clients. + +Clients can see which identity server the user has chosen through the +`m.identity_server` account data event, as described below. Clients +SHOULD refrain from making requests to any identity server until the +presence of `m.identity_server` is confirmed as (not) present. If +present, the client SHOULD check for the presence of the `base_url` +property in the event's content. If the `base_url` is present, the +client SHOULD use the identity server in that property as the identity +server for the user. If the `base_url` is missing, or the account data +event is not present, the client SHOULD use whichever default value it +normally would for an identity server, if applicable. Clients SHOULD NOT +update the account data with the default identity server when the user +is missing an identity server in their account data. + +Clients SHOULD listen for changes to the `m.identity_server` account +data event and update the identity server they are contacting as a +result. + +If the client offers a way to set the identity server to use, it MUST +update the value of `m.identity_server` accordingly. A `base_url` of +`null` MUST be treated as though the user does not want to use an +identity server, disabling all related functionality as a result. + +Clients SHOULD refrain from populating the account data as a migration +step for users who are lacking the account data, unless the user sets +the identity server within the client to a value. For example, a user +which has no `m.identity_server` account data event should not end up +with the client's default identity server in their account data, unless +the user first visits their account settings to set the identity server. + +{{m\_identity\_server\_event}} + +## Capabilities negotiation + +A homeserver may not support certain operations and clients must be able +to query for what the homeserver can and can't offer. For example, a +homeserver may not support users changing their password as it is +configured to perform authentication against an external system. + +The capabilities advertised through this system are intended to +advertise functionality which is optional in the API, or which depend in +some way on the state of the user or server. This system should not be +used to advertise unstable or experimental features - this is better +done by the `/versions` endpoint. + +Some examples of what a reasonable capability could be are: + +- Whether the server supports user presence. +- Whether the server supports optional features, such as the user or + room directories. +- The rate limits or file type restrictions imposed on clients by the + server. + +Some examples of what should **not** be a capability are: + +- Whether the server supports a feature in the `unstable` + specification. +- Media size limits - these are handled by the + `/media/%CLIENT_MAJOR_VERSION%/config` API. +- Optional encodings or alternative transports for communicating with + the server. + +Capabilities prefixed with `m.` are reserved for definition in the +Matrix specification while other values may be used by servers using the +Java package naming convention. The capabilities supported by the Matrix +specification are defined later in this section. + +{{capabilities\_cs\_http\_api}} + +### `m.change_password` capability + +This capability has a single flag, `enabled`, which indicates whether or +not the user can use the `/account/password` API to change their +password. If not present, the client should assume that password changes +are possible via the API. When present, clients SHOULD respect the +capability's `enabled` flag and indicate to the user if they are unable +to change their password. + +An example of the capability API's response for this capability is: + + { + "capabilities": { + "m.change_password": { + "enabled": false + } + } + } + +### `m.room_versions` capability + +This capability describes the default and available room versions a +server supports, and at what level of stability. Clients should make use +of this capability to determine if users need to be encouraged to +upgrade their rooms. + +An example of the capability API's response for this capability is: + + { + "capabilities": { + "m.room_versions": { + "default": "1", + "available": { + "1": "stable", + "2": "stable", + "3": "unstable", + "custom-version": "unstable" + } + } + } + } + +This capability mirrors the same restrictions of [room +versions](../index.html#room-versions) to describe which versions are +stable and unstable. Clients should assume that the `default` version is +`stable`. Any version not explicitly labelled as `stable` in the +`available` versions is to be treated as `unstable`. For example, a +version listed as `future-stable` should be treated as `unstable`. + +The `default` version is the version the server is using to create new +rooms. Clients should encourage users with sufficient permissions in a +room to upgrade their room to the `default` version when the room is +using an `unstable` version. + +When this capability is not listed, clients should use `"1"` as the +default and only stable `available` room version. + +## Pagination + +Note + +The paths referred to in this section are not actual endpoints. They +only serve as examples to explain how pagination functions. + +Pagination is the process of dividing a dataset into multiple discrete +pages. Matrix makes use of pagination to allow clients to view extremely +large datasets. These datasets are not limited to events in a room (for +example clients may want to paginate a list of rooms in addition to +events within those rooms). Regardless of what is being paginated, there +is a common approach which is used to give clients an easy way of +selecting subsets of a potentially changing dataset. Each endpoint that +uses pagination may use different parameters. However the theme among +them is that they take a `from` and `to` token, and occasionally a +`limit` and `dir`. Together, these parameters describe the position in a +data set, where `from` and `to` are known as "stream tokens" matching +the regular expression `[a-zA-Z0-9.=_-]+`. If supported, the `dir` +defines the direction of events to return: either forwards (`f`) or +backwards (`b`). The response may contain tokens that can be used for +retrieving results before or after the returned set. These tokens may be +called start or prev\_batch for retrieving the previous result +set, or end, next\_batch or next\_token for retrieving the next result set. + +In the following examples, 'START' and 'END' are placeholders to signify +the start and end of the data sets respectively. + +For example, if an endpoint had events E1 -> E15. The client wants +the last 5 events and doesn't know any previous events: + + S E + |-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-| + | | | + | _____| <--backwards-- | + |__________________ | | ________| + | | | | + GET /somepath?to=START&limit=5&dir=b&from=END + Returns: + E15,E14,E13,E12,E11 + +Another example: a public room list has rooms R1 -> R17. The client +is showing 5 rooms at a time on screen, and is on page 2. They want to +now show page 3 (rooms R11 -> 15): + + S E + | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token + |-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room + |____________| |________________| + | | + Currently | + viewing | + | + GET /roomslist?from=9&to=END&limit=5 + Returns: R11,R12,R13,R14,R15 + +Note that tokens are treated in an *exclusive*, not inclusive, manner. +The end token from the initial request was '9' which corresponded to +R10. When the 2nd request was made, R10 did not appear again, even +though from=9 was specified. If you know the token, you already have the +data. + +Responses for pagination-capable endpoints SHOULD have a `chunk` array +alongside the applicable stream tokens to represent the result set. + +In general, when the end of a result set is reached the applicable +stream token will be excluded from the response. For example, if a user +was backwards-paginating events in a room they'd eventually reach the +first event in the room. In this scenario, the `prev_batch` token would +be excluded from the response. Some paginated endpoints are open-ended +in one direction, such as endpoints which expose an event stream for an +active room. In this case, it is not possible for the client to reach +the true "end" of the data set and therefore should always be presented +with a token to keep moving forwards. + +## Filtering + +Filters can be created on the server and can be passed as a parameter to +APIs which return events. These filters alter the data returned from +those APIs. Not all APIs accept filters. + +### Lazy-loading room members + +Membership events often take significant resources for clients to track. +In an effort to reduce the number of resources used, clients can enable +"lazy-loading" for room members. By doing this, servers will attempt to +only send membership events which are relevant to the client. + +It is important to understand that lazy-loading is not intended to be a +perfect optimisation, and that it may not be practical for the server to +calculate precisely which membership events are relevant to the client. +As a result, it is valid for the server to send redundant membership +events to the client to ease implementation, although such redundancy +should be minimised where possible to conserve bandwidth. + +In terms of filters, lazy-loading is enabled by enabling +`lazy_load_members` on a `RoomEventFilter` (or a `StateFilter` in the +case of `/sync` only). When enabled, lazy-loading aware endpoints (see +below) will only include membership events for the `sender` of events +being included in the response. For example, if a client makes a `/sync` +request with lazy-loading enabled, the server will only return +membership events for the `sender` of events in the timeline, not all +members of a room. + +When processing a sequence of events (e.g. by looping on `/sync` or +paginating `/messages`), it is common for blocks of events in the +sequence to share a similar set of senders. Rather than responses in the +sequence sending duplicate membership events for these senders to the +client, the server MAY assume that clients will remember membership +events they have already been sent, and choose to skip sending +membership events for members whose membership has not changed. These +are called 'redundant membership events'. Clients may request that +redundant membership events are always included in responses by setting +`include_redundant_members` to true in the filter. + +The expected pattern for using lazy-loading is currently: + +- Client performs an initial /sync with lazy-loading enabled, and + receives only the membership events which relate to the senders of + the events it receives. +- Clients which support display-name tab-completion or other + operations which require rapid access to all members in a room + should call /members for the currently selected room, with an `?at` + parameter set to the /sync response's from token. The member list + for the room is then maintained by the state in subsequent + incremental /sync responses. +- Clients which do not support tab-completion may instead pull in + profiles for arbitrary users (e.g. read receipts, typing + notifications) on demand by querying the room state or `/profile`. + +The current endpoints which support lazy-loading room members are: + +- `/sync`\_ +- `/rooms//messages`\_ +- `/rooms/{roomId}/context/{eventId}`\_ + +### API endpoints + +{{filter\_cs\_http\_api}} + +## Events + +The model of conversation history exposed by the client-server API can +be considered as a list of events. The server 'linearises' the +eventually-consistent event graph of events into an 'event stream' at +any given point in time: + + [E0]->[E1]->[E2]->[E3]->[E4]->[E5] + +Warning + +The format of events can change depending on room version. Check the +[room version specification](../index.html#room-versions) for specific +details on what to expect for event formats. Examples contained within +the client-server specification are expected to be compatible with all +specified room versions, however some differences may still apply. + +For this version of the specification, clients only need to worry about +the event ID format being different depending on room version. Clients +should not be parsing the event ID, and instead be treating it as an +opaque string. No changes should be required to support the currently +available room versions. + +### Types of room events + +Room events are split into two categories: + +State Events +These are events which update the metadata state of the room (e.g. room +topic, room membership etc). State is keyed by a tuple of event `type` +and a `state_key`. State in the room with the same key-tuple will be +overwritten. + +Message events +These are events which describe transient "once-off" activity in a room: +typically communication such as sending an instant message or setting up +a VoIP call. + +This specification outlines several events, all with the event type +prefix `m.`. (See [Room Events](#room-events) for the m. event +specification.) However, applications may wish to add their own type of +event, and this can be achieved using the REST API detailed in the +following sections. If new events are added, the event `type` key SHOULD +follow the Java package naming convention, e.g. +`com.example.myapp.event`. This ensures event types are suitably +namespaced for each application and reduces the risk of clashes. + +Note + +Events are not limited to the types defined in this specification. New +or custom event types can be created on a whim using the Java package +naming convention. For example, a `com.example.game.score` event can be +sent by clients and other clients would receive it through Matrix, +assuming the client has access to the `com.example` namespace. + +Note that the structure of these events may be different than those in +the server-server API. + +{{common\_event\_fields}} + +{{common\_room\_event\_fields}} + +##### State Event Fields + +In addition to the fields of a Room Event, State Events have the +following fields. + + +++++ + + + + + + + + + + + + + + + + + + + +
KeyTypeDescription
state_keystringRequired. A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. State keys starting with an @ are reserved for referencing user IDs, such as room members. With the exception of a few events, state events set with a given user's ID as the state key MUST only be set by that user.
prev_contentEventContentOptional. The previous content for this event. If there is no previous content, this key will be missing.
+ +### Size limits + +The complete event MUST NOT be larger than 65535 bytes, when formatted +as a [PDU for the Server-Server +protocol](../server_server/%SERVER_RELEASE_LABEL%#pdus), including any +signatures, and encoded as [Canonical +JSON](../appendices.html#canonical-json). + +There are additional restrictions on sizes per key: + +- `sender` MUST NOT exceed 255 bytes (including domain). +- `room_id` MUST NOT exceed 255 bytes. +- `state_key` MUST NOT exceed 255 bytes. +- `type` MUST NOT exceed 255 bytes. +- `event_id` MUST NOT exceed 255 bytes. + +Some event types have additional size restrictions which are specified +in the description of the event. Additional keys have no limit other +than that implied by the total 65 KB limit on events. + +### Room Events + +Note + +This section is a work in progress. + +This specification outlines several standard event types, all of which +are prefixed with `m.` + +{{m\_room\_canonical\_alias\_event}} + +{{m\_room\_create\_event}} + +{{m\_room\_join\_rules\_event}} + +{{m\_room\_member\_event}} + +{{m\_room\_power\_levels\_event}} + +##### Historical events + +Some events within the `m.` namespace might appear in rooms, however +they serve no significant meaning in this version of the specification. +They are: + +- `m.room.aliases` + +Previous versions of the specification have more information on these +events. + +### Syncing + +To read events, the intended flow of operation is for clients to first +call the `/sync`\_ API without a `since` parameter. This returns the +most recent message events for each room, as well as the state of the +room at the start of the returned timeline. The response also includes a +`next_batch` field, which should be used as the value of the `since` +parameter in the next call to `/sync`. Finally, the response includes, +for each room, a `prev_batch` field, which can be passed as a `start` +parameter to the `/rooms//messages`\_ API to retrieve earlier +messages. + +You can visualise the range of events being returned as: + + [E0]->[E1]->[E2]->[E3]->[E4]->[E5] + ^ ^ + | | + prev_batch: '1-2-3' next_batch: 'a-b-c' + +Clients then receive new events by "long-polling" the homeserver via the +`/sync` API, passing the value of the `next_batch` field from the +response to the previous call as the `since` parameter. The client +should also pass a `timeout` parameter. The server will then hold open +the HTTP connection for a short period of time waiting for new events, +returning early if an event occurs. Only the `/sync` API (and the +deprecated `/events` API) support long-polling in this way. + +The response for such an incremental sync can be visualised as: + + [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6] + ^ ^ + | | + | next_batch: 'x-y-z' + prev_batch: 'a-b-c' + +Normally, all new events which are visible to the client will appear in +the response to the `/sync` API. However, if a large number of events +arrive between calls to `/sync`, a "limited" timeline is returned, +containing only the most recent message events. A state "delta" is also +returned, summarising any state changes in the omitted part of the +timeline. The client may therefore end up with "gaps" in its knowledge +of the message timeline. The client can fill these gaps using the +`/rooms//messages`\_ API. This situation looks like this: + + | gap | + | <-> | + [E0]->[E1]->[E2]->[E3]->[E4]->[E5]->[E6]->[E7]->[E8]->[E9]->[E10] + ^ ^ + | | + prev_batch: 'd-e-f' next_batch: 'u-v-w' + +Warning + +Events are ordered in this API according to the arrival time of the +event on the homeserver. This can conflict with other APIs which order +events based on their partial ordering in the event graph. This can +result in duplicate events being received (once per distinct API +called). Clients SHOULD de-duplicate events based on the event ID when +this happens. + +Note + +The `/sync` API returns a `state` list which is separate from the +`timeline`. This `state` list allows clients to keep their model of the +room state in sync with that on the server. In the case of an initial +(`since`-less) sync, the `state` list represents the complete state of +the room at the **start** of the returned timeline (so in the case of a +recently-created room whose state fits entirely in the `timeline`, the +`state` list will be empty). + +In the case of an incremental sync, the `state` list gives a delta +between the state of the room at the `since` parameter and that at the +start of the returned `timeline`. (It will therefore be empty unless the +timeline was `limited`.) + +In both cases, it should be noted that the events returned in the +`state` list did **not** necessarily take place just before the returned +`timeline`, so clients should not display them to the user in the +timeline. + +Rationale + +An early design of this specification made the `state` list represent +the room state at the end of the returned timeline, instead of the +start. This was unsatisfactory because it led to duplication of events +between the `state` list and the `timeline`, but more importantly, it +made it difficult for clients to show the timeline correctly. + +In particular, consider a returned timeline \[M0, S1, M2\], where M0 and +M2 are both messages sent by the same user, and S1 is a state event +where that user changes their displayname. If the `state` list +represents the room state at the end of the timeline, the client must +take a copy of the state dictionary, and *rewind* S1, in order to +correctly calculate the display name for M0. + +{{sync\_cs\_http\_api}} + +{{old\_sync\_cs\_http\_api}} + +### Getting events for a room + +There are several APIs provided to `GET` events for a room: + +{{rooms\_cs\_http\_api}} + +{{message\_pagination\_cs\_http\_api}} + +{{room\_initial\_sync\_cs\_http\_api}} + +### Sending events to a room + +{{room\_state\_cs\_http\_api}} + +**Examples** + +Valid requests look like: + + PUT /rooms/!roomid:domain/state/m.example.event + { "key" : "without a state key" } + + PUT /rooms/!roomid:domain/state/m.another.example.event/foo + { "key" : "with 'foo' as the state key" } + +In contrast, these requests are invalid: + + POST /rooms/!roomid:domain/state/m.example.event/ + { "key" : "cannot use POST here" } + + PUT /rooms/!roomid:domain/state/m.another.example.event/foo/11 + { "key" : "txnIds are not supported" } + +Care should be taken to avoid setting the wrong `state key`: + + PUT /rooms/!roomid:domain/state/m.another.example.event/11 + { "key" : "with '11' as the state key, but was probably intended to be a txnId" } + +The `state_key` is often used to store state about individual users, by +using the user ID as the `state_key` value. For example: + + PUT /rooms/!roomid:domain/state/m.favorite.animal.event/%40my_user%3Aexample.org + { "animal" : "cat", "reason": "fluffy" } + +In some cases, there may be no need for a `state_key`, so it can be +omitted: + + PUT /rooms/!roomid:domain/state/m.room.bgd.color + { "color": "red", "hex": "#ff0000" } + +{{room\_send\_cs\_http\_api}} + +### Redactions + +Since events are extensible it is possible for malicious users and/or +servers to add keys that are, for example offensive or illegal. Since +some events cannot be simply deleted, e.g. membership events, we instead +'redact' events. This involves removing all keys from an event that are +not required by the protocol. This stripped down event is thereafter +returned anytime a client or remote server requests it. Redacting an +event cannot be undone, allowing server owners to delete the offending +content from the databases. Events that have been redacted include a +`redacted_because` key whose value is the event that caused it to be +redacted, which may include a reason. + +The exact algorithm to apply against an event is defined in the [room +version specification](../index.html#room-versions). + +The server should add the event causing the redaction to the `unsigned` +property of the redacted event, under the `redacted_because` key. When a +client receives a redaction event it should change the redacted event in +the same way a server does. + +Note + +Redacted events can still affect the state of the room. When redacted, +state events behave as though their properties were simply not +specified, except those protected by the redaction algorithm. For +example, a redacted `join` event will still result in the user being +considered joined. Similarly, a redacted topic does not necessarily +cause the topic to revert to what it was prior to the event - it causes +the topic to be removed from the room. + +##### Events + +{{m\_room\_redaction\_event}} + +##### Client behaviour + +{{redaction\_cs\_http\_api}} + +## Rooms + +### Creation + +The homeserver will create an `m.room.create` event when a room is +created, which serves as the root of the event graph for this room. This +event also has a `creator` key which contains the user ID of the room +creator. It will also generate several other events in order to manage +permissions in this room. This includes: + +- `m.room.power_levels` : Sets the power levels of users and required power + levels for various actions within the room such as sending events. + +- `m.room.join_rules` : Whether the room is "invite-only" or not. + +See [Room Events](#room-events) for more information on these events. To +create a room, a client has to use the following API. + +{{create\_room\_cs\_http\_api}} + +### Room aliases + +Servers may host aliases for rooms with human-friendly names. Aliases +take the form `#friendlyname:server.name`. + +As room aliases are scoped to a particular homeserver domain name, it is +likely that a homeserver will reject attempts to maintain aliases on +other domain names. This specification does not provide a way for +homeservers to send update requests to other servers. However, +homeservers MUST handle `GET` requests to resolve aliases on other +servers; they should do this using the federation API if necessary. + +Rooms do not store a list of all aliases present on a room, though +members of the room with relevant permissions may publish preferred +aliases through the `m.room.canonical_alias` state event. The aliases in +the state event should point to the room ID they are published within, +however room aliases can and do drift to other room IDs over time. +Clients SHOULD NOT treat the aliases as accurate. They SHOULD be checked +before they are used or shared with another user. If a room appears to +have a room alias of `#alias:example.com`, this SHOULD be checked to +make sure that the room's ID matches the `room_id` returned from the +request. + +{{directory\_cs\_http\_api}} + +### Permissions + +Note + +This section is a work in progress. + +Permissions for rooms are done via the concept of power levels - to do +any action in a room a user must have a suitable power level. Power +levels are stored as state events in a given room. The power levels +required for operations and the power levels for users are defined in +`m.room.power_levels`, where both a default and specific users' power +levels can be set. By default all users have a power level of 0, other +than the room creator whose power level defaults to 100. Users can grant +other users increased power levels up to their own power level. For +example, user A with a power level of 50 could increase the power level +of user B to a maximum of level 50. Power levels for users are tracked +per-room even if the user is not present in the room. The keys contained +in `m.room.power_levels` determine the levels required for certain +operations such as kicking, banning and sending state events. See +[m.room.power\_levels]() for more information. + +Clients may wish to assign names to particular power levels. A suggested +mapping is as follows: - 0 User - 50 Moderator - 100 Admin + +### Room membership + +Users need to be a member of a room in order to send and receive events +in that room. There are several states in which a user may be, in +relation to a room: + +- Unrelated (the user cannot send or receive events in the room) +- Invited (the user has been invited to participate in the room, but + is not yet participating) +- Joined (the user can send and receive events in the room) +- Banned (the user is not allowed to join the room) + +There is an exception to the requirement that a user join a room before +sending events to it: users may send an `m.room.member` event to a room +with `content.membership` set to `leave` to reject an invitation if they +have currently been invited to a room but have not joined it. + +Some rooms require that users be invited to it before they can join; +others allow anyone to join. Whether a given room is an "invite-only" +room is determined by the room config key `m.room.join_rules`. It can +have one of the following values: + +`public` +This room is free for anyone to join without an invite. + +`invite` +This room can only be joined if you were invited. + +The allowable state transitions of membership are: + + /ban + +------------------------------------------------------+ + | | + | +----------------+ +----------------+ | + | | /leave | | | | + | | v v | | + /invite +--------+ +-------+ | | + ------------>| invite |<----------| leave |----+ | | + +--------+ /invite +-------+ | | | + | | ^ | | | + | | | | | | + /join | +---------------+ | | | | + | | /join if | | | | + | | join_rules | | /ban | /unban | + | | public /leave | | | | + v v or | | | | + +------+ /kick | | | | + ------------>| join |-------------------+ | | | + /join +------+ v | | + if | +-----+ | | + join_rules +-------------------------->| ban |-----+ | + public /ban +-----+ | + ^ ^ | + | | | + ----------------------------------------------+ +----------------------+ + /ban + +{{list\_joined\_rooms\_cs\_http\_api}} + +##### Joining rooms + +{{inviting\_cs\_http\_api}} + +{{joining\_cs\_http\_api}} + +##### Leaving rooms + +A user can leave a room to stop receiving events for that room. A user +must have been invited to or have joined the room before they are +eligible to leave the room. Leaving a room to which the user has been +invited rejects the invite. Once a user leaves a room, it will no longer +appear in the response to the `/sync`\_ API unless it is explicitly +requested via a filter with the `include_leave` field set to `true`. + +Whether or not they actually joined the room, if the room is an +"invite-only" room the user will need to be re-invited before they can +re-join the room. + +A user can also forget a room which they have left. Rooms which have +been forgotten will never appear the response to the `/sync`\_ API, +until the user re-joins or is re-invited. + +A user may wish to force another user to leave a room. This can be done +by 'kicking' the other user. To do so, the user performing the kick MUST +have the required power level. Once a user has been kicked, the +behaviour is the same as if they had left of their own accord. In +particular, the user is free to re-join if the room is not +"invite-only". + +{{leaving\_cs\_http\_api}} + +{{kicking\_cs\_http\_api}} + +##### Banning users in a room + +A user may decide to ban another user in a room. 'Banning' forces the +target user to leave the room and prevents them from re-joining the +room. A banned user will not be treated as a joined user, and so will +not be able to send or receive events in the room. In order to ban +someone, the user performing the ban MUST have the required power level. +To ban a user, a request should be made to `/rooms//ban`\_ +with: + + { + "user_id": "" + "reason": "string: " + } + +Banning a user adjusts the banned member's membership state to `ban`. +Like with other membership changes, a user can directly adjust the +target member's state, by making a request to +`/rooms//state/m.room.member/`: + + { + "membership": "ban" + } + +A user must be explicitly unbanned with a request to +`/rooms//unban`\_ before they can re-join the room or be +re-invited. + +{{banning\_cs\_http\_api}} + +### Listing rooms + +{{list\_public\_rooms\_cs\_http\_api}} + +## User Data + +### User Directory + +{{users\_cs\_http\_api}} + +### Profiles + +{{profile\_cs\_http\_api}} + +##### Events on Change of Profile Information + +Because the profile display name and avatar information are likely to be +used in many places of a client's display, changes to these fields cause +an automatic propagation event to occur, informing likely-interested +parties of the new values. This change is conveyed using two separate +mechanisms: + +- an `m.room.member` event (with a `join` membership) is sent to every + room the user is a member of, to update the `displayname` and + `avatar_url`. +- an `m.presence` presence status update is sent, again containing the + new values of the `displayname` and `avatar_url` keys, in addition + to the required `presence` key containing the current presence state + of the user. + +Both of these should be done automatically by the homeserver when a user +successfully changes their display name or avatar URL fields. + +Additionally, when homeservers emit room membership events for their own +users, they should include the display name and avatar URL fields in +these events so that clients already have these details to hand, and do +not have to perform extra round trips to query it. + +## Security + +### Rate limiting + +Homeservers SHOULD implement rate limiting to reduce the risk of being +overloaded. If a request is refused due to rate limiting, it should +return a standard error response of the form: + + { + "errcode": "M_LIMIT_EXCEEDED", + "error": "string", + "retry_after_ms": integer (optional) + } + +The `retry_after_ms` key SHOULD be included to tell the client how long +they have to wait in milliseconds before they can try again. + +[1] A request to an endpoint that uses User-Interactive Authentication +never succeeds without auth. Homeservers may allow requests that don't +require auth by offering a stage with only the `m.login.dummy` auth +type, but they must still give a 401 response to requests with no auth +data. diff --git a/content/identity-service-api.md b/content/identity-service-api.md index 354a9282..07d7420b 100644 --- a/content/identity-service-api.md +++ b/content/identity-service-api.md @@ -3,3 +3,470 @@ title: "Identity Service API" weight: 40 type: docs --- + +# Identity Service API + +{{unstable\_warning\_block\_IDENTITY\_RELEASE\_LABEL}} + +The Matrix client-server and server-server APIs are largely expressed in +Matrix user identifiers. From time to time, it is useful to refer to +users by other ("third-party") identifiers, or "3PID"s, e.g. their email +address or phone number. This Identity Service Specification describes +how mappings between third-party identifiers and Matrix user identifiers +can be established, validated, and used. This description technically +may apply to any 3PID, but in practice has only been applied +specifically to email addresses and phone numbers. + +Table of Contents + +## Changelog + +**Version: %IDENTITY\_RELEASE\_LABEL%** + +{{identity\_service\_changelog}} + +This version of the specification is generated from +[matrix-doc](https://github.com/matrix-org/matrix-doc) as of Git commit +[{{git\_version}}](https://github.com/matrix-org/matrix-doc/tree/%7B%7Bgit_rev%7D%7D). + +For the full historical changelog, see + + +### Other versions of this specification + +The following other versions are also available, in reverse +chronological order: + +- [HEAD](https://matrix.org/docs/spec/identity_service/unstable.html): + Includes all changes since the latest versioned release. +- [r0.3.0](https://matrix.org/docs/spec/identity_service/r0.3.0.html) +- [r0.2.1](https://matrix.org/docs/spec/identity_service/r0.2.1.html) +- [r0.2.0](https://matrix.org/docs/spec/identity_service/r0.2.0.html) +- [r0.1.0](https://matrix.org/docs/spec/identity_service/r0.1.0.html) + +## General principles + +The purpose of an identity server is to validate, store, and answer +questions about the identities of users. In particular, it stores +associations of the form "identifier X represents the same user as +identifier Y", where identities may exist on different systems (such as +email addresses, phone numbers, Matrix user IDs, etc). + +The identity server has some private-public keypairs. When asked about +an association, it will sign details of the association with its private +key. Clients may validate the assertions about associations by verifying +the signature with the public key of the identity server. + +In general, identity servers are treated as reliable oracles. They do +not necessarily provide evidence that they have validated associations, +but claim to have done so. Establishing the trustworthiness of an +individual identity server is left as an exercise for the client. + +3PID types are described in [3PID Types](../appendices.html#pid-types) +Appendix. + +## API standards + +The mandatory baseline for identity server communication in Matrix is +exchanging JSON objects over HTTP APIs. HTTPS is required for +communication, and all API calls use a Content-Type of +`application/json`. In addition, strings MUST be encoded as UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard +error response". This is a JSON object which looks like: + + { + "errcode": "", + "error": "" + } + +The `error` string will be a human-readable error message, usually a +sentence explaining what went wrong. The `errcode` string will be a +unique string which can be used to handle an error message e.g. +`M_FORBIDDEN`. There may be additional keys depending on the error, but +the keys `error` and `errcode` MUST always be present. + +Some standard error codes are below: + +`M_NOT_FOUND` +The resource requested could not be located. + +`M_MISSING_PARAMS` +The request was missing one or more parameters. + +`M_INVALID_PARAM` +The request contained one or more invalid parameters. + +`M_SESSION_NOT_VALIDATED` +The session has not been validated. + +`M_NO_VALID_SESSION` +A session could not be located for the given parameters. + +`M_SESSION_EXPIRED` +The session has expired and must be renewed. + +`M_INVALID_EMAIL` +The email address provided was not valid. + +`M_EMAIL_SEND_ERROR` +There was an error sending an email. Typically seen when attempting to +verify ownership of a given email address. + +`M_INVALID_ADDRESS` +The provided third party address was not valid. + +`M_SEND_ERROR` +There was an error sending a notification. Typically seen when +attempting to verify ownership of a given third party address. + +`M_UNRECOGNIZED` +The request contained an unrecognised value, such as an unknown token or +medium. + +`M_THREEPID_IN_USE` +The third party identifier is already in use by another user. Typically +this error will have an additional `mxid` property to indicate who owns +the third party identifier. + +`M_UNKNOWN` +An unknown error has occurred. + +## Privacy + +Identity is a privacy-sensitive issue. While the identity server exists +to provide identity information, access should be restricted to avoid +leaking potentially sensitive data. In particular, being able to +construct large-scale connections between identities should be avoided. +To this end, in general APIs should allow a 3PID to be mapped to a +Matrix user identity, but not in the other direction (i.e. one should +not be able to get all 3PIDs associated with a Matrix user ID, or get +all 3PIDs associated with a 3PID). + +## Version 1 API deprecation + +As described on each of the version 1 endpoints, the v1 API is +deprecated in favour of the v2 API described here. The major difference, +with the exception of a few isolated cases, is that the v2 API requires +authentication to ensure the user has given permission for the identity +server to operate on their data. + +The v1 API is planned to be removed from the specification in a future +version. + +Clients SHOULD attempt the v2 endpoints first, and if they receive a +`404`, `400`, or similar error they should try the v1 endpoint or fail +the operation. Clients are strongly encouraged to warn the user of the +risks in using the v1 API, if they are planning on using it. + +## Web browser clients + +It is realistic to expect that some clients will be written to be run +within a web browser or similar environment. In these cases, the +identity server should respond to pre-flight requests and supply +Cross-Origin Resource Sharing (CORS) headers on all requests. + +When a client approaches the server with a pre-flight (OPTIONS) request, +the server should respond with the CORS headers for that route. The +recommended CORS headers to be returned by servers on all requests are: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + +## Authentication + +Most `v2` endpoints in the Identity Service API require authentication +in order to ensure that the requesting user has accepted all relevant +policies and is otherwise permitted to make the request. The `v1` API +(currently deprecated) does not require this authentication, however +using `v1` is strongly discouraged as it will be removed in a future +release. + +Identity Servers use a scheme similar to the Client-Server API's concept +of access tokens to authenticate users. The access tokens provided by an +Identity Server cannot be used to authenticate Client-Server API +requests. + +An access token is provided to an endpoint in one of two ways: + +1. Via a query string parameter, `access_token=TheTokenHere`. +2. Via a request header, `Authorization: Bearer TheTokenHere`. + +Clients are encouraged to the use the `Authorization` header where +possible to prevent the access token being leaked in access/HTTP logs. +The query string should only be used in cases where the `Authorization` +header is inaccessible for the client. + +When credentials are required but missing or invalid, the HTTP call will +return with a status of 401 and the error code `M_UNAUTHORIZED`. + +{{v2\_auth\_is\_http\_api}} + +## Terms of service + +Identity Servers are encouraged to have terms of service (or similar +policies) to ensure that users have agreed to their data being processed +by the server. To facilitate this, an identity server can respond to +almost any authenticated API endpoint with an HTTP 403 and the error +code `M_TERMS_NOT_SIGNED`. The error code is used to indicate that the +user must accept new terms of service before being able to continue. + +All endpoints which support authentication can return the +`M_TERMS_NOT_SIGNED` error. When clients receive the error, they are +expected to make a call to `GET /terms` to find out what terms the +server offers. The client compares this to the `m.accepted_terms` +account data for the user (described later) and presents the user with +option to accept the still-missing terms of service. After the user has +made their selection, if applicable, the client sends a request to +`POST /terms` to indicate the user's acceptance. The server cannot +expect that the client will send acceptance for all pending terms, and +the client should not expect that the server will not respond with +another `M_TERMS_NOT_SIGNED` on their next request. The terms the user +has just accepted are appended to `m.accepted_terms`. + +{{m\_accepted\_terms\_event}} + +{{v2\_terms\_is\_http\_api}} + +## Status check + +{{ping\_is\_http\_api}} + +{{v2\_ping\_is\_http\_api}} + +## Key management + +An identity server has some long-term public-private keypairs. These are +named in a scheme `algorithm:identifier`, e.g. `ed25519:0`. When signing +an association, the standard [Signing +JSON](../appendices.html#signing-json) algorithm applies. + +The identity server may also keep track of some short-term +public-private keypairs, which may have different usage and lifetime +characteristics than the service's long-term keys. + +{{pubkey\_is\_http\_api}} + +{{v2\_pubkey\_is\_http\_api}} + +## Association lookup + +{{lookup\_is\_http\_api}} + +{{v2\_lookup\_is\_http\_api}} + +### Client behaviour + +Note + +This section only covers the v2 lookup endpoint. The v1 endpoint is +described in isolation above. + +Prior to performing a lookup clients SHOULD make a request to the +`/hash_details` endpoint to determine what algorithms the server +supports (described in more detail below). The client then uses this +information to form a `/lookup` request and receive known bindings from +the server. + +Clients MUST support at least the `sha256` algorithm. + +### Server behaviour + +Note + +This section only covers the v2 lookup endpoint. The v1 endpoint is +described in isolation above. + +Servers, upon receipt of a `/lookup` request, will compare the query +against known bindings it has, hashing the identifiers it knows about as +needed to verify exact matches to the request. + +Servers MUST support at least the `sha256` algorithm. + +### Algorithms + +Some algorithms are defined as part of the specification, however other +formats can be negotiated between the client and server using +`/hash_details`. + +#### `sha256` + +This algorithm MUST be supported by clients and servers at a minimum. It +is additionally the preferred algorithm for lookups. + +When using this algorithm, the client converts the query first into +strings separated by spaces in the format `
`. +The `` is retrieved from `/hash_details`, the `` is +typically `email` or `msisdn` (both lowercase), and the `
` is +the 3PID to search for. For example, if the client wanted to know about +`alice@example.org`'s bindings, it would first format the query as +`alice@example.org email ThePepperGoesHere`. + +Rationale + +Mediums and peppers are appended to the address to prevent a common +prefix for each 3PID, helping prevent attackers from pre-computing the +internal state of the hash function. + +After formatting each query, the string is run through SHA-256 as +defined by [RFC 4634](https://tools.ietf.org/html/rfc4634). The +resulting bytes are then encoded using URL-Safe [Unpadded +Base64](../appendices.html#unpadded-base64) (similar to [room version +4's event ID format](../rooms/v4.html#event-ids)). + +An example set of queries when using the pepper `matrixrocks` would be: + + "alice@example.com email matrixrocks" -> "4kenr7N9drpCJ4AfalmlGQVsOn3o2RHjkADUpXJWZUc" + "bob@example.com email matrixrocks" -> "LJwSazmv46n0hlMlsb_iYxI0_HXEqy_yj6Jm636cdT8" + "18005552067 msisdn matrixrocks" -> "nlo35_T5fzSGZzJApqu8lgIudJvmOQtDaHtr-I4rU7I" + +The set of hashes is then given as the `addresses` array in `/lookup`. +Note that the pepper used MUST be supplied as `pepper` in the `/lookup` +request. + +#### `none` + +This algorithm performs plaintext lookups on the identity server. +Typically this algorithm should not be used due to the security concerns +of unhashed identifiers, however some scenarios (such as LDAP-backed +identity servers) prevent the use of hashed identifiers. Identity +servers (and optionally clients) can use this algorithm to perform those +kinds of lookups. + +Similar to the `sha256` algorithm, the client converts the queries into +strings separated by spaces in the format `
` - note +the lack of ``. For example, if the client wanted to know about +`alice@example.org`'s bindings, it would format the query as +`alice@example.org email`. + +The formatted strings are then given as the `addresses` in `/lookup`. +Note that the `pepper` is still required, and must be provided to ensure +the client has made an appropriate request to `/hash_details` first. + +### Security considerations + +Note + +[MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) has much +more information about the security considerations made for this section +of the specification. This section covers the high-level details for why +the specification is the way it is. + +Typically the lookup endpoint is used when a client has an unknown 3PID +it wants to find a Matrix User ID for. Clients normally do this kind of +lookup when inviting new users to a room or searching a user's address +book to find any Matrix users they may not have discovered yet. Rogue or +malicious identity servers could harvest this unknown information and do +nefarious things with it if it were sent in plain text. In order to +protect the privacy of users who might not have a Matrix identifier +bound to their 3PID addresses, the specification attempts to make it +difficult to harvest 3PIDs. + +Rationale + +Hashing identifiers, while not perfect, helps make the effort required +to harvest identifiers significantly higher. Phone numbers in particular +are still difficult to protect with hashing, however hashing is +objectively better than not. + +An alternative to hashing would be using bcrypt or similar with many +rounds, however by nature of needing to serve mobile clients and clients +on limited hardware the solution needs be kept relatively lightweight. + +Clients should be cautious of servers not rotating their pepper very +often, and potentially of servers which use a weak pepper - these +servers may be attempting to brute force the identifiers or use rainbow +tables to mine the addresses. Similarly, clients which support the +`none` algorithm should consider at least warning the user of the risks +in sending identifiers in plain text to the identity server. + +Addresses are still potentially reversable using a calculated rainbow +table given some identifiers, such as phone numbers, common email +address domains, and leaked addresses are easily calculated. For +example, phone numbers can have roughly 12 digits to them, making them +an easier target for attack than email addresses. + +## Establishing associations + +The flow for creating an association is session-based. + +Within a session, one may prove that one has ownership of a 3PID. Once +this has been established, the user can form an association between that +3PID and a Matrix user ID. Note that this association is only proved one +way; a user can associate *any* Matrix user ID with a validated 3PID, +i.e. I can claim that any email address I own is associated with +@billg:microsoft.com. + +Sessions are time-limited; a session is considered to have been modified +when it was created, and then when a validation is performed within it. +A session can only be checked for validation, and validation can only be +performed within a session, within a 24-hour period since its most +recent modification. Any attempts to perform these actions after the +expiry will be rejected, and a new session should be created and used +instead. + +To start a session, the client makes a request to the appropriate +`/requestToken` endpoint. The identity server then sends a validation +token to the user, and the user provides the token to the client. The +client then provides the token to the appropriate `/submitToken` +endpoint, completing the session. At this point, the client should +`/bind` the third party identifier or leave it for another entity to +bind. + +### Format of a validation token + +The format of the validation token is left up to the identity server: it +should choose one appropriate to the 3PID type. (For example, it would +be inappropriate to expect a user to copy a long passphrase including +punctuation from an SMS message into a client.) + +Whatever format the identity server uses, the validation token must +consist of at most 255 Unicode codepoints. Clients must pass the token +through without modification. + +### Email associations + +{{email\_associations\_is\_http\_api}} + +{{v2\_email\_associations\_is\_http\_api}} + +### Phone number associations + +{{phone\_associations\_is\_http\_api}} + +{{v2\_phone\_associations\_is\_http\_api}} + +### General + +{{associations\_is\_http\_api}} + +{{v2\_associations\_is\_http\_api}} + +## Invitation storage + +An identity server can store pending invitations to a user's 3PID, which +will be retrieved and can be either notified on or look up when the 3PID +is associated with a Matrix user ID. + +At a later point, if the owner of that particular 3PID binds it with a +Matrix user ID, the identity server will attempt to make an HTTP POST to +the Matrix user's homeserver via the +[/3pid/onbind](../server_server/%SERVER_RELEASE_LABEL%.html#put-matrix-federation-v1-3pid-onbind) +endpoint. The request MUST be signed with a long-term private key for +the identity server. + +{{store\_invite\_is\_http\_api}} + +{{v2\_store\_invite\_is\_http\_api}} + +## Ephemeral invitation signing + +To aid clients who may not be able to perform crypto themselves, the +identity server offers some crypto functionality to help in accepting +invitations. This is less secure than the client doing it itself, but +may be useful where this isn't possible. + +{{invitation\_signing\_is\_http\_api}} + +{{v2\_invitation\_signing\_is\_http\_api}} diff --git a/content/proposals.md b/content/proposals.md index 629e3ab7..0164faf5 100644 --- a/content/proposals.md +++ b/content/proposals.md @@ -3,3 +3,597 @@ title: "Spec Change Proposals" weight: 60 type: docs --- + +%proposalscssinjection% + +Proposals for Spec Changes to Matrix + +Table of Contents + +# Proposals for Spec Changes to Matrix + +If you are interested in submitting a change to the Matrix +Specification, please take note of the following guidelines. + +Most changes to the Specification require a formal proposal. Bug fixes, +typos, and clarifications to existing behaviour do not need proposals - +see the [contributing +guide](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst) +for more information on what does and does not need a proposal. + +The proposal process involves some technical writing, having it reviewed +by everyone, having the proposal being accepted, then actually having +your ideas implemented as committed changes to the [Specification +repository](https://github.com/matrix-org/matrix-doc). + +Meet the [members of the Core Team](https://matrix.org/foundation), a +group of individuals tasked with ensuring the spec process is as smooth +and painless as possible. Members of the Spec Core Team will do their +best to participate in discussion, summarise when things become +long-winded, and generally try to act towards the benefit of everyone. +As a majority, team members have the ability to change the state of a +proposal, and individually have the final say in proposal discussion. + +# Guiding Principles + +Proposals **must** act to the greater benefit of the entire Matrix +ecosystem, rather than benefiting or privileging any single player or +subset of players -and must not contain any patent encumbered +intellectual property. Members of the Core Team pledge to act as a +neutral custodian for Matrix on behalf of the whole ecosystem. + +For clarity: the Matrix ecosystem is anyone who uses the Matrix +protocol. That includes client users, server admins, client developers, +bot developers, bridge and application service developers, users and +admins who are indirectly using Matrix via 3rd party networks which +happen to be bridged, server developers, room moderators and admins, +companies/projects building products or services on Matrix, spec +contributors, translators, and those who created it in the first place. + +"Greater benefit" could include maximising: + +- the number of end-users reachable on the open Matrix network +- the number of regular users on the Matrix network (e.g. 30-day + retained federated users) +- the number of online servers in the open federation +- the number of developers building on Matrix +- the number of independent implementations which use Matrix +- the number of bridged end-users reachable on the open Matrix network +- the signal-to-noise ratio of the content on the open Matrix network + (i.e. minimising spam) +- the ability for users to discover content on their terms (empowering + them to select what to see and what not to see) +- the quality and utility of the Matrix spec (as defined by ease and + ability with which a developer can implement spec-compliant clients, + servers, bots, bridges, and other integrations without needing to + refer to any other external material) + +In addition, proposal authors are expected to uphold the following +values in their proposed changes to the Matrix protocol: + +- Supporting the whole long-term ecosystem rather than individual + stakeholder gain +- Openness rather than proprietary lock-in +- Interoperability rather than fragmentation +- Cross-platform rather than platform-specific +- Collaboration rather than competition +- Accessibility rather than elitism +- Transparency rather than stealth +- Empathy rather than contrariness +- Pragmatism rather than perfection +- Proof rather than conjecture + +Please [see +MSC1779](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1779-open-governance.md) +for full details of the project's Guiding Principles. + +# Technical notes + +Proposals **must** develop Matrix as a layered protocol: with new +features building on layers of shared abstractions rather than +introducing tight vertical coupling within the stack. This ensures that +new features can evolve rapidly by building on existing layers and +swapping out old features without impacting the rest of the stack or +requiring substantial upgrades to the whole ecosystem. This is critical +for Matrix to rapidly evolve and compete effectively with centralised +systems, despite being a federated protocol. + +For instance, new features should be implemented using the highest layer +abstractions possible (e.g. new event types, which layer on top of the +existing room semantics, and so don't even require any API changes). +Failing that, the next recourse would be backwards-compatible changes to +the next layer down (e.g. room APIs); failing that, considering changes +to the format of events or the DAG; etc. It would be a very unusual +feature which doesn't build on the existing infrastructure provided by +the spec and instead created new primitives or low level APIs. + +Backwards compatibility is very important for Matrix, but not at the +expense of hindering the protocol's evolution. Backwards incompatible +changes to endpoints are allowed when no other alternative exists, and +must be versioned under a new major release of the API. Backwards +incompatible changes to the room algorithm are also allowed when no +other alternative exists, and must be versioned under a new version of +the room algorithm. + +There is sometimes a dilemma over where to include higher level +features: for instance, should video conferencing be formalised in the +spec, or should it be implemented via widgets? Should reputation systems +be specified? Should search engine behaviour be specified? + +There is no universal answer to this, but the following guidelines +should be applied: + +1. If the feature would benefit the whole Matrix ecosystem and is + aligned with the guiding principles above, then it should be + supported by the spec. +2. If the spec already makes the feature possible without changing any + of the implementations and spec, then it may not need to be added to + the spec. +3. However, if the best user experience for a feature does require + custom implementation behaviour then the behaviour should be defined + in the spec such that all implementations may implement it. +4. However, the spec must never add dependencies on + unspecified/nonstandardised 3rd party behaviour. + +As a worked example: + +1. Video conferencing is clearly a feature which would benefit the + whole ecosystem, and so the spec should find a way to make it + happen. +2. Video conferencing can be achieved by widgets without requiring any + compulsory changes to clients nor servers to work, and so could be + omitted from the spec. +3. A better experience could be achieved by embedding Jitsi natively + into clients rather than using a widget... +4. ...except that would add a dependency on unspecified/nonstandardised + 3rd party behaviour, so must not be added to the spec. + +Therefore, our two options in the specific case of video conferencing +are either to spec SFU conferencing semantics for WebRTC (or refer to an +existing spec for doing so), or to keep it as a widget-based approach +(optionally with widget extensions specific for more deeply integrating +video conferencing use cases). + +As an alternative example: it's very unlikely that "how to visualise +Magnetic Resonance Imaging data over Matrix" would ever be added to the +Matrix spec (other than perhaps a custom event type in a wider +standardised Matrix event registry) given that the spec's existing +primitives of file transfer and extensible events (MSC1767) give +excellent tools for transferring and visualising arbitrary rich data. + +Supporting public search engines are likely to not require custom spec +features (other than possibly better bulk access APIs), given they can +be implemented as clients using the existing CS API. An exception could +be API features required by decentralised search infrastructure +(avoiding centralisation of power by a centralised search engine). + +Features such as reactions, threaded messages, editable messages, +spam/abuse/content filtering (and reputation systems), are all features +which would clearly benefit the whole Matrix ecosystem, and cannot be +implemented in an interoperable way using the current spec; so they +necessitate a spec change. + +# Process + +The process for submitting a Matrix Spec Change (MSC) Proposal in detail +is as follows: + +- Create a first draft of your proposal using [GitHub-flavored + Markdown](https://help.github.com/articles/basic-writing-and-formatting-syntax/) + - In the document, clearly state the problem being solved, and the + possible solutions being proposed for solving it and their + respective trade-offs. + - Proposal documents are intended to be as lightweight and + flexible as the author desires; there is no formal template; the + intention is to iterate as quickly as possible to get to a good + design. + - However, a [template with suggested + headers](https://github.com/matrix-org/matrix-doc/blob/master/proposals/0000-proposal-template.md) + is available to get you started if necessary. + - Take care in creating your proposal. Specify your intended + changes, and give reasoning to back them up. Changes without + justification will likely be poorly received by the community. +- Fork and make a PR to the + [matrix-doc](https://github.com/matrix-org/matrix-doc) repository. + The ID of your PR will become the MSC ID for the lifetime of your + proposal. + - The proposal must live in the `proposals/` directory with a + filename that follows the format `1234-my-new-proposal.md` where + `1234` is the MSC ID. + - Your PR description must include a link to the rendered Markdown + document and a summary of the proposal. + - It is often very helpful to link any related MSCs or [matrix-doc + issues](https://github.com/matrix-org/matrix-doc/issues) to give + context for the proposal. + - Additionally, please be sure to sign off your proposal PR as per + the guidelines listed on + [CONTRIBUTING.rst](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst). +- Gather feedback as widely as possible. + - The aim is to get maximum consensus towards an optimal solution. + Sometimes trade-offs are required to meet this goal. Decisions + should be made to the benefit of all major use cases. + - A good place to ask for feedback on a specific proposal is + [\#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). + If preferred, an alternative room can be created and advertised + in \#matrix-spec:matrix.org. Please also link to the room in + your PR description. + - For additional discussion areas, know that + \#matrix-dev:matrix.org is for developers using existing Matrix + APIs, \#matrix:matrix.org is for users trying to run Matrix apps + (clients & servers) and \#matrix-architecture:matrix.org is for + cross-cutting discussion of Matrix's architectural design. + - The point of the spec proposal process is to be collaborative + rather than competitive, and to try to solve the problem in + question with the optimal set of trade-offs. The author should + neutrally gather the various viewpoints and get consensus, but + this can sometimes be time-consuming (or the author may be + biased), in which case an impartial 'shepherd' can be assigned + to help guide the proposal through this process instead. A + shepherd is typically a neutral party from the Spec Core Team or + an experienced member of the community. There is no formal + process for assignment. Simply ask for a shepherd to help get + your proposal through and one will be assigned based on + availability. Having a shepherd is not a requirement for + proposal acceptance. +- Members of the Spec Core Team and community will review and discuss + the PR in the comments and in relevant rooms on Matrix. Discussion + outside of GitHub should be summarised in a comment on the PR. +- When a member of the Spec Core Team believes that no new discussion + points are being made, and the proposal has suitable evidence of + working (see [implementing a proposal](#implementing-a-proposal) + below), they will propose a motion for a final comment period (FCP), + along with a *disposition* of either merge, close or postpone. This + FCP is provided to allow a short period of time for any invested + party to provide a final objection before a major decision is made. + If sufficient reasoning is given, an FCP can be cancelled. It is + often preceded by a comment summarising the current state of the + discussion, along with reasoning for its occurrence. +- A concern can be raised by a Spec Core Team member at any time, + which will block an FCP from beginning. An FCP will only begin when + 75% of the members of the Spec Core Team agree on its outcome, and + all existing concerns have been resolved. +- The FCP will then begin and last for 5 days, giving anyone else some + time to speak up before it concludes. On its conclusion, the + disposition of the FCP will be carried out. If sufficient reasoning + against the disposition is raised, the FCP can be cancelled and the + MSC will continue to evolve accordingly. +- Once the proposal has been accepted and merged, it is time to submit + the actual change to the Specification that your proposal reasoned + about. This is known as a spec PR. However in order for the spec PR + to be accepted, an implementation **must** be shown to prove that it + works well in practice. A link to the implementation should be + included in the PR description. In addition, any significant + unforeseen changes to the original idea found during this process + will warrant another MSC. Any minor, non-fundamental changes are + allowed but **must** be documented in the original proposal + document. This ensures that someone reading a proposal in the future + doesn't assume old information wasn't merged into the spec. + - Similar to the proposal PR, please sign off the spec PR as per + the guidelines on + [CONTRIBUTING.rst](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst). +- Your PR will then be reviewed and hopefully merged on the grounds it + is implemented sufficiently. If so, then give yourself a pat on the + back knowing you've contributed to the Matrix protocol for the + benefit of users and developers alike :) + +The process for handling proposals is shown visually in the following +diagram. Note that the lifetime of a proposal is tracked through the +corresponding labels for each stage on the +[matrix-doc](https://github.com/matrix-org/matrix-doc) issue and pull +request trackers. + + + + + Proposals | Spec PRs | Additional States + +-------+ | +------+ | +---------------+ + | | + +----------------------+ | +---------+ | +-----------+ + | | | | | | | | + | Proposal | | +------= Spec PR | | | Postponed | + | Drafting and Initial | | | | Missing | | | | + | Feedback Gathering | | | | | | +-----------+ + | | | | +----+----+ | + +----------+-----------+ | | | | +----------+ + | | | v | | | + v | | +-----------------+ | | Closed | + +-------------------+ | | | | | | | + | | | | | Spec PR Created | | +----------+ + | Proposal PR | | | | and In Review | | + | In Review | | | | | | + | | | | +--------+--------+ | + +---------+---------+ | | | | + | | | v | + v | | +-----------+ | + +----------------------+ | | | | | + | | | | | Spec PR | | + | Proposed Final | | | | Merged! | | + | Comment Period | | | | | | + | | | | +-----------+ | + +----------+-----------+ | | | + | | | | + v | | | + +----------------------+ | | | + | | | | | + | Final Comment Period | | | | + | | | | | + +----------+-----------+ | | | + | | | | + v | | | + +----------------------+ | | | + | | | | | + | Final Comment Period | | | | + | Complete | | | | + | | | | | + +----------+-----------+ | | | + | | | | + +-----------------+ | + | | + + + + +# Lifetime States + +**Note:** All labels are to be placed on the proposal PR. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameGitHub LabelDescription
Proposal Drafting and FeedbackN/AA proposal document which is still work-in-progress but is being shared to incorporate feedback. Please prefix your proposal's title with [WIP] to make it easier for reviewers to skim their notifications list.
Proposal In Reviewproposal-in-reviewA proposal document which is now ready and waiting for review by the Spec Core Team and community
Proposed Final Comment Periodproposed-final-comment-periodCurrently awaiting signoff of a 75% majority of team members in order to enter the final comment period
Final Comment Periodfinal-comment-periodA proposal document which has reached final comment period either for merge, closure or postponement
Final Comment Period Completefinished-final-comment-periodThe final comment period has been completed. Waiting for a demonstration implementation
Spec PR Missingspec-pr-missingThe proposal has been agreed, and proven with a demonstration implementation. Waiting for a PR against the Spec
Spec PR In Reviewspec-pr-in-reviewThe spec PR has been written, and is currently under review
Spec PR MergedmergedA proposal with a sufficient working implementation and whose Spec PR has been merged!

Postponed

proposal-postponed

A proposal that is temporarily blocked or a feature that may not be useful currently but perhaps sometime in the future

Closedproposal-closedA proposal which has been reviewed and deemed unsuitable for acceptance
ObsoleteobsoleteA proposal which has been made obsolete by another proposal or decision elsewhere.
+ +# Categories + +We use category labels on MSCs to place them into a track of work. The +Spec Core Team decides which of the tracks they are focusing on for the +next while and generally makes an effort to pull MSCs out of that +category when possible. + +The current categories are: + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameGitHub LabelDescription
Corekind:coreImportant for the protocol's success.
Featurekind:featureNice to have additions to the spec.
Maintenancekind:maintenanceFixes or clarifies existing spec.
+ +Some examples of core MSCs would be aggregations, cross-signing, and +groups/communities. These are the sorts of things that if not +implemented could cause the protocol to fail or become second-class. +Features would be areas like enhanced media APIs, new transports, and +bookmarks in comparison. Finally, maintenance MSCs would include +improving error codes, clarifying what is required of an API, and adding +properties to an API which makes it easier to use. + +The Spec Core Team assigns a category to each MSC based on the +descriptions above. This can mean that new MSCs get categorized into an +area the team isn't focused on, though that can always change as +priorities evolve. We still encourage that MSCs be opened, even if not +the focus for the time being, as they can still make progress and even +be merged without the Spec Core Team focusing on them specifically. + +# Implementing a proposal + +As part of the proposal process the spec core team will require evidence +of the MSC working in order for it to move into FCP. This can usually be +a branch/pull request to whichever implementation of choice that proves +the MSC works in practice, though in some cases the MSC itself will be +small enough to be considered proven. Where it's unclear if an MSC will +require an implementation proof, ask in +[\#matrix-spec:matrix.org](https://matrix.to/#/#matrix-spec:matrix.org). + +## Early release of an MSC/idea + +To help facilitate early releases of software dependent on a spec +release, implementations are required to use the following process to +ensure that the official Matrix namespace is not cluttered with +development or testing data. + +Note + +Unreleased implementations (including proofs-of-concept demonstrating +that a particular MSC works) do not have to follow this process. + +1. Have an idea for a feature. +2. Implement the feature using unstable endpoints, vendor prefixes, and + unstable feature flags as appropriate. + - When using unstable endpoints, they MUST include a vendor + prefix. For example: + `/_matrix/client/unstable/com.example/login`. Vendor prefixes + throughout Matrix always use the Java package naming convention. + The MSC for the feature should identify which preferred vendor + prefix is to be used by early adopters. + - Note that unstable namespaces do not automatically inherit + endpoints from stable namespaces: for example, the fact that + `/_matrix/client/r0/sync` exists does not imply that + `/_matrix/client/unstable/com.example/sync` exists. + - If the client needs to be sure the server supports the feature, + an unstable feature flag that MUST be vendor prefixed is to be + used. This kind of flag shows up in the `unstable_features` + section of `/versions` as, for example, `com.example.new_login`. + The MSC for the feature should identify which preferred feature + flag is to be used by early adopters. + - When using this approach correctly, the implementation can + ship/release the feature at any time, so long as the + implementation is able to accept the technical debt that results + from needing to provide adequate backwards and forwards + compatibility. The implementation MUST support the flag (and + server-side implementation) disappearing and be generally safe + for users. Note that implementations early in the MSC review + process may also be required to provide backwards compatibility + with earlier editions of the proposal. + - If the implementation cannot support the technical debt (or if + it's impossible to provide forwards/backwards compatibility - + e.g. a user authentication change which can't be safely rolled + back), the implementation should not attempt to implement the + feature and should instead wait for a spec release. + - If at any point after early release, the idea changes in a + backwards-incompatible way, the feature flag should also change + so that implementations can adapt as needed. +3. In parallel, or ahead of implementation, open an MSC and solicit + review per above. +4. Before FCP can be called, the Spec Core Team will require evidence + of the MSC working as proposed. A typical example of this is an + implementation of the MSC, though the implementation does not need + to be shipped anywhere and can therefore avoid the + forwards/backwards compatibility concerns mentioned here. +5. The FCP process is completed, and assuming nothing is flagged the + MSC lands. +6. A spec PR is written to incorporate the changes into Matrix. +7. A spec release happens. +8. Implementations switch to using stable prefixes (e.g.: `/r0`) if the + server supports the specification version released. If the server + doesn't advertise the specification version, but does have the + feature flag, unstable prefixes should still be used. +9. A transition period of about 2 months starts immediately after the + spec release, before implementations start to encourage other + implementations to switch to stable endpoints. For example, a server + implementation should start asking client implementations to support + the stable endpoints 2 months after the spec release, if they + haven't already. The same applies in the reverse: if clients cannot + switch to stable prefixes because server implementations haven't + started supporting the new spec release, some noise should be raised + in the general direction of the implementation. + +Note + +MSCs MUST still describe what the stable endpoints/feature looks like +with a note towards the bottom for what the unstable feature +flag/prefixes are. For example, an MSC would propose /\_matrix/client/r0/new/endpoint, not /\_matrix/client/unstable/ +com.example/new/endpoint. + +In summary: + +- Implementations MUST NOT use stable endpoints before the MSC is in + the spec. This includes NOT using stable endpoints in the period + between completion of FCP and release of the spec. passed. +- Implementations are able to ship features that are exposed to users + by default before an MSC has been merged to the spec, provided they + follow the process above. +- Implementations SHOULD be wary of the technical debt they are + incurring by moving faster than the spec. +- The vendor prefix is chosen by the developer of the feature, using + the Java package naming convention. The foundation's preferred + vendor prefix is org.matrix. +- The vendor prefixes, unstable feature flags, and unstable endpoints + should be included in the MSC, though the MSC MUST be written in a + way that proposes new stable endpoints. Typically this is solved by + a small table at the bottom mapping the various values from stable + to unstable. + +# Proposal Tracking + +This is a living document generated from the list of proposals on the +issue and pull request trackers of the +[matrix-doc](https://github.com/matrix-org/matrix-doc) repo. + +We use labels and some metadata in MSC PR descriptions to generate this +page. Labels are assigned by the Spec Core Team whilst triaging the +proposals based on those which exist in the +[matrix-doc](https://github.com/matrix-org/matrix-doc) repo already. + +It is worth mentioning that a previous version of the MSC process used a +mixture of GitHub issues and PRs, leading to some MSC numbers deriving +from GitHub issue IDs instead. A useful feature of GitHub is that it +does automatically resolve to an issue, if an issue ID is placed in a +pull URL. This means that + will correctly +resolve to the desired MSC, whether it started as an issue or a PR. + +Other metadata: + +- The MSC number is taken from the GitHub Pull Request ID. This is + carried for the lifetime of the proposal. These IDs do not + necessarily represent a chronological order. +- The GitHub PR title will act as the MSC's title. +- Please link to the spec PR (if any) by adding a "PRs: \#1234" line + in the issue description. +- The creation date is taken from the GitHub PR, but can be overridden + by adding a "Date: yyyy-mm-dd" line in the PR description. +- Updated Date is taken from GitHub. +- Author is the creator of the MSC PR, but can be overridden by adding + an "Author: @username" line in the body of the issue description. + Please make sure @username is a GitHub user (include the @!) +- A shepherd can be assigned by adding a "Shepherd: @username" line in + the issue description. Again, make sure this is a real GitHub user. diff --git a/content/push-gateway-api.md b/content/push-gateway-api.md index ab4c35fa..022e6c5b 100644 --- a/content/push-gateway-api.md +++ b/content/push-gateway-api.md @@ -3,3 +3,81 @@ title: "Push Gateway API" weight: 50 type: docs --- + +# Push Gateway API + +{{unstable\_warning\_block\_PUSH\_GATEWAY\_RELEASE\_LABEL}} + +Clients may want to receive push notifications when events are received +at the homeserver. This is managed by a distinct entity called the Push +Gateway. + +Table of Contents + +## Changelog + +**Version: %PUSH\_GATEWAY\_RELEASE\_LABEL%** + +{{push\_gateway\_changelog}} + +This version of the specification is generated from +[matrix-doc](https://github.com/matrix-org/matrix-doc) as of Git commit +[{{git\_version}}](https://github.com/matrix-org/matrix-doc/tree/%7B%7Bgit_rev%7D%7D). + +For the full historical changelog, see + + +### Other versions of this specification + +The following other versions are also available, in reverse +chronological order: + +- [HEAD](https://matrix.org/docs/spec/push_gateway/unstable.html): + Includes all changes since the latest versioned release. +- [r0.1.0](https://matrix.org/docs/spec/push_gateway/r0.1.0.html) + +## Overview + +A client's homeserver forwards information about received events to the +push gateway. The gateway then submits a push notification to the push +notification provider (e.g. APNS, GCM). + + +--------------------+ +-------------------+ + Matrix HTTP | | | | + Notification Protocol | App Developer | | Device Vendor | + | | | | + +-------------------+ | +----------------+ | | +---------------+ | + | | | | | | | | | | + | Matrix homeserver +-----> Push Gateway +------> Push Provider | | + | | | | | | | | | | + +-^-----------------+ | +----------------+ | | +----+----------+ | + | | | | | | + Matrix | | | | | | + Client/Server API + | | | | | + | | +--------------------+ +-------------------+ + | +--+-+ | + | | <-------------------------------------------+ + +---+ | + | | Provider Push Protocol + +----+ + + Mobile Device or Client + +## Homeserver behaviour + +This describes the format used by "HTTP" pushers to send notifications +of events to Push Gateways. If the endpoint returns an HTTP error code, +the homeserver SHOULD retry for a reasonable amount of time using +exponential backoff. + +When pushing notifications for events, the homeserver is expected to +include all of the event-related fields in the `/notify` request. When +the homeserver is performing a push where the `format` is +`"event_id_only"`, only the `event_id`, `room_id`, `counts`, and +`devices` are required to be populated. + +Note that most of the values and behaviour of this endpoint is described +by the Client-Server API's [Push +Module](../client_server/%CLIENT_RELEASE_LABEL%.html#module-push). + +{{push\_notifier\_push\_http\_api}} diff --git a/content/rooms/v1.md b/content/rooms/v1.md index 912e57b4..532e547b 100644 --- a/content/rooms/v1.md +++ b/content/rooms/v1.md @@ -3,3 +3,287 @@ title: Room Version 1 type: docs weight: 10 --- + +# Room Version 1 + +This room version is the first ever version for rooms, and contains the +building blocks for other room versions. + +Table of Contents + +## Client considerations + +Clients may need to consider some algorithms performed by the server for +their own implementation. + +### Redactions + +Upon receipt of a redaction event, the server must strip off any keys +not in the following list: + +- `event_id` +- `type` +- `room_id` +- `sender` +- `state_key` +- `content` +- `hashes` +- `signatures` +- `depth` +- `prev_events` +- `prev_state` +- `auth_events` +- `origin` +- `origin_server_ts` +- `membership` + +The content object must also be stripped of all keys, unless it is one +of one of the following event types: + +- `m.room.member` allows key `membership`. +- `m.room.create` allows key `creator`. +- `m.room.join_rules` allows key `join_rule`. +- `m.room.power_levels` allows keys `ban`, `events`, `events_default`, + `kick`, `redact`, `state_default`, `users`, `users_default`. +- `m.room.aliases` allows key `aliases`. +- `m.room.history_visibility` allows key `history_visibility`. + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. + +The algorithms defined here should only apply to version 1 rooms. Other +algorithms may be used by other room versions, and as such servers +should be aware of which version room they are dealing with prior to +executing a given algorithm. + +Warning + +Although there are many rooms using room version 1, it is known to have +undesirable effects. Servers implementing support for room version 1 +should be aware that restrictions should be generally relaxed and that +inconsistencies may occur. + +### State resolution + +Warning + +Room version 1 is known to have bugs that can cause the state of rooms +to reset to older versions of the room's state. For example this could +mean that users who had joined the room may be removed from the room, +admins and moderators could lose their power level, and users who have +been banned from the room may be able to rejoin. Other state events such +as the the room's name or topic could also reset to a previous version. + +This is fixed in the state resolution algorithm introduced in room +version 2. + +The room state *S*′(*E*) after an event *E* is defined in terms of the +room state *S*(*E*) before *E*, and depends on whether *E* is a state +event or a message event: + +- If *E* is a message event, then *S*′(*E*) = *S*(*E*). +- If *E* is a state event, then *S*′(*E*) is *S*(*E*), except that its + entry corresponding to *E*'s `event_type` and `state_key` is + replaced by *E*'s `event_id`. + +The room state *S*(*E*) before *E* is the *resolution* of the set of +states {*S*′(*E*′), *S*′(*E*″), …} consisting of the states after each +of *E*'s `prev_event`s {*E*′, *E*″, …}. + +The *resolution* of a set of states is defined as follows. The resolved +state is built up in a number of passes; here we use *R* to refer to the +results of the resolution so far. + +- Start by setting *R* to the union of the states to be resolved, + excluding any *conflicting* events. +- First we resolve conflicts between `m.room.power_levels` events. If + there is no conflict, this step is skipped, otherwise: + - Assemble all the `m.room.power_levels` events from the states to + be resolved into a list. + - Sort the list by ascending `depth` then descending + `sha1(event_id)`. + - Add the first event in the list to *R*. + - For each subsequent event in the list, check that the event + would be allowed by the authorization rules for a room in state + *R*. If the event would be allowed, then update *R* with the + event and continue with the next event in the list. If it would + not be allowed, stop and continue below with `m.room.join_rules` + events. +- Repeat the above process for conflicts between `m.room.join_rules` + events. +- Repeat the above process for conflicts between `m.room.member` + events. +- No other events affect the authorization rules, so for all other + conflicts, just pick the event with the highest depth and lowest + `sha1(event_id)` that passes authentication in *R* and add it to + *R*. + +A *conflict* occurs between states where those states have different +`event_ids` for the same `(event_type, state_key)`. The events thus +affected are said to be *conflicting* events. + +### Authorization rules + +The types of state events that affect authorization are: + +- `m.room.create` +- `m.room.member` +- `m.room.join_rules` +- `m.room.power_levels` +- `m.room.third_party_invite` + +Note + +Power levels are inferred from defaults when not explicitly supplied. +For example, mentions of the `sender`'s power level can also refer to +the default power level for users in the room. + +The rules are as follows: + +1. If type is `m.room.create`: + 1. If it has any previous events, reject. + 2. If the domain of the `room_id` does not match the domain of the + `sender`, reject. + 3. If `content.room_version` is present and is not a recognised + version, reject. + 4. If `content` has no `creator` field, reject. + 5. Otherwise, allow. +2. Reject if event has `auth_events` that: + 1. have duplicate entries for a given `type` and `state_key` pair + 2. have entries whose `type` and `state_key` don't match those + specified by the [auth events + selection](../server_server/%SERVER_RELEASE_LABEL%.html#auth-events-selection) + algorithm described in the server specification. +3. If event does not have a `m.room.create` in its `auth_events`, + reject. +4. If type is `m.room.aliases`: + 1. If event has no `state_key`, reject. + 2. If sender's domain doesn't matches `state_key`, reject. + 3. Otherwise, allow. +5. If type is `m.room.member`: + 1. If no `state_key` key or `membership` key in `content`, reject. + 2. If `membership` is `join`: + 1. If the only previous event is an `m.room.create` and the + `state_key` is the creator, allow. + 2. If the `sender` does not match `state_key`, reject. + 3. If the `sender` is banned, reject. + 4. If the `join_rule` is `invite` then allow if membership + state is `invite` or `join`. + 5. If the `join_rule` is `public`, allow. + 6. Otherwise, reject. + 3. If `membership` is `invite`: + 1. If `content` has `third_party_invite` key: + 1. If *target user* is banned, reject. + 2. If `content.third_party_invite` does not have a `signed` + key, reject. + 3. If `signed` does not have `mxid` and `token` keys, + reject. + 4. If `mxid` does not match `state_key`, reject. + 5. If there is no `m.room.third_party_invite` event in the + current room state with `state_key` matching `token`, + reject. + 6. If `sender` does not match `sender` of the + `m.room.third_party_invite`, reject. + 7. If any signature in `signed` matches any public key in + the `m.room.third_party_invite` event, allow. The public + keys are in `content` of `m.room.third_party_invite` as: + 1. A single public key in the `public_key` field. + 2. A list of public keys in the `public_keys` field. + 8. Otherwise, reject. + 2. If the `sender`'s current membership state is not `join`, + reject. + 3. If *target user*'s current membership state is `join` or + `ban`, reject. + 4. If the `sender`'s power level is greater than or equal to + the *invite level*, allow. + 5. Otherwise, reject. + 4. If `membership` is `leave`: + 1. If the `sender` matches `state_key`, allow if and only if + that user's current membership state is `invite` or `join`. + 2. If the `sender`'s current membership state is not `join`, + reject. + 3. If the *target user*'s current membership state is `ban`, + and the `sender`'s power level is less than the *ban level*, + reject. + 4. If the `sender`'s power level is greater than or equal to + the *kick level*, and the *target user*'s power level is + less than the `sender`'s power level, allow. + 5. Otherwise, reject. + 5. If `membership` is `ban`: + 1. If the `sender`'s current membership state is not `join`, + reject. + 2. If the `sender`'s power level is greater than or equal to + the *ban level*, and the *target user*'s power level is less + than the `sender`'s power level, allow. + 3. Otherwise, reject. + 6. Otherwise, the membership is unknown. Reject. +6. If the `sender`'s current membership state is not `join`, reject. +7. If type is `m.room.third_party_invite`: + 1. Allow if and only if `sender`'s current power level is greater + than or equal to the *invite level*. +8. If the event type's *required power level* is greater than the + `sender`'s power level, reject. +9. If the event has a `state_key` that starts with an `@` and does not + match the `sender`, reject. +10. If type is `m.room.power_levels`: + 1. If `users` key in `content` is not a dictionary with keys that + are valid user IDs with values that are integers (or a string + that is an integer), reject. + 2. If there is no previous `m.room.power_levels` event in the room, + allow. + 3. For the keys `users_default`, `events_default`, `state_default`, + `ban`, `redact`, `kick`, `invite` check if they were added, + changed or removed. For each found alteration: + 1. If the current value is higher than the `sender`'s current + power level, reject. + 2. If the new value is higher than the `sender`'s current power + level, reject. + 4. For each entry being added, changed or removed in both the + `events` and `users` keys: + 1. If the current value is higher than the `sender`'s current + power level, reject. + 2. If the new value is higher than the `sender`'s current power + level, reject. + 5. For each entry being changed under the `users` key, other than + the `sender`'s own entry: + 1. If the current value is equal to the `sender`'s current + power level, reject. + 6. Otherwise, allow. +11. If type is `m.room.redaction`: + 1. If the `sender`'s power level is greater than or equal to the + *redact level*, allow. + 2. If the domain of the `event_id` of the event being redacted is + the same as the domain of the `event_id` of the + `m.room.redaction`, allow. + 3. Otherwise, reject. +12. Otherwise, allow. + +Note + +Some consequences of these rules: + +- Unless you are a member of the room, the only permitted operations + (apart from the initial create/join) are: joining a public room; + accepting or rejecting an invitation to a room. +- To unban somebody, you must have power level greater than or equal + to both the kick *and* ban levels, *and* greater than the target + user's power level. + +### Event format + +Events in version 1 rooms have the following structure: + +{{definition\_ss\_pdu}} + +### Canonical JSON + +Servers MUST NOT strictly enforce the JSON format specified in the +[appendices](../appendices.html#canonical-json) for the reasons +described there. diff --git a/content/rooms/v2.md b/content/rooms/v2.md index 328726fc..806ab8e2 100644 --- a/content/rooms/v2.md +++ b/content/rooms/v2.md @@ -3,3 +3,190 @@ title: Room Version 2 type: docs weight: 20 --- + +# Room Version 2 + +This room version builds off of [version 1](v1.html) with an improved +state resolution algorithm. + +Table of Contents + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the details contained here, and can safely ignore their +presence. + +Room version 2 uses the base components of [room version 1](v1.html), +changing only the state resolution algorithm. + +### State resolution + +The room state *S*′(*E*) after an event *E* is defined in terms of the +room state *S*(*E*) before *E*, and depends on whether *E* is a state +event or a message event: + +- If *E* is a message event, then *S*′(*E*) = *S*(*E*). +- If *E* is a state event, then *S*′(*E*) is *S*(*E*), except that its + entry corresponding to *E*'s `event_type` and `state_key` is + replaced by *E*'s `event_id`. + +The room state *S*(*E*) before *E* is the *resolution* of the set of +states {*S*′(*E*1), *S*′(*E*2), …} consisting of +the states after each of *E*'s `prev_event`s +{*E*1, *E*2, …}, where the resolution of a set of +states is given in the algorithm below. + +#### Definitions + +The state resolution algorithm for version 2 rooms uses the following +definitions, given the set of room states +{*S*1, *S*2, …}: + +Power events +A *power event* is a state event with type `m.room.power_levels` or +`m.room.join_rules`, or a state event with type `m.room.member` where +the `membership` is `leave` or `ban` and the `sender` does not match the +`state_key`. The idea behind this is that power events are events that +might remove someone's ability to do something in the room. + +Unconflicted state map and conflicted state set +The *unconflicted state map* is the state where the value of each key +exists and is the same in each state *S**i*. The *conflicted +state set* is the set of all other state events. Note that the +unconflicted state map only has one event per `(event_type, state_key)`, +whereas the conflicted state set may have multiple events. + +Auth difference +The *auth difference* is calculated by first calculating the full auth +chain for each state *S**i*, that is the union of the auth +chains for each event in *S**i*, and then taking every event +that doesn't appear in every auth chain. If *C**i* is the +full auth chain of *S**i*, then the auth difference is + ∪ *C**i* −  ∩ *C**i*. + +Full conflicted set +The *full conflicted set* is the union of the conflicted state set and +the auth difference. + +Reverse topological power ordering +The *reverse topological power ordering* of a set of events is the +lexicographically smallest topological ordering based on the DAG formed +by auth events. The reverse topological power ordering is ordered from +earliest event to latest. For comparing two topological orderings to +determine which is the lexicographically smallest, the following +comparison relation on events is used: for events *x* and *y*, +*x* < *y* if + +1. *x*'s sender has *greater* power level than *y*'s sender, when + looking at their respective `auth_event`s; or +2. the senders have the same power level, but *x*'s `origin_server_ts` + is *less* than *y*'s `origin_server_ts`; or +3. the senders have the same power level and the events have the same + `origin_server_ts`, but *x*'s `event_id` is *less* than *y*'s + `event_id`. + +The reverse topological power ordering can be found by sorting the +events using Kahn's algorithm for topological sorting, and at each step +selecting, among all the candidate vertices, the smallest vertex using +the above comparison relation. + +Mainline ordering +Given an `m.room.power_levels` event *P*, the *mainline of* *P* is the +list of events generated by starting with *P* and recursively taking the +`m.room.power_levels` events from the `auth_events`, ordered such that +*P* is last. Given another event *e*, the *closest mainline event to* +*e* is the first event encountered in the mainline when iteratively +descending through the `m.room.power_levels` events in the `auth_events` +starting at *e*. If no mainline event is encountered when iteratively +descending through the `m.room.power_levels` events, then the closest +mainline event to *e* can be considered to be a dummy event that is +before any other event in the mainline of *P* for the purposes of +condition 1 below. + +The *mainline ordering based on* *P* of a set of events is the ordering, +from smallest to largest, using the following comparison relation on +events: for events *x* and *y*, *x* < *y* if + +1. the closest mainline event to *x* appears *before* the closest + mainline event to *y*; or +2. the closest mainline events are the same, but *x*'s + `origin_server_ts` is *less* than *y*'s `origin_server_ts`; or +3. the closest mainline events are the same and the events have the + same `origin_server_ts`, but *x*'s `event_id` is *less* than *y*'s + `event_id`. + +Iterative auth checks +The *iterative auth checks algorithm* takes as input an initial room +state and a sorted list of state events, and constructs a new room state +by iterating through the event list and applying the state event to the +room state if the state event is allowed by the [authorization +rules](../server_server/%SERVER_RELEASE_LABEL%.html#authorization-rules). +If the state event is not allowed by the authorization rules, then the +event is ignored. If a `(event_type, state_key)` key that is required +for checking the authorization rules is not present in the state, then +the appropriate state event from the event's `auth_events` is used if +the auth event is not rejected. + +#### Algorithm + +The *resolution* of a set of states is obtained as follows: + +1. Take all *power events* and any events in their auth chains, + recursively, that appear in the *full conflicted set* and order them + by the *reverse topological power ordering*. +2. Apply the *iterative auth checks algorithm*, starting from the + *unconflicted state map*, to the list of events from the previous + step to get a partially resolved state. +3. Take all remaining events that weren't picked in step 1 and order + them by the mainline ordering based on the power level in the + partially resolved state obtained in step 2. +4. Apply the *iterative auth checks algorithm* on the partial resolved + state and the list of events from the previous step. +5. Update the result by replacing any event with the event with the + same key from the *unconflicted state map*, if such an event exists, + to get the final resolved state. + +#### Rejected events + +Events that have been rejected due to failing auth based on the state at +the event (rather than based on their auth chain) are handled as usual +by the algorithm, unless otherwise specified. + +Note that no events rejected due to failure to auth against their auth +chain should appear in the process, as they should not appear in state +(the algorithm only uses events that appear in either the state sets or +in the auth chain of the events in the state sets). + +Rationale + +This helps ensure that different servers' view of state is more likely +to converge, since rejection state of an event may be different. This +can happen if a third server gives an incorrect version of the state +when a server joins a room via it (either due to being faulty or +malicious). Convergence of state is a desirable property as it ensures +that all users in the room have a (mostly) consistent view of the state +of the room. If the view of the state on different servers diverges it +can lead to bifurcation of the room due to e.g. servers disagreeing on +who is in the room. + +Intuitively, using rejected events feels dangerous, however: + +1. Servers cannot arbitrarily make up state, since they still need to + pass the auth checks based on the event's auth chain (e.g. they + can't grant themselves power levels if they didn't have them + before). +2. For a previously rejected event to pass auth there must be a set of + state that allows said event. A malicious server could therefore + produce a fork where it claims the state is that particular set of + state, duplicate the rejected event to point to that fork, and send + the event. The duplicated event would then pass the auth checks. + Ignoring rejected events would therefore not eliminate any potential + attack vectors. + +Rejected auth events are deliberately excluded from use in the iterative +auth checks, as auth events aren't re-authed (although non-auth events +are) during the iterative auth checks. diff --git a/content/rooms/v3.md b/content/rooms/v3.md index 2520b2e7..f8b3f89f 100644 --- a/content/rooms/v3.md +++ b/content/rooms/v3.md @@ -3,3 +3,102 @@ title: Room Version 3 type: docs weight: 30 --- + +# Room Version 3 + +This room version builds on [version 2](v2.html) with an improved event +format. + +Table of Contents + +## Client considerations + +This room version changes the format for event IDs sent to clients. +Clients should be aware that these event IDs may contain slashes and +other potentially problematic characters. Clients should be treating +event IDs as opaque identifiers and should not be attempting to parse +them into a usable form, just like with other room versions. + +Clients should expect to see event IDs changed from the format of +`$randomstring:example.org` to something like +`$acR1l0raoZnm60CBwAVgqbZqoO/mYU81xysh1u7XcJk` (note the lack of domain +and the potentially problematic slash). + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. + +Room version 3 uses the state resolution algorithm defined in [room +version 2](v2.html), and the event format defined here. + +### Event IDs + +Rationale + +In other room versions (namely version 1 and 2) the event ID is a +distinct field from the remainder of the event, which must be tracked as +such. This leads to complications where servers receive multiple events +with the same ID in either the same or different rooms where the server +cannot easily keep track of which event it should be using. By removing +the use of a dedicated event ID, servers are required to track the +hashes on an event to determine its ID. + +The event ID is the [reference +hash](../server_server/%SERVER_RELEASE_LABEL%.html#reference-hashes) of +the event encoded using [Unpadded +Base64](../appendices.html#unpadded-base64), prefixed with `$`. A +resulting event ID using this approach should look similar to +`$CD66HAED5npg6074c6pDtLKalHjVfYb2q4Q3LZgrW6o`. + +Event IDs should not be sent over federation to servers when the room +uses this room version. On the receiving end of an event, the server +should compute the relevant event ID for itself. + +Additionally, the `auth_events` and `prev_events` have had a format +change compared to other room versions to make it easier to handle. +Instead of a tuple of values, they are now plain lists of events. + +{{definition\_ss\_pdu\_v3}} + +### Changes to APIs + +Due to the event ID being removed from the event, some APIs need to +change. All APIs which currently accept an event ID must do so with the +new format. Servers must append the calculated event ID to all events +sent to clients where an event ID would normally be expected. + +Because the format of events has changed, servers must be aware of the +room version where the event resides so that the server may parse and +handle the event. The federation API has taken this concern into +consideration by ensuring that servers are aware of (or can find) the +room version during a request. + +### Authorization rules for events + +The authorization rules for a given event have changed in this room +version due to the change in event format: + +- The event no longer needs to be signed by the domain of the event ID + (as there is no domain in the event ID), but still needs to be + signed by the sender's domain. +- In past room versions, redactions were only permitted to enter the + DAG if the sender's domain matched the domain in the event ID being + redacted, or the sender had appropriate permissions per the power + levels. Due to servers now not being able to determine where an + event came from during event authorization, redaction events are + always accepted (provided the event is allowed by `events` and + `events_default` in the power levels). However, servers should not + apply or send redactions to clients until both the redaction event + and original event have been seen, and are valid. Servers should + only apply redactions to events where the sender's domains match, or + the sender of the redaction has the appropriate permissions per the + power levels. + +The remaining rules are the same as [room version +1](v1.html#authorization-rules). diff --git a/content/rooms/v4.md b/content/rooms/v4.md index 06381b0d..5e5cb0c4 100644 --- a/content/rooms/v4.md +++ b/content/rooms/v4.md @@ -3,3 +3,63 @@ title: Room Version 4 type: docs weight: 40 --- + +# Room Version 4 + +This room version builds on [version 3](v3.html) using a different +encoding for event IDs. + +Table of Contents + +## Client considerations + +This room version changes the format form event IDs sent to clients. +Clients should already be treating event IDs as opaque identifiers, and +should not be concerned with the format of them. Clients should still +encode the event ID when including it in a request path. + +Clients should expect to see event IDs changed from the format of +`$randomstring:example.org` to something like +`$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg` (note the lack of +domain). + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. + +Room version 4 uses the same algorithms defined in [room version +3](v3.html), however using URL-safe base64 to generate the event ID. + +### Event IDs + +Rationale + +Room version 3 generated event IDs that were difficult for client +implementations which were not encoding the event ID to function in +those rooms. It additionally raised concern due to the `/` character +being interpretted differently by some reverse proxy software, and +generally made administration harder. + +The event ID is the [reference +hash](../server_server/%SERVER_RELEASE_LABEL%.html#reference-hashes) of +the event encoded using a variation of [Unpadded +Base64](../appendices.html#unpadded-base64) which replaces the 62nd and +63rd characters with `-` and `_` instead of using `+` and `/`. This +matches [RFC4648's definition of URL-safe +base64](https://tools.ietf.org/html/rfc4648#section-5). Event IDs are +still prefixed with `$` and may result in looking like +`$Rqnc-F-dvnEYJTyHq_iKxU2bZ1CI92-kuZq3a5lr5Zg`. + +Just like in room version 3, event IDs should not be sent over +federation to servers when the room uses this room version. On the +receiving end of an event, the server should compute the relevant event +ID for itself. Room version 3 also changes the format of `auth_events` +and `prev_events` in a PDU. + +{{definition\_ss\_pdu\_v4}} diff --git a/content/rooms/v5.md b/content/rooms/v5.md index 455ba2e3..1cc1e1f4 100644 --- a/content/rooms/v5.md +++ b/content/rooms/v5.md @@ -3,3 +3,47 @@ title: Room Version 5 type: docs weight: 50 --- + +# Room Version 5 + +This room version builds on [version 4](v4.html) while enforcing signing +key validity periods for events. + +Table of Contents + +## Client considerations + +There are no specific requirements for clients in this room version. +Clients should be aware of event ID changes in [room version +4](v4.html), however. + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. + +Room version 5 uses the same algorithms defined in [room version +4](v4.html), ensuring that signing key validity is respected. + +### Signing key validity period + +When validating event signatures, servers MUST enforce the +`valid_until_ts` property from a key request is at least as large as the +`origin_server_ts` for the event being validated. Servers missing a copy +of the signing key MUST try to obtain one via the [GET +/\_matrix/key/v2/server](../server_server/%SERVER_RELEASE_LABEL%.html#get-matrix-key-v2-server-keyid) +or [POST +/\_matrix/key/v2/query](../server_server/%SERVER_RELEASE_LABEL%.html#post-matrix-key-v2-query) +APIs. When using the `/query` endpoint, servers MUST set the +`minimum_valid_until_ts` property to prompt the notary server to attempt +to refresh the key if appropriate. + +Servers MUST use the lesser of `valid_until_ts` and 7 days into the +future when determining if a key is valid. This is to avoid a situation +where an attacker publishes a key which is valid for a significant +amount of time without a way for the homeserver owner to revoke it. diff --git a/content/rooms/v6.md b/content/rooms/v6.md index db20e1e4..7854d9f2 100644 --- a/content/rooms/v6.md +++ b/content/rooms/v6.md @@ -3,3 +3,86 @@ title: Room Version 6 type: docs weight: 60 --- + +# Room Version 6 + +This room version builds on [version 5](v5.html) while changing various +authorization rules performed on events. + +Table of Contents + +## Client considerations + +The redaction algorithm has changed from [room version 1](v1.html) to +remove all rules against events of type `m.room.aliases`. Room versions +2, 3, 4, and 5 all use v1's redaction algorithm. The algorithm is +otherwise unchanged. + +## Server implementation components + +Warning + +The information contained in this section is strictly for server +implementors. Applications which use the Client-Server API are generally +unaffected by the intricacies contained here. The section above +regarding client considerations is the resource that Client-Server API +use cases should reference. + +Room version 6 makes the following alterations to algorithms described +in [room version 5](v5.html). + +### Redactions + +As mentioned in the client considerations portion of this specification, +all special meaning has been removed for events of type +`m.room.aliases`. The algorithm is otherwise unchanged. + +### Authorization rules for events + +Like redactions, all rules relating specifically to events of type +`m.room.aliases` are removed. They must still pass authorization checks +relating to state events. + +Additionally, the authorization rules for events of type +`m.room.power_levels` now include the content key `notifications`. This +new rule takes the place of the rule which checks the `events` and +`users` keys. + +For completeness, the changes to the auth rules can be represented as +follows: + + ... + + -If type is `m.room.aliases`: + - + - a. If event has no `state_key`, reject. + - b. If sender's domain doesn't matches `state_key`, reject. + - c. Otherwise, allow. + + ... + + If type is `m.room.power_levels`: + + ... + + - * For each entry being added, changed or removed in both the `events` and `users` keys: + + * For each entry being added, changed or removed in the `events`, `users`, and `notifications` keys: + + i. If the current value is higher than the `sender`'s current power level, reject. + + ii. If the new value is higher than the `sender`'s current power level, reject. + + ... + +The remaining rules are the same as in [room version +3](v3.html#authorization-rules-for-events) (the last inherited room +version to specify the authorization rules). + +### Canonical JSON + +Servers MUST strictly enforce the JSON format specified in the +[appendices](../appendices.html#canonical-json). This translates to a +400 `M_BAD_JSON` error on most endpoints, or discarding of events over +federation. For example, the Federation API's `/send` endpoint would +discard the event whereas the Client Server API's `/send/{eventType}` +endpoint would return a `M_BAD_JSON` error. diff --git a/content/server-server-api.md b/content/server-server-api.md index 1203098a..00f33afa 100644 --- a/content/server-server-api.md +++ b/content/server-server-api.md @@ -3,3 +3,1177 @@ title: "Server-Server API" weight: 20 type: docs --- + +# Federation API + +{{unstable\_warning\_block\_SERVER\_RELEASE\_LABEL}} + +Matrix homeservers use the Federation APIs (also known as server-server +APIs) to communicate with each other. Homeservers use these APIs to push +messages to each other in real-time, to retrieve historic messages from +each other, and to query profile and presence information about users on +each other's servers. + +The APIs are implemented using HTTPS requests between each of the +servers. These HTTPS requests are strongly authenticated using public +key signatures at the TLS transport layer and using public key +signatures in HTTP Authorization headers at the HTTP layer. + +There are three main kinds of communication that occur between +homeservers: + +Persisted Data Units (PDUs): +These events are broadcast from one homeserver to any others that have +joined the same room (identified by Room ID). They are persisted in +long-term storage and record the history of messages and state for a +room. + +Like email, it is the responsibility of the originating server of a PDU +to deliver that event to its recipient servers. However PDUs are signed +using the originating server's private key so that it is possible to +deliver them through third-party servers. + +Ephemeral Data Units (EDUs): +These events are pushed between pairs of homeservers. They are not +persisted and are not part of the history of a room, nor does the +receiving homeserver have to reply to them. + +Queries: +These are single request/response interactions between a given pair of +servers, initiated by one side sending an HTTPS GET request to obtain +some information, and responded by the other. They are not persisted and +contain no long-term significant history. They simply request a snapshot +state at the instant the query is made. + +EDUs and PDUs are further wrapped in an envelope called a Transaction, +which is transferred from the origin to the destination homeserver using +an HTTPS PUT request. + +Table of Contents + +## Changelog + +**Version: %SERVER\_RELEASE\_LABEL%** + +{{server\_server\_changelog}} + +This version of the specification is generated from +[matrix-doc](https://github.com/matrix-org/matrix-doc) as of Git commit +[{{git\_version}}](https://github.com/matrix-org/matrix-doc/tree/%7B%7Bgit_rev%7D%7D). + +For the full historical changelog, see + + +### Other versions of this specification + +The following other versions are also available, in reverse +chronological order: + +- [HEAD](https://matrix.org/docs/spec/server_server/unstable.html): + Includes all changes since the latest versioned release. +- [r0.1.4](https://matrix.org/docs/spec/server_server/r0.1.4.html) +- [r0.1.3](https://matrix.org/docs/spec/server_server/r0.1.3.html) +- [r0.1.2](https://matrix.org/docs/spec/server_server/r0.1.2.html) +- [r0.1.1](https://matrix.org/docs/spec/server_server/r0.1.1.html) +- [r0.1.0](https://matrix.org/docs/spec/server_server/r0.1.0.html) + +## API standards + +The mandatory baseline for client-server communication in Matrix is +exchanging JSON objects over HTTP APIs. More efficient optional +transports will in future be supported as optional extensions - e.g. a +packed binary encoding over stream-cipher encrypted TCP socket for +low-bandwidth/low-roundtrip mobile usage. For the default HTTP +transport, all API calls use a Content-Type of `application/json`. In +addition, all strings MUST be encoded as UTF-8. + +## Server discovery + +### Resolving server names + +Each Matrix homeserver is identified by a server name consisting of a +hostname and an optional port, as described by the +[grammar](../appendices.html#server-name). Where applicable, a delegated +server name uses the same grammar. + +Server names are resolved to an IP address and port to connect to, and +have various conditions affecting which certificates and `Host` headers +to send. The process overall is as follows: + +1. If the hostname is an IP literal, then that IP address should be + used, together with the given port number, or 8448 if no port is + given. The target server must present a valid certificate for the IP + address. The `Host` header in the request should be set to the + server name, including the port if the server name included one. +2. If the hostname is not an IP literal, and the server name includes + an explicit port, resolve the IP address using AAAA or A records. + Requests are made to the resolved IP address and given port with a + `Host` header of the original server name (with port). The target + server must present a valid certificate for the hostname. +3. If the hostname is not an IP literal, a regular HTTPS request is + made to `https:///.well-known/matrix/server`, expecting + the schema defined later in this section. 30x redirects should be + followed, however redirection loops should be avoided. Responses + (successful or otherwise) to the `/.well-known` endpoint should be + cached by the requesting server. Servers should respect the cache + control headers present on the response, or use a sensible default + when headers are not present. The recommended sensible default is 24 + hours. Servers should additionally impose a maximum cache time for + responses: 48 hours is recommended. Errors are recommended to be + cached for up to an hour, and servers are encouraged to + exponentially back off for repeated failures. The schema of the + `/.well-known` request is later in this section. If the response is + invalid (bad JSON, missing properties, non-200 response, etc), skip + to step 4. If the response is valid, the `m.server` property is + parsed as `[:]` and processed as + follows: + - If `` is an IP literal, then that IP address + should be used together with the `` or 8448 if + no port is provided. The target server must present a valid TLS + certificate for the IP address. Requests must be made with a + `Host` header containing the IP address, including the port if + one was provided. + - If `` is not an IP literal, and + `` is present, an IP address is discovered by + looking up an AAAA or A record for ``. The + resulting IP address is used, alongside the ``. + Requests must be made with a `Host` header of + `:`. The target server must + present a valid certificate for ``. + - If `` is not an IP literal and no + `` is present, an SRV record is looked up for + `_matrix._tcp.`. This may result in another + hostname (to be resolved using AAAA or A records) and port. + Requests should be made to the resolved IP address and port with + a `Host` header containing the ``. The + target server must present a valid certificate for + ``. + - If no SRV record is found, an IP address is resolved using AAAA + or A records. Requests are then made to the resolve IP address + and a port of 8448, using a `Host` header of + ``. The target server must present a valid + certificate for ``. +4. If the `/.well-known` request resulted in an error response, a + server is found by resolving an SRV record for + `_matrix._tcp.`. This may result in a hostname (to be + resolved using AAAA or A records) and port. Requests are made to the + resolved IP address and port, using 8448 as a default port, with a + `Host` header of ``. The target server must present a + valid certificate for ``. +5. If the `/.well-known` request returned an error response, and the + SRV record was not found, an IP address is resolved using AAAA and A + records. Requests are made to the resolved IP address using port + 8448 and a `Host` header containing the ``. The target + server must present a valid certificate for ``. + +The TLS certificate provided by the target server must be signed by a +known Certificate Authority. Servers are ultimately responsible for +determining the trusted Certificate Authorities, however are strongly +encouraged to rely on the operating system's judgement. Servers can +offer administrators a means to override the trusted authorities list. +Servers can additionally skip the certificate validation for a given +whitelist of domains or netmasks for the purposes of testing or in +networks where verification is done elsewhere, such as with `.onion` +addresses. Servers should respect SNI when making requests where +possible: a SNI should be sent for the certificate which is expected, +unless that certificate is expected to be an IP address in which case +SNI is not supported and should not be sent. + +Servers are encouraged to make use of the [Certificate +Transparency](https://www.certificate-transparency.org/) project. + +{{wellknown\_ss\_http\_api}} + +### Server implementation + +{{version\_ss\_http\_api}} + +### Retrieving server keys + +Note + +There was once a "version 1" of the key exchange. It has been removed +from the specification due to lack of significance. It may be reviewed +[from the historical +draft](https://github.com/matrix-org/matrix-doc/blob/51faf8ed2e4a63d4cfd6d23183698ed169956cc0/specification/server_server_api.rst#232version-1). + +Each homeserver publishes its public keys under +`/_matrix/key/v2/server/{keyId}`. Homeservers query for keys by either +getting `/_matrix/key/v2/server/{keyId}` directly or by querying an +intermediate notary server using a +`/_matrix/key/v2/query/{serverName}/{keyId}` API. Intermediate notary +servers query the `/_matrix/key/v2/server/{keyId}` API on behalf of +another server and sign the response with their own key. A server may +query multiple notary servers to ensure that they all report the same +public keys. + +This approach is borrowed from the [Perspectives +Project](https://web.archive.org/web/20170702024706/https://perspectives-project.org/), +but modified to include the NACL keys and to use JSON instead of XML. It +has the advantage of avoiding a single trust-root since each server is +free to pick which notary servers they trust and can corroborate the +keys returned by a given notary server by querying other servers. + +#### Publishing Keys + +Homeservers publish their signing keys in a JSON object at +`/_matrix/key/v2/server/{key_id}`. The response contains a list of +`verify_keys` that are valid for signing federation requests made by the +homeserver and for signing events. It contains a list of +`old_verify_keys` which are only valid for signing events. + +{{keys\_server\_ss\_http\_api}} + +#### Querying Keys Through Another Server + +Servers may query another server's keys through a notary server. The +notary server may be another homeserver. The notary server will retrieve +keys from the queried servers through use of the +`/_matrix/key/v2/server/{keyId}` API. The notary server will +additionally sign the response from the queried server before returning +the results. + +Notary servers can return keys for servers that are offline or having +issues serving their own keys by using cached responses. Keys can be +queried from multiple servers to mitigate against DNS spoofing. + +{{keys\_query\_ss\_http\_api}} + +## Authentication + +### Request Authentication + +Every HTTP request made by a homeserver is authenticated using public +key digital signatures. The request method, target and body are signed +by wrapping them in a JSON object and signing it using the JSON signing +algorithm. The resulting signatures are added as an Authorization header +with an auth scheme of `X-Matrix`. Note that the target field should +include the full path starting with `/_matrix/...`, including the `?` +and any query parameters if present, but should not include the leading +`https:`, nor the destination server's hostname. + +Step 1 sign JSON: + + { + "method": "GET", + "uri": "/target", + "origin": "origin.hs.example.com", + "destination": "destination.hs.example.com", + "content": , + "signatures": { + "origin.hs.example.com": { + "ed25519:key1": "ABCDEF..." + } + } + } + +The server names in the JSON above are the server names for each +homeserver involved. Delegation from the [server name resolution +section](#resolving-server-names) above do not affect these - the server +names from before delegation would take place are used. This same +condition applies throughout the request signing process. + +Step 2 add Authorization header: + + GET /target HTTP/1.1 + Authorization: X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..." + Content-Type: application/json + + + +Example python code: + + def authorization_headers(origin_name, origin_signing_key, + destination_name, request_method, request_target, + content=None): + request_json = { + "method": request_method, + "uri": request_target, + "origin": origin_name, + "destination": destination_name, + } + + if content is not None: + request_json["content"] = content + + signed_json = sign_json(request_json, origin_name, origin_signing_key) + + authorization_headers = [] + + for key, sig in signed_json["signatures"][origin_name].items(): + authorization_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + origin_name, key, sig, + ) + )) + + return ("Authorization", authorization_headers) + +### Response Authentication + +Responses are authenticated by the TLS server certificate. A homeserver +should not send a request until it has authenticated the connected +server to avoid leaking messages to eavesdroppers. + +### Client TLS Certificates + +Requests are authenticated at the HTTP layer rather than at the TLS +layer because HTTP services like Matrix are often deployed behind load +balancers that handle the TLS and these load balancers make it difficult +to check TLS client certificates. + +A homeserver may provide a TLS client certificate and the receiving +homeserver may check that the client certificate matches the certificate +of the origin homeserver. + +## Transactions + +The transfer of EDUs and PDUs between homeservers is performed by an +exchange of Transaction messages, which are encoded as JSON objects, +passed over an HTTP PUT request. A Transaction is meaningful only to the +pair of homeservers that exchanged it; they are not globally-meaningful. + +Transactions are limited in size; they can have at most 50 PDUs and 100 +EDUs. + +{{transactions\_ss\_http\_api}} + +## PDUs + +Each PDU contains a single Room Event which the origin server wants to +send to the destination. + +The `prev_events` field of a PDU identifies the "parents" of the event, +and thus establishes a partial ordering on events within the room by +linking them into a Directed Acyclic Graph (DAG). The sending server +should populate this field with all of the events in the room for which +it has not yet seen a child - thus demonstrating that the event comes +after all other known events. + +For example, consider a room whose events form the DAG shown below. A +server creating a new event in this room should populate the new event's +`prev_events` field with `E4` and `E5`, since neither event yet has a +child: + + E1 + ^ + | + +-> E2 <-+ + | | + E3 E5 + ^ + | + E4 + +For a full schema of what a PDU looks like, see the [room version +specification](../index.html#room-versions). + +### Checks performed on receipt of a PDU + +Whenever a server receives an event from a remote server, the receiving +server must ensure that the event: + +1. Is a valid event, otherwise it is dropped. +2. Passes signature checks, otherwise it is dropped. +3. Passes hash checks, otherwise it is redacted before being processed + further. +4. Passes authorization rules based on the event's auth events, + otherwise it is rejected. +5. Passes authorization rules based on the state at the event, + otherwise it is rejected. +6. Passes authorization rules based on the current state of the room, + otherwise it is "soft failed". + +Further details of these checks, and how to handle failures, are +described below. + +The [Signing Events](#signing-events) section has more information on +which hashes and signatures are expected on events, and how to calculate +them. + +#### Definitions + +Required Power Level +A given event type has an associated *required power level*. This is +given by the current `m.room.power_levels` event. The event type is +either listed explicitly in the `events` section or given by either +`state_default` or `events_default` depending on if the event is a state +event or not. + +Invite Level, Kick Level, Ban Level, Redact Level +The levels given by the `invite`, `kick`, `ban`, and `redact` properties +in the current `m.room.power_levels` state. Each defaults to 50 if +unspecified. + +Target User +For an `m.room.member` state event, the user given by the `state_key` of +the event. + +#### Authorization rules + +The rules governing whether an event is authorized depends on a set of +state. A given event is checked multiple times against different sets of +state, as specified above. Each room version can have a different +algorithm for how the rules work, and which rules are applied. For more +detailed information, please see the [room version +specification](../index.html#room-versions). + +##### Auth events selection + +The `auth_events` field of a PDU identifies the set of events which give +the sender permission to send the event. The `auth_events` for the +`m.room.create` event in a room is empty; for other events, it should be +the following subset of the room state: + +- The `m.room.create` event. + +- The current `m.room.power_levels` event, if any. + +- The sender's current `m.room.member` event, if any. + +- If type is `m.room.member`: + + > - The target's current `m.room.member` event, if any. + > - If `membership` is `join` or `invite`, the current + > `m.room.join_rules` event, if any. + > - If membership is `invite` and `content` contains a + > `third_party_invite` property, the current + > `m.room.third_party_invite` event with `state_key` matching + > `content.third_party_invite.signed.token`, if any. + +#### Rejection + +If an event is rejected it should neither be relayed to clients nor be +included as a prev event in any new events generated by the server. +Subsequent events from other servers that reference rejected events +should be allowed if they still pass the auth rules. The state used in +the checks should be calculated as normal, except not updating with the +rejected event where it is a state event. + +If an event in an incoming transaction is rejected, this should not +cause the transaction request to be responded to with an error response. + +Note + +This means that events may be included in the room DAG even though they +should be rejected. + +Note + +This is in contrast to redacted events which can still affect the state +of the room. For example, a redacted `join` event will still result in +the user being considered joined. + +#### Soft failure + +Rationale + +It is important that we prevent users from evading bans (or other power +restrictions) by creating events which reference old parts of the DAG. +For example, a banned user could continue to send messages to a room by +having their server send events which reference the event before they +were banned. Note that such events are entirely valid, and we cannot +simply reject them, as it is impossible to distinguish such an event +from a legitimate one which has been delayed. We must therefore accept +such events and let them participate in state resolution and the +federation protocol as normal. However, servers may choose not to send +such events on to their clients, so that end users won't actually see +the events. + +When this happens it is often fairly obvious to servers, as they can see +that the new event doesn't actually pass auth based on the "current +state" (i.e. the resolved state across all forward extremities). While +the event is technically valid, the server can choose to not notify +clients about the new event. + +This discourages servers from sending events that evade bans etc. in +this way, as end users won't actually see the events. + +When the homeserver receives a new event over federation it should also +check whether the event passes auth checks based on the current state of +the room (as well as based on the state at the event). If the event does +not pass the auth checks based on the *current state* of the room (but +does pass the auth checks based on the state at that event) it should be +"soft failed". + +When an event is "soft failed" it should not be relayed to the client +nor be referenced by new events created by the homeserver (i.e. they +should not be added to the server's list of forward extremities of the +room). Soft failed events are otherwise handled as usual. + +Note + +Soft failed events participate in state resolution as normal if further +events are received which reference it. It is the job of the state +resolution algorithm to ensure that malicious events cannot be injected +into the room state via this mechanism. + +Note + +Because soft failed state events participate in state resolution as +normal, it is possible for such events to appear in the current state of +the room. In that case the client should be told about the soft failed +event in the usual way (e.g. by sending it down in the `state` section +of a sync response). + +Note + +A soft failed event should be returned in response to federation +requests where appropriate (e.g. in `/event/`). Note that soft +failed events are returned in `/backfill` and `/get_missing_events` +responses only if the requests include events referencing the soft +failed events. + +Example + +As an example consider the event graph: + + A + / + B + +where `B` is a ban of a user `X`. If the user `X` tries to set the topic +by sending an event `C` while evading the ban: + + A + / \ + B C + +servers that receive `C` after `B` should soft fail event `C`, and so +will neither relay `C` to its clients nor send any events referencing +`C`. + +If later another server sends an event `D` that references both `B` and +`C` (this can happen if it received `C` before `B`): + + A + / \ + B C + \ / + D + +then servers will handle `D` as normal. `D` is sent to the servers' +clients (assuming `D` passes auth checks). The state at `D` may resolve +to a state that includes `C`, in which case clients should also to be +told that the state has changed to include `C`. (*Note*: This depends on +the exact state resolution algorithm used. In the original version of +the algorithm `C` would be in the resolved state, whereas in latter +versions the algorithm tries to prioritise the ban over the topic +change.) + +Note that this is essentially equivalent to the situation where one +server doesn't receive `C` at all, and so asks another server for the +state of the `C` branch. + +Let's go back to the graph before `D` was sent: + + A + / \ + B C + +If all the servers in the room saw `B` before `C` and so soft fail `C`, +then any new event `D'` will not reference `C`: + + A + / \ + B C + | + D + +#### Retrieving event authorization information + +The homeserver may be missing event authorization information, or wish +to check with other servers to ensure it is receiving the correct auth +chain. These APIs give the homeserver an avenue for getting the +information it needs. + +{{event\_auth\_ss\_http\_api}} + +## EDUs + +EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of +"previous" IDs. They are intended to be non-persistent data such as user +presence, typing notifications, etc. + +{{definition\_ss\_edu}} + +## Room State Resolution + +The *state* of a room is a map of `(event_type, state_key)` to +`event_id`. Each room starts with an empty state, and each state event +which is accepted into the room updates the state of that room. + +Where each event has a single `prev_event`, it is clear what the state +of the room after each event should be. However, when two branches in +the event graph merge, the state of those branches might differ, so a +*state resolution* algorithm must be used to determine the resultant +state. + +For example, consider the following event graph (where the oldest event, +E0, is at the top): + + E0 + | + E1 + / \ + E2 E4 + | | + E3 | + \ / + E5 + +Suppose E3 and E4 are both `m.room.name` events which set the name of +the room. What should the name of the room be at E5? + +The algorithm to be used for state resolution depends on the room +version. For a description of each room version's algorithm, please see +the [room version specification](../index.html#room-versions). + +## Backfilling and retrieving missing events + +Once a homeserver has joined a room, it receives all the events emitted +by other homeservers in that room, and is thus aware of the entire +history of the room from that moment onwards. Since users in that room +are able to request the history by the `/messages` client API endpoint, +it's possible that they might step backwards far enough into history +before the homeserver itself was a member of that room. + +To cover this case, the federation API provides a server-to-server +analog of the `/messages` client API, allowing one homeserver to fetch +history from another. This is the `/backfill` API. + +To request more history, the requesting homeserver picks another +homeserver that it thinks may have more (most likely this should be a +homeserver for some of the existing users in the room at the earliest +point in history it has currently), and makes a `/backfill` request. + +Similar to backfilling a room's history, a server may not have all the +events in the graph. That server may use the `/get_missing_events` API +to acquire the events it is missing. + +{{backfill\_ss\_http\_api}} + +## Retrieving events + +In some circumstances, a homeserver may be missing a particular event or +information about the room which cannot be easily determined from +backfilling. These APIs provide homeservers with the option of getting +events and the state of the room at a given point in the timeline. + +{{events\_ss\_http\_api}} + +## Joining Rooms + +When a new user wishes to join a room that the user's homeserver already +knows about, the homeserver can immediately determine if this is +allowable by inspecting the state of the room. If it is acceptable, it +can generate, sign, and emit a new `m.room.member` state event adding +the user into that room. When the homeserver does not yet know about the +room it cannot do this directly. Instead, it must take a longer +multi-stage handshaking process by which it first selects a remote +homeserver which is already participating in that room, and use it to +assist in the joining process. This is the remote join handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting +the room alias the user is requesting to join with, and a homeserver +where existing room members are already present (referred to as the +"resident" server). + +In summary, the remote join handshake consists of the joining server +querying the directory server for information about the room alias; +receiving a room ID and a list of join candidates. The joining server +then requests information about the room from one of the residents. It +uses this information to construct an `m.room.member` event which it +finally sends to a resident server. + +Conceptually these are three different roles of homeserver. In practice +the directory server is likely to be resident in the room, and so may be +selected by the joining server to be the assisting resident. Likewise, +it is likely that the joining server picks the same candidate resident +for both phases of event construction, though in principle any valid +candidate may be used at each time. Thus, any join handshake can +potentially involve anywhere from two to four homeservers, though most +in practice will use just two. + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response + +The first part of the handshake usually involves using the directory +server to request the room ID and join candidates through the +`/query/directory`\_ API endpoint. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver +could optimise this step away by picking the origin server of that +invite message as the join candidate. However, the joining server should +be aware that the origin server of the invite might since have left the +room, so should be prepared to fall back on the regular join flow if +this optimisation fails. + +Once the joining server has the room ID and the join candidates, it then +needs to obtain enough information about the room to fill in the +required fields of the `m.room.member` event. It obtains this by +selecting a resident from the candidate list, and using the +`GET /make_join` endpoint. The resident server will then reply with +enough information for the joining server to fill in the event. + +The joining server is expected to add or replace the `origin`, +`origin_server_ts`, and `event_id` on the templated event received by +the resident server. This event is then signed by the joining server. + +To complete the join handshake, the joining server must now submit this +new event to a resident homeserver, by using the `PUT /send_join` +endpoint. + +The resident homeserver then accepts this event into the room's event +graph, and responds to the joining server with the full set of state for +the newly-joined room. The resident server must also send the event to +other servers participating in the room. + +{{joins\_v1\_ss\_http\_api}} + +{{joins\_v2\_ss\_http\_api}} + +## Inviting to a room + +When a user on a given homeserver invites another user on the same +homeserver, the homeserver may sign the membership event itself and skip +the process defined here. However, when a user invites another user on a +different homeserver, a request to that homeserver to have the event +signed and verified must be made. + +{{invites\_v1\_ss\_http\_api}} + +{{invites\_v2\_ss\_http\_api}} + +## Leaving Rooms (Rejecting Invites) + +Normally homeservers can send appropriate `m.room.member` events to have +users leave the room, or to reject local invites. Remote invites from +other homeservers do not involve the server in the graph and therefore +need another approach to reject the invite. Joining the room and +promptly leaving is not recommended as clients and servers will +interpret that as accepting the invite, then leaving the room rather +than rejecting the invite. + +Similar to the [Joining Rooms](#joining-rooms) handshake, the server +which wishes to leave the room starts with sending a `/make_leave` +request to a resident server. In the case of rejecting invites, the +resident server may be the server which sent the invite. After receiving +a template event from `/make_leave`, the leaving server signs the event +and replaces the `event_id` with its own. This is then sent to the +resident server via `/send_leave`. The resident server will then send +the event to other servers in the room. + +{{leaving\_v1\_ss\_http\_api}} + +{{leaving\_v2\_ss\_http\_api}} + +## Third-party invites + +Note + +More information about third party invites is available in the +[Client-Server API](../client_server/%CLIENT_RELEASE_LABEL%.html) under +the Third Party Invites module. + +When a user wants to invite another user in a room but doesn't know the +Matrix ID to invite, they can do so using a third-party identifier (e.g. +an e-mail or a phone number). + +This identifier and its bindings to Matrix IDs are verified by an +identity server implementing the [Identity Service +API](../identity_service/%IDENTITY_RELEASE_LABEL%.html). + +### Cases where an association exists for a third-party identifier + +If the third-party identifier is already bound to a Matrix ID, a lookup +request on the identity server will return it. The invite is then +processed by the inviting homeserver as a standard `m.room.member` +invite event. This is the simplest case. + +### Cases where an association doesn't exist for a third-party identifier + +If the third-party identifier isn't bound to any Matrix ID, the inviting +homeserver will request the identity server to store an invite for this +identifier and to deliver it to whoever binds it to its Matrix ID. It +will also send an `m.room.third_party_invite` event in the room to +specify a display name, a token and public keys the identity server +provided as a response to the invite storage request. + +When a third-party identifier with pending invites gets bound to a +Matrix ID, the identity server will send a POST request to the ID's +homeserver as described in the [Invitation +Storage](../identity_service/%IDENTITY_RELEASE_LABEL%.html#invitation-storage) +section of the Identity Service API. + +The following process applies for each invite sent by the identity +server: + +The invited homeserver will create an `m.room.member` invite event +containing a special `third_party_invite` section containing the token +and a signed object, both provided by the identity server. + +If the invited homeserver is in the room the invite came from, it can +auth the event and send it. + +However, if the invited homeserver isn't in the room the invite came +from, it will need to request the room's homeserver to auth the event. + +{{third\_party\_invite\_ss\_http\_api}} + +#### Verifying the invite + +When a homeserver receives an `m.room.member` invite event for a room +it's in with a `third_party_invite` object, it must verify that the +association between the third-party identifier initially invited to the +room and the Matrix ID that claims to be bound to it has been verified +without having to rely on a third-party server. + +To do so, it will fetch from the room's state events the +`m.room.third_party_invite` event for which the state key matches with +the value for the `token` key in the `third_party_invite` object from +the `m.room.member` event's content to fetch the public keys initially +delivered by the identity server that stored the invite. + +It will then use these keys to verify that the `signed` object (in the +`third_party_invite` object from the `m.room.member` event's content) +was signed by the same identity server. + +Since this `signed` object can only be delivered once in the POST +request emitted by the identity server upon binding between the +third-party identifier and the Matrix ID, and contains the invited +user's Matrix ID and the token delivered when the invite was stored, +this verification will prove that the `m.room.member` invite event comes +from the user owning the invited third-party identifier. + +## Public Room Directory + +To complement the [Client-Server +API](../client_server/%CLIENT_RELEASE_LABEL%.html)'s room directory, +homeservers need a way to query the public rooms for another server. +This can be done by making a request to the `/publicRooms` endpoint for +the server the room directory should be retrieved for. + +{{public\_rooms\_ss\_http\_api}} + +## Typing Notifications + +When a server's users send typing notifications, those notifications +need to be sent to other servers in the room so their users are aware of +the same state. Receiving servers should verify that the user is in the +room, and is a user belonging to the sending server. + +{{definition\_ss\_event\_schemas\_m\_typing}} + +## Presence + +The server API for presence is based entirely on exchange of the +following EDUs. There are no PDUs or Federation Queries involved. + +Servers should only send presence updates for users that the receiving +server would be interested in. Such as the receiving server sharing a +room with a given user. + +{{definition\_ss\_event\_schemas\_m\_presence}} + +## Receipts + +Receipts are EDUs used to communicate a marker for a given event. +Currently the only kind of receipt supported is a "read receipt", or +where in the event graph the user has read up to. + +Read receipts for events that a user sent do not need to be sent. It is +implied that by sending the event the user has read up to the event. + +{{definition\_ss\_event\_schemas\_m\_receipt}} + +## Querying for information + +Queries are a way to retrieve information from a homeserver about a +resource, such as a user or room. The endpoints here are often called in +conjunction with a request from a client on the client-server API in +order to complete the call. + +There are several types of queries that can be made. The generic +endpoint to represent all queries is described first, followed by the +more specific queries that can be made. + +{{query\_ss\_http\_api}} + +## OpenID + +Third party services can exchange an access token previously generated +by the Client-Server API for information +about a user. This can help verify that a user is who they say they are +without granting full access to the user's account. + +Access tokens generated by the OpenID API are only good for the OpenID +API and nothing else. + +{{openid\_ss\_http\_api}} + +## Device Management + +Details of a user's devices must be efficiently published to other users +and kept up-to-date. This is critical for reliable end-to-end +encryption, in order for users to know which devices are participating +in a room. It's also required for to-device messaging to work. This +section is intended to complement the [Device Management +module](../client_server/%CLIENT_RELEASE_LABEL%.html#device-management) +of the Client-Server API. + +Matrix currently uses a custom pubsub system for synchronising +information about the list of devices for a given user over federation. +When a server wishes to determine a remote user's device list for the +first time, it should populate a local cache from the result of a +`/user/keys/query` API on the remote server. However, subsequent updates +to the cache should be applied by consuming `m.device_list_update` EDUs. +Each new `m.device_list_update` EDU describes an incremental change to +one device for a given user which should replace any existing entry in +the local server's cache of that device list. Servers must send +`m.device_list_update` EDUs to all the servers who share a room with a +given local user, and must be sent whenever that user's device list +changes (i.e. for new or deleted devices, when that user joins a room +which contains servers which are not already receiving updates for that +user's device list, or changes in device information such as the +device's human-readable name). + +Servers send `m.device_list_update` EDUs in a sequence per origin user, +each with a unique `stream_id`. They also include a pointer to the most +recent previous EDU(s) that this update is relative to in the `prev_id` +field. To simplify implementation for clustered servers which could send +multiple EDUs at the same time, the `prev_id` field should include all +`m.device_list_update` EDUs which have not been yet been referenced in a +EDU. If EDUs are emitted in series by a server, there should only ever +be one `prev_id` in the EDU. + +This forms a simple directed acyclic graph of `m.device_list_update` +EDUs, showing which EDUs a server needs to have received in order to +apply an update to its local copy of the remote user's device list. If a +server receives an EDU which refers to a `prev_id` it does not +recognise, it must resynchronise its list by calling the +`/user/keys/query API` and resume the process. The response contains a +`stream_id` which should be used to correlate with subsequent +`m.device_list_update` EDUs. + +{{user\_devices\_ss\_http\_api}} + +{{definition\_ss\_event\_schemas\_m\_device\_list\_update}} + +## End-to-End Encryption + +This section complements the [End-to-End Encryption +module](../client_server/%CLIENT_RELEASE_LABEL%.html#end-to-end-encryption) +of the Client-Server API. For detailed information about end-to-end +encryption, please see that module. + +The APIs defined here are designed to be able to proxy much of the +client's request through to federation, and have the response also be +proxied through to the client. + +{{user\_keys\_ss\_http\_api}} + +{{definition\_ss\_event\_schemas\_m\_signing\_key\_update}} + +## Send-to-device messaging + +The server API for send-to-device messaging is based on the +`m.direct_to_device` EDU. There are no PDUs or Federation Queries +involved. + +Each send-to-device message should be sent to the destination server +using the following EDU: + +{{definition\_ss\_event\_schemas\_m\_direct\_to\_device}} + +## Content Repository + +Attachments to events (images, files, etc) are uploaded to a homeserver +via the Content Repository described in the [Client-Server +API](../client_server/%CLIENT_RELEASE_LABEL%.html). When a server wishes +to serve content originating from a remote server, it needs to ask the +remote server for the media. + +Servers should use the server described in the Matrix Content URI, which +has the format `mxc://{ServerName}/{MediaID}`. Servers should use the +download endpoint described in the [Client-Server +API](../client_server/%CLIENT_RELEASE_LABEL%.html), being sure to use +the `allow_remote` parameter (set to `false`). + +## Server Access Control Lists (ACLs) + +Server ACLs and their purpose are described in the [Server +ACLs](../client_server/%CLIENT_RELEASE_LABEL%.html#module-server-acls) +section of the Client-Server API. + +When a remote server makes a request, it MUST be verified to be allowed +by the server ACLs. If the server is denied access to a room, the +receiving server MUST reply with a 403 HTTP status code and an `errcode` +of `M_FORBIDDEN`. + +The following endpoint prefixes MUST be protected: + +- `/_matrix/federation/v1/send` (on a per-PDU basis) +- `/_matrix/federation/v1/make_join` +- `/_matrix/federation/v1/make_leave` +- `/_matrix/federation/v1/send_join` +- `/_matrix/federation/v2/send_join` +- `/_matrix/federation/v1/send_leave` +- `/_matrix/federation/v2/send_leave` +- `/_matrix/federation/v1/invite` +- `/_matrix/federation/v2/invite` +- `/_matrix/federation/v1/state` +- `/_matrix/federation/v1/state_ids` +- `/_matrix/federation/v1/backfill` +- `/_matrix/federation/v1/event_auth` +- `/_matrix/federation/v1/get_missing_events` + +## Signing Events + +Signing events is complicated by the fact that servers can choose to +redact non-essential parts of an event. + +### Adding hashes and signatures to outgoing events + +Before signing the event, the *content hash* of the event is calculated +as described below. The hash is encoded using [Unpadded +Base64](../appendices.html#unpadded-base64) and stored in the event +object, in a `hashes` object, under a `sha256` key. + +The event object is then *redacted*, following the [redaction +algorithm](../client_server/%CLIENT_RELEASE_LABEL%.html#redactions). +Finally it is signed as described in [Signing +JSON](../appendices.html#signing-json), using the server's signing key +(see also [Retrieving server keys](#retrieving-server-keys)). + +The signature is then copied back to the original event object. + +See [Persistent Data Unit schema](#Persistent Data Unit schema) for an +example of a signed event. + +### Validating hashes and signatures on received events + +When a server receives an event over federation from another server, the +receiving server should check the hashes and signatures on that event. + +First the signature is checked. The event is redacted following the +[redaction +algorithm](../client_server/%CLIENT_RELEASE_LABEL%.html#redactions), and +the resultant object is checked for a signature from the originating +server, following the algorithm described in [Checking for a +signature](../appendices.html#checking-for-a-signature). Note that this +step should succeed whether we have been sent the full event or a +redacted copy. + +The signatures expected on an event are: + +- The `sender`'s server, unless the invite was created as a result of + 3rd party invite. The sender must already match the 3rd party + invite, and the server which actually sends the event may be a + different server. +- For room versions 1 and 2, the server which created the `event_id`. + Other room versions do not track the `event_id` over federation and + therefore do not need a signature from those servers. + +If the signature is found to be valid, the expected content hash is +calculated as described below. The content hash in the `hashes` property +of the received event is base64-decoded, and the two are compared for +equality. + +If the hash check fails, then it is assumed that this is because we have +only been given a redacted version of the event. To enforce this, the +receiving server should use the redacted copy it calculated rather than +the full copy it received. + +### Calculating the reference hash for an event + +The *reference hash* of an event covers the essential fields of an +event, including content hashes. It is used for event identifiers in +some room versions. See the [room version +specification](../index.html#room-versions) for more information. It is +calculated as follows. + +1. The event is put through the redaction algorithm. +2. The `signatures`, `age_ts`, and `unsigned` properties are removed + from the event, if present. +3. The event is converted into [Canonical + JSON](../appendices.html#canonical-json). +4. A sha256 hash is calculated on the resulting JSON object. + +### Calculating the content hash for an event + +The *content hash* of an event covers the complete event including the +*unredacted* contents. It is calculated as follows. + +First, any existing `unsigned`, `signature`, and `hashes` members are +removed. The resulting object is then encoded as [Canonical +JSON](../appendices.html#canonical-json), and the JSON is hashed using +SHA-256. + +### Example code + + def hash_and_sign_event(event_object, signing_key, signing_name): + # First we need to hash the event object. + content_hash = compute_content_hash(event_object) + event_object["hashes"] = {"sha256": encode_unpadded_base64(content_hash)} + + # Strip all the keys that would be removed if the event was redacted. + # The hashes are not stripped and cover all the keys in the event. + # This means that we can tell if any of the non-essential keys are + # modified or removed. + stripped_object = strip_non_essential_keys(event_object) + + # Sign the stripped JSON object. The signature only covers the + # essential keys and the hashes. This means that we can check the + # signature even if the event is redacted. + signed_object = sign_json(stripped_object, signing_key, signing_name) + + # Copy the signatures from the stripped event to the original event. + event_object["signatures"] = signed_object["signatures"] + + def compute_content_hash(event_object): + # take a copy of the event before we remove any keys. + event_object = dict(event_object) + + # Keys under "unsigned" can be modified by other servers. + # They are useful for conveying information like the age of an + # event that will change in transit. + # Since they can be modified we need to exclude them from the hash. + event_object.pop("unsigned", None) + + # Signatures will depend on the current value of the "hashes" key. + # We cannot add new hashes without invalidating existing signatures. + event_object.pop("signatures", None) + + # The "hashes" key might contain multiple algorithms if we decide to + # migrate away from SHA-2. We don't want to include an existing hash + # output in our hash so we exclude the "hashes" dict from the hash. + event_object.pop("hashes", None) + + # Encode the JSON using a canonical encoding so that we get the same + # bytes on every server for the same JSON object. + event_json_bytes = encode_canonical_json(event_object) + + return hashlib.sha256(event_json_bytes) + +## Security considerations + +When a domain's ownership changes, the new controller of the domain can +masquerade as the previous owner, receiving messages (similarly to +email) and request past messages from other servers. In the future, +proposals like +[MSC1228](https://github.com/matrix-org/matrix-doc/issues/1228) will +address this issue.