From 36e035c79e5efccb76a429194efdc19fcc450d0b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Jul 2015 19:31:11 +0100 Subject: [PATCH 001/177] Add some specification for end-to-end --- specification/41_end_to_end_encryption.rst | 215 +++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 02388152..a09c8fc6 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -13,3 +13,218 @@ participating homeservers. End-to-end crypto is still being designed and prototyped - notes on the design may be found at https://lwn.net/Articles/634144/ + +Overview +======== + +.. code:: + + 1) Bob publishes the public keys and supported algorithms for his device. + + +----------+ +--------------+ + | Bob's HS | | Bob's Device | + +----------+ +--------------+ + | | + |<=============| + /keys/upload + + 2) Alice requests Bob's public key and supported algorithms. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/query + + 3) Alice selects an algorithm takes any one time keys needed. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/take + + 4) Alice sends an encrypted message to Bob. + + +----------------+ +------------+ +----------+ +--------------+ + | Alice's Device | | Alice's HS | | Bob's HS | | Bob's Device | + +----------------+ +------------+ +----------+ +--------------+ + | | | | + |----------------->|-------------->|------------->| + /send/ + + +Client Behaviour +---------------- + +Uploading Keys +~~~~~~~~~~~~~~ + +Keys are uploaded as a signed JSON object. The JSON object must include an +ed25519 key and must be signed by that key. A device may only have one ed25519 +signing key. This key is used as the fingerprint for a device by other clients. + + +.. code:: http + + POST /_matrix/client/v2_alpha/keys/upload/ HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "" { + ":": "" + } } }, + "one_time_keys": { + ":": "" + }, + } + + +Downloading Keys +~~~~~~~~~~~~~~~~ + +Keys are downloaded a collection of signed JSON objects. There +will be JSON object per device per user. If one of the user's +devices doesn't support end-to-end encryption then their +homeserver will synthesise a JSON object without any device keys +for that device. + +The JSON must be signed by both the homeserver of +the user querying the keys and by the homeserver of the device +being queried. This provides an audit trail if either homeserver +lies about the keys a user owns. + +.. code:: http + + POST /keys/query HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "": [""] + } } + + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_keys": { + "": { + "": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "": { + ":": "" + }, + "": { + ":": "" + }, + "": { + ":": "" + } } } } } } + + +Taking One Time Keys +~~~~~~~~~~~~~~~~~~~~ + +Some algorithms require one time keys to improve their secrecy and deniability. +Theses keys are used once during session establishment, and are then thrown +away. In order for these keys to be useful for improving deniability they +must not be signed using the ed25519 key for a device. + +A device will generate a number of these keys and publish them onto their +homeserver. A device will periodically check how many one time keys their +homeserver still has. If the number has become too small then the device will +generate new one time keys and upload them to the homeserver. + +Devices will store the private part of each one time key they upload. They can +discard the private part of the one time key when they receive a message using +that key. However one-keys given out by a homeserver may never end up being +used. Therefore a device may end up trying to store too many private keys. A +device that is trying to store too many private keys may discard keys starting +with the oldest. + +A homeserver should ratelimit the number of one time keys that a given user or +remote server can take. A homeserver should discard the public part of a one +time key once it has given that key to another user. + + +.. code:: http + + POST /keys/take HTTP/1.1 + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": "" + } } } + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": { + ":": "" + } } } } + + +Sending a Message +~~~~~~~~~~~~~~~~~ + +Encrypted messages are sent in the form. + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "" + } + } + + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "ciphertexts": { + "" { + ": { + "type": 0, + "body": "" + } } } } } From 01927cee9b6d8152579eaa24d4ae6572933e9811 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:21:25 +0100 Subject: [PATCH 002/177] Rename "take" to "claim". Hyphenate "one-time". --- specification/41_end_to_end_encryption.rst | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a09c8fc6..f30fc09d 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -37,14 +37,14 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm takes any one time keys needed. + 3) Alice selects an algorithm claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | +----------------+ +------------+ +----------+ | | | |=================>|==============>| - /keys/take + /keys/claim 4) Alice sends an encrypted message to Bob. @@ -97,7 +97,7 @@ signing key. This key is used as the fingerprint for a device by other clients. Downloading Keys ~~~~~~~~~~~~~~~~ -Keys are downloaded a collection of signed JSON objects. There +Keys are downloaded as a collection of signed JSON objects. There will be JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver will synthesise a JSON object without any device keys @@ -150,34 +150,35 @@ lies about the keys a user owns. } } } } } } -Taking One Time Keys +Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~ -Some algorithms require one time keys to improve their secrecy and deniability. -Theses keys are used once during session establishment, and are then thrown +Some algorithms require one-time keys to improve their secrecy and deniability. +These keys are used once during session establishment, and are then thrown away. In order for these keys to be useful for improving deniability they must not be signed using the ed25519 key for a device. -A device will generate a number of these keys and publish them onto their -homeserver. A device will periodically check how many one time keys their -homeserver still has. If the number has become too small then the device will -generate new one time keys and upload them to the homeserver. - -Devices will store the private part of each one time key they upload. They can -discard the private part of the one time key when they receive a message using -that key. However one-keys given out by a homeserver may never end up being -used. Therefore a device may end up trying to store too many private keys. A -device that is trying to store too many private keys may discard keys starting -with the oldest. - -A homeserver should ratelimit the number of one time keys that a given user or -remote server can take. A homeserver should discard the public part of a one +A device must generate a number of these keys and publish them onto their +homeserver. A device must periodically check how many one-time keys their +homeserver still has. If the number has become too small then the device must +generate new one-time keys and upload them to the homeserver. + +Devices must store the private part of each one-time key they upload. They can +discard the private part of the one-time key when they receive a message using +that key. However it's possible that a one-time key given out by a homeserver +will never be used, so the device that generates the key will never know that +it can discard the key. Therefore a device could end up trying to store too +many private keys. A device that is trying to store too many private keys may +discard keys starting with the oldest. + +A homeserver should ratelimit the number of one-time keys that a given user or +remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. .. code:: http - POST /keys/take HTTP/1.1 + POST /keys/claim HTTP/1.1 Content-Type: application/json { @@ -211,8 +212,7 @@ Encrypted messages are sent in the form. "content": {} "encrypted": { "algorithm": "" - } - } + } } .. code:: json From 42ad1f861240998658fb643c568db70cc2d86c2e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:38:08 +0100 Subject: [PATCH 003/177] Add a link to signing JSON section of the spec. Fixup the markup a bit --- specification/41_end_to_end_encryption.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f30fc09d..19ee0cf1 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -66,6 +66,8 @@ Keys are uploaded as a signed JSON object. The JSON object must include an ed25519 key and must be signed by that key. A device may only have one ed25519 signing key. This key is used as the fingerprint for a device by other clients. +The JSON object is signed using the process given by `Signing JSON`_. + .. code:: http @@ -90,17 +92,16 @@ signing key. This key is used as the fingerprint for a device by other clients. } } }, "one_time_keys": { ":": "" - }, - } + } } Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be JSON object per device per user. If one of the user's +will be a JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their -homeserver will synthesise a JSON object without any device keys +homeserver must synthesise a JSON object without any device keys for that device. The JSON must be signed by both the homeserver of @@ -151,7 +152,7 @@ lies about the keys a user owns. Claiming One Time Keys -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Some algorithms require one-time keys to improve their secrecy and deniability. These keys are used once during session establishment, and are then thrown @@ -228,3 +229,4 @@ Encrypted messages are sent in the form. "type": 0, "body": "" } } } } } + From 6f69707c71c50f783d1327c55179f2007129cbdb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Jul 2015 19:30:36 +0100 Subject: [PATCH 004/177] Update e2e spec: Group ciphertext by device key rather than device id, add return to docs for /keys/upload, Use "m.room.encrypted" for now, rather than trying to add an encrypted content to arbitrary event types --- specification/41_end_to_end_encryption.rst | 36 ++++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 19ee0cf1..bf762a3d 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -94,6 +94,17 @@ The JSON object is signed using the process given by `Signing JSON`_. ":": "" } } +.. code:: http + + 200 OK + Content-Type: application/json + + { + "one_time_key_counts": { + "": 50 + } + } + Downloading Keys ~~~~~~~~~~~~~~~~ @@ -209,10 +220,10 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.message" - "content": {} - "encrypted": { + "type": "m.room.encrypted" + "content": { "algorithm": "" + } } } @@ -220,13 +231,18 @@ Encrypted messages are sent in the form. { "type": "m.room.message" - "content": {} - "encrypted": { + "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": , "ciphertexts": { - "" { - ": { - "type": 0, - "body": "" - } } } } } + ": { + "type": 0, + "body": "" + } } } } + + +The plaintext payload is of the form: + +.. code:: json + TODO: SPEC the JSON plaintext format From 41d204e72c22b768ae4ac0b5dcda59cdcaa09acd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 09:43:46 +0100 Subject: [PATCH 005/177] Name the key 'ciphertext' rather than 'ciphertexts' --- specification/41_end_to_end_encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index bf762a3d..553664f2 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -223,8 +223,7 @@ Encrypted messages are sent in the form. "type": "m.room.encrypted" "content": { "algorithm": "" - } - } } + } } } .. code:: json @@ -234,7 +233,7 @@ Encrypted messages are sent in the form. "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": , - "ciphertexts": { + "ciphertext": { ": { "type": 0, "body": "" @@ -245,4 +244,5 @@ The plaintext payload is of the form: .. code:: json - TODO: SPEC the JSON plaintext format + { + } From 6597aaa448023b31fa62e23183e4a4eeeae0e22d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 11:21:14 +0100 Subject: [PATCH 006/177] Start describing the plaintext payload format for encrypted messages, add the exact URLs used for key queries from clients and for key queries for federation --- specification/41_end_to_end_encryption.rst | 57 ++++++++++++++++------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 553664f2..4c64e92f 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -96,7 +96,7 @@ The JSON object is signed using the process given by `Signing JSON`_. .. code:: http - 200 OK + HTTP/1.1 200 OK Content-Type: application/json { @@ -162,6 +162,12 @@ lies about the keys a user owns. } } } } } } +Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/query``. + + Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~~~ @@ -212,6 +218,11 @@ time key once it has given that key to another user. } } } } +Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/claim``. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -220,24 +231,27 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" - "content": { - "algorithm": "" - } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "" + } } +Using Olm +######### + .. code:: json { - "type": "m.room.message" - "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": , - "ciphertext": { - ": { - "type": 0, - "body": "" - } } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "", + "ciphertext": { + "": { + "type": 0, + "body": "" + } } } } The plaintext payload is of the form: @@ -245,4 +259,19 @@ The plaintext payload is of the form: .. code:: json { + "type": "", + "content": "", + "room_id": "", + "fingerprint": "" } + +The type and content of the plaintext message event are given in the payload. +Encyrpting state events is not supported. + +We include the room ID in the payload, because otherwise the homeserver would +be able to change the room a message was sent in. We include a hash of the +participating keys so that clients can detect if another device is unexpectedly +included in the conversation. + +Clients must confirm that the ``sender_key`` actually belongs to the device +that sent the message. From c83e8480e83db63e76ffcbe62734de773794fb8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 15:55:21 +0100 Subject: [PATCH 007/177] Fix JSON syntax --- specification/41_end_to_end_encryption.rst | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 4c64e92f..5f486f66 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -86,8 +86,8 @@ The JSON object is signed using the process given by `Signing JSON`_. "keys": { ":": "", }, - "signatures:" { - "" { + "signatures": { + "": { ":": "" } } }, "one_time_keys": { @@ -150,7 +150,7 @@ lies about the keys a user owns. "keys": { ":": "", }, - "signatures:" { + "signatures": { "": { ":": "" }, @@ -163,9 +163,10 @@ lies about the keys a user owns. Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to -claim keys for any user they wish to contact. Homeservers will respond with the +query keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/query``. +``/_matrix/federation/v1/user/keys/query`` over federation to the remote +server. Claiming One Time Keys @@ -221,7 +222,9 @@ time key once it has given that key to another user. Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to claim keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/claim``. +``/_matrix/federation/v1/user/keys/claim`` over federation to the remote +server. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -231,7 +234,7 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "" } } @@ -243,7 +246,7 @@ Using Olm .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "", @@ -273,5 +276,5 @@ be able to change the room a message was sent in. We include a hash of the participating keys so that clients can detect if another device is unexpectedly included in the conversation. -Clients must confirm that the ``sender_key`` actually belongs to the device -that sent the message. +Clients must confirm that the ``sender_key`` belongs to the user that sent the +message. From e1f201f9e636fad8cd3ec2b031a58a0fd7018729 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:07:03 +0100 Subject: [PATCH 008/177] Add description of the olm type and body JSON keys --- specification/41_end_to_end_encryption.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 5f486f66..1e60fad9 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -256,6 +256,21 @@ Using Olm "body": "" } } } } +The ciphertext is a mapping from device curve25519 key to an encypted payload +for that device. The ``body`` is a base64 encoded message body. The type is an +integer indicating the type of the message body: 0 for the intial pre-key +message, 1 for ordinary messages. + +Olm sessions will generate messages with a type of 0 until they receive a +message. Once a session has decrypted a message it will produce messages with +a type of 1. + +When a client receives a message with a type of 0 it must first check if it +already has a matching session. If it does then it will use that session to +try to decrypt the message. If there is no existing session then the client +must create a new session and use the new session to decrypt the message. A +client must not persist a session or remove one-time keys used by a session +until it has sucessfully decrypted a message using that session. The plaintext payload is of the form: From d06580a481f5396c53574ff86e4830b0cb47f7b4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:09:14 +0100 Subject: [PATCH 009/177] Spelling --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 1e60fad9..f0d8a500 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -258,7 +258,7 @@ Using Olm The ciphertext is a mapping from device curve25519 key to an encypted payload for that device. The ``body`` is a base64 encoded message body. The type is an -integer indicating the type of the message body: 0 for the intial pre-key +integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. Olm sessions will generate messages with a type of 0 until they receive a @@ -270,7 +270,7 @@ already has a matching session. If it does then it will use that session to try to decrypt the message. If there is no existing session then the client must create a new session and use the new session to decrypt the message. A client must not persist a session or remove one-time keys used by a session -until it has sucessfully decrypted a message using that session. +until it has successfully decrypted a message using that session. The plaintext payload is of the form: @@ -284,7 +284,7 @@ The plaintext payload is of the form: } The type and content of the plaintext message event are given in the payload. -Encyrpting state events is not supported. +Encrypting state events is not supported. We include the room ID in the payload, because otherwise the homeserver would be able to change the room a message was sent in. We include a hash of the From 88176ef14845af72cc968d04ad093d8927bdbc1b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:05:51 +0100 Subject: [PATCH 010/177] Add notes on algorithm naming. Fix some typos --- specification/41_end_to_end_encryption.rst | 77 +++++++++++++++++----- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f0d8a500..a2b4ff39 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -15,7 +15,7 @@ may be found at https://lwn.net/Articles/634144/ Overview -======== +-------- .. code:: @@ -37,7 +37,7 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm claims any one-time keys needed. + 3) Alice selects an algorithm and claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | @@ -56,6 +56,45 @@ Overview /send/ +Algorithms +---------- + +There are two kinds of algorithms: messaging algorithms and key algorithms. +Messaging algorithms are used to securely send messages between devices. +Key algorithms are used for key agreement and digital signatures. + +Messaging Algorithm Names +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messaging algorithm names use the extensible naming scheme used throughout this +specification. Algorithm names that start with "m." are reserved for algorithms +defined by this specification. Implementations wanting to experiment with new +algorithms are encouraged to pick algorithm names that start with their +domain to reduce the risk of collisions. + +The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for +ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the +hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 +for authenticated encryption. + +Algorithm names should be short and meaningful. A name of "m.olm.v1" is too +short. However a name of +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +is too long despite giving a more precise description of the algorithm. + +Algorithm names should list the primitives used by the algorithm so that it +easier to see if the algorithm is using a broken primitive. + +Key Algorithms +~~~~~~~~~~~~~~ + +The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +a Base64 encoded 32-byte Ed25519 public key. + +The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +a Base64 encoded 32-byte Curve25519 public key. + Client Behaviour ---------------- @@ -81,17 +120,17 @@ The JSON object is signed using the process given by `Signing JSON`_. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { - ":": "", + ":": "", }, "signatures": { "": { - ":": "" + ":": "" } } }, "one_time_keys": { - ":": "" + ":": "" } } .. code:: http @@ -101,7 +140,7 @@ The JSON object is signed using the process given by `Signing JSON`_. { "one_time_key_counts": { - "": 50 + "": 50 } } @@ -145,20 +184,20 @@ lies about the keys a user owns. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { ":": "", }, "signatures": { "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" } } } } } } @@ -190,7 +229,7 @@ it can discard the key. Therefore a device could end up trying to store too many private keys. A device that is trying to store too many private keys may discard keys starting with the oldest. -A homeserver should ratelimit the number of one-time keys that a given user or +A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. @@ -203,7 +242,7 @@ time key once it has given that key to another user. { "one_time_keys": { "": { - "": "" + "": "" } } } .. code:: http @@ -215,7 +254,7 @@ time key once it has given that key to another user. "one_time_keys": { "": { "": { - ":": "" + ":": "" } } } } @@ -225,7 +264,6 @@ keys for their local users and forward requests for remote users to ``/_matrix/federation/v1/user/keys/claim`` over federation to the remote server. - Sending a Message ~~~~~~~~~~~~~~~~~ @@ -236,13 +274,18 @@ Encrypted messages are sent in the form. { "type": "m.room.encrypted", "content": { - "algorithm": "" + "algorithm": "", + "": "" } } Using Olm ######### +Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their +list of supported chat algorithms, must list a Curve25519 device key, and +must publish Curve25519 one-time keys. + .. code:: json { @@ -256,7 +299,7 @@ Using Olm "body": "" } } } } -The ciphertext is a mapping from device curve25519 key to an encypted payload +The ciphertext is a mapping from device curve25519 key to an encrypted payload for that device. The ``body`` is a base64 encoded message body. The type is an integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. From 7d805f105e1123aa9fbfb3fe3ba2b8f7fd80950a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:08:52 +0100 Subject: [PATCH 011/177] Mention that Olm uses AES in CBC mode --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a2b4ff39..023a5684 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -75,12 +75,12 @@ domain to reduce the risk of collisions. The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the -hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 -for authenticated encryption. +hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated +HMAC-SHA-256 for authenticated encryption. Algorithm names should be short and meaningful. A name of "m.olm.v1" is too short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" is too long despite giving a more precise description of the algorithm. Algorithm names should list the primitives used by the algorithm so that it From 12e33a3b09ac85af785d077f80543984e1f71dc5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:14:24 +0000 Subject: [PATCH 012/177] Document a v2 api for setting tags on rooms --- api/client-server/v2_alpha/tags.yaml | 148 +++++++++++++++++++++++++ event-schemas/examples/v1/m.tag | 8 ++ event-schemas/schema/v1/m.tag | 23 ++++ specification/modules/room_tagging.rst | 21 ++++ 4 files changed, 200 insertions(+) create mode 100644 api/client-server/v2_alpha/tags.yaml create mode 100644 event-schemas/examples/v1/m.tag create mode 100644 event-schemas/schema/v1/m.tag create mode 100644 specification/modules/room_tagging.rst diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml new file mode 100644 index 00000000..91b81212 --- /dev/null +++ b/api/client-server/v2_alpha/tags.yaml @@ -0,0 +1,148 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server tag API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/rooms/{roomId}/tags": + get: + summary: List the tags for a room. + description: |- + List the tags set by a user on a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to get tags for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to get tags for. + x-example: "!726s6s6q:example.com" + responses: + 200: + description: + The list of tags for the user for the room. + schema: + type: object + properties: + tags: + type: array + items: + type: string + examples: + application/json: |- + { + "tags": [ + "work", + "pinned" + ] + } + "/user/{userId}/rooms/{roomId}/tags/{tag}": + put: + summary: Add a tag to a room. + description: |- + Add a tag to the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to add a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to add a tag to. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to add. + x-example: "work" + - in: body + name: body + required: true + description: |- + An empty JSON object. + schema: + type: object + example: |- + {} + responses: + 200: + description: + The tag was successfully added. + schema: + type: object + examples: + application/json: |- + {} + delete: + summary: Remove a tag from the room. + description: |- + Remove a tag from the room. + security: + - access_token: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to remove a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to remove a tag from. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to remove. + x-example: "work" + responses: + 200: + description: + The tag was successfully removed + schema: + type: object + examples: + application/json: |- + {} diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag new file mode 100644 index 00000000..28431b73 --- /dev/null +++ b/event-schemas/examples/v1/m.tag @@ -0,0 +1,8 @@ +{ + "type": "m.tag", + "content": { + "tags": [ + "work" + ] + } +} diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag new file mode 100644 index 00000000..235d442d --- /dev/null +++ b/event-schemas/schema/v1/m.tag @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "Tag Event", + "description": "Informs the client of tags on a room.", + "properties": { + "type": { + "type": "string", + "enum": ["m.tag"] + }, + "content": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "required": ["type", "content"] +} diff --git a/specification/modules/room_tagging.rst b/specification/modules/room_tagging.rst new file mode 100644 index 00000000..76fff125 --- /dev/null +++ b/specification/modules/room_tagging.rst @@ -0,0 +1,21 @@ +Room Tagging +============ + +.. _module:tagging: + +Users can add tags to rooms. Tags are short strings used to label rooms, e.g. +"work", "familly". A room may have multiple tags. Tags are only visible to the +user that set them but are shared across all their devices. + +Events +------ + +The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` +section of a v2 room sync. + +{{m_tag_event}} + +Client Behaviour +---------------- + +{{v2_tags_http_api}} From 9b0d20315ab6cc5e13cb6a72d02c0cb328b2eb9d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:22:04 +0000 Subject: [PATCH 013/177] Add the tags module to the specification targets --- specification/modules/{room_tagging.rst => tags.rst} | 0 specification/targets.yaml | 1 + 2 files changed, 1 insertion(+) rename specification/modules/{room_tagging.rst => tags.rst} (100%) diff --git a/specification/modules/room_tagging.rst b/specification/modules/tags.rst similarity index 100% rename from specification/modules/room_tagging.rst rename to specification/modules/tags.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfd..2c6a8f20 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/tags.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 65066a76b36de8595bf5b67e075c631c8e77d5b1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 18:30:04 +0000 Subject: [PATCH 014/177] Add the m.tags event to a ``private_user_data`` key rather than including it under the ``ephemeral`` key --- api/client-server/v2_alpha/sync.yaml | 17 ++++++++++++++++- specification/modules/tags.rst | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 266f27bc..3a1d79db 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -121,6 +121,14 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" + private_user_data: + title: Private User Data + type: object + description: |- + The private data that this user has attached to + this room. + allOf: + - $ref: "definitions/event_batch.json" invited: title: Invited type: object @@ -253,11 +261,18 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } ] + }, + "private_user_data": { + "events": [ + { + "type": "m.tags", + "content": {"tags": ["work"]} + } + ] } } }, diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 76fff125..1a364a43 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,8 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` -section of a v2 room sync. +The tags on a room are passed as single ``m.tag`` event in the +``private_user_data`` section of a room in v2 sync. {{m_tag_event}} From 81a60a25ccd78b413d5fc0a25e6267dcf05db23d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:46:10 +0000 Subject: [PATCH 015/177] Update 3pid spec based on new implementation --- event-schemas/examples/v1/m.room.member | 15 ---------- event-schemas/schema/v1/m.room.member | 20 ++----------- specification/modules/third_party_invites.rst | 29 ++++++++++--------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a..d7605dcd 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -4,21 +4,6 @@ "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70..1d09a0dd 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -26,18 +26,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -57,13 +45,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31..140bab86 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -15,13 +15,15 @@ The homeserver asks the identity server whether a Matrix user ID is known for that identifier. If it is, an invite is simply issued for that user. If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets +the invitation, and to notify the invitee's homeserver of this pending invitation if it gets a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property whichi contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -39,9 +41,10 @@ Server behaviour All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -91,16 +94,16 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. + When the third party user validates their identity, their homeserver, H3, + is notified, and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. + H3 validates the signature given to it by the identity server. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 + Having validated these things, H1 writes the invite event to the room, and H3 begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking From b92a0f2b4de47093b229e9f0521aec2bb008435c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:49:22 +0000 Subject: [PATCH 016/177] Remove extra trailing comma --- event-schemas/examples/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index d7605dcd..e2ca5668 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,7 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", + "displayname": "Alice Margatroid" }, "invite_room_state": [ { From 176f919fc89f5d01669aea17003aef8da903ce4e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:00:53 +0000 Subject: [PATCH 017/177] Show multiple examples where present --- .../v1/m.room.member#third_party_invite | 41 +++++++++++++++++++ templating/matrix_templates/sections.py | 2 +- .../matrix_templates/templates/events.tmpl | 4 +- templating/matrix_templates/units.py | 11 +++-- 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite new file mode 100644 index 00000000..4ef571ec --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -0,0 +1,41 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid", + "third_party_invite": { + "signed": { + "mxid": "@alice:localhost", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + }, + "token": "abc123" + } + } + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index e75a75af..81b2bb3c 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -35,7 +35,7 @@ class MatrixSections(Sections): if not filterFn(event_name): continue sections.append(template.render( - example=examples[event_name], + examples=examples[event_name], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440..fbefe17f 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -21,8 +21,10 @@ ======================= ================= =========================================== {% endfor %} -Example: +Example{% if examples | length > 1 %}s{% endif %}: +{% for example in examples %} .. code:: json {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c4342141..d33ba81d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -495,9 +495,14 @@ class MatrixUnits(Units): if not filename.startswith("m."): continue with open(os.path.join(path, filename), "r") as f: - examples[filename] = json.loads(f.read()) - if filename == "m.room.message#m.text": - examples["m.room.message"] = examples[filename] + event_name = filename.split("#")[0] + example = json.loads(f.read()) + + examples[filename] = examples.get(event_name, []) + examples[filename].append(example) + if filename != event_name: + examples[event_name] = examples.get(event_name, []) + examples[event_name].append(example) return examples def load_event_schemas(self): From 810922bb381a0f7162941605ec631f00927071ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:06:44 +0000 Subject: [PATCH 018/177] Fix schema validator for multiple examples --- event-schemas/check_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index e54d3a1c..b10b2d0e 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir): continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) + if schemapath.find("#") >= 0: + schemapath = schemapath[:schemapath.find("#")] try: check_example_file(examplepath, schemapath) except Exception as e: From 5b6f858802332e8fcee368e9db6cdfac7533c995 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 28 Oct 2015 20:28:49 +0000 Subject: [PATCH 019/177] Some initial notes by way of the remote join handshake; with several TODOs and unanswered questions --- specification/server_server_api.rst | 149 +++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 26e040ca..f0239128 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -533,6 +533,150 @@ part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. + +To join a room:: + + GET .../make_join// + Response: JSON encoding of a join proto-event + + PUT .../send_join// + Response: JSON encoding of the state of the room at the time of the event + +Performs the room join handshake. For more information, see "Joining Rooms" +below. + +Joining Rooms +------------- + +When a new user wishes to join room that the user's homeserver already knows +about, the homeserver can immediately determine if this is allowable by +inspecting the state of the room, and if it is acceptable, it can generate, +sign, and emit a new ``m.room.member`` state event adding the user into that +room. When the homeserver does not yet know about the room it cannot do this +directly. Instead, it must take a longer multi-stage handshaking process by +which it first selects a remote homeserver which is already participating in +that room, and uses it to assist in the joining procss. This is the remote join +handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting the +room alias the user is requesting to join with, and a homeserver where existing +room members are already present (referred to as the "resident" server). + +In summary, the remote join handshake consists of the joining server querying +the directory server for information about the room alias; receiving a room ID +and a list of join candidates. The joining server then requests information +about the current state of the room from one of the residents. It uses this +information to construct a ``m.room.member`` event which it finally sends to +a resident server. + +Conceptually these are three different roles of homeserver. In practice the +directory server is likely to be resident in the room, and so may be selected +by the joining server to be the assisting resident. Likewise, it is likely that +the joining server picks the same candidate resident for both phases of event +construction, though in principle any valid candidate may be used at each time. +Thus, any join handshake can potentially involve anywhere from two to four +homeservers, though most in practice will use just two. + +.. TODO-doc + - Consider drawing a request/response diagram here + +The first part of the handshake involves using the directory server to request +the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. + +Once the joining server has the room ID and the join candidates, it then needs +to obtain enough of the current state of the room to fill in the required +fields of the ``m.room.member`` event. It obtains this by selecting a resident +from the candidate list, and requesting the ``make_join`` endpoint using a +``GET`` request, specifying the room ID and the user ID of the new member who +is attempting to join. + +The resident server replies to this request with a JSON-encoded object having a +single key called ``event``; within this is an object whose fields contain some +of the information that the joining server will need. Despite its name, this +object is not a full object; notably it does not need to be hashed or signed by +the assisting resident. The required fields are: + +==================== ======== ============ + Key Type Description +==================== ======== ============ +``type`` String The value ``m.room.member`` +``auth_events`` String An event-reference list containing the + authorization events that would allow this member + to join +``content`` Object The event content +``depth`` Integer (this field must be present but is ignored; it + may be 0) +``event_id`` String A new event ID specified by the assisting + resident +``origin`` String The name of the assisting resident homeserver +``origin_server_ts`` Integer A timestamp added by the resident homeserver +``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_state`` List (TODO(paul): ? - I notice these can be blank) +``room_id`` String The room ID of the room +``sender`` String The user ID of the joining member +``state_key`` String The user ID of the joining member +==================== ======== ============ + +The ``content`` field itself must be an object, containing: + +============== ====== ============ + Key Type Description +============== ====== ============ +``membership`` String The value ``join`` +============== ====== ============ + +The joining server now has sufficient information to construct the real join +event from these protoevent fields. It copies the values of most of them, +adding (or replacing) the following fields: + +==================== ======= ============ + Key Type Description +==================== ======= ============ +``event_id`` String A new event ID specified by the joining homeserver +``origin`` String The name of the joining homeserver +``origin_server_ts`` Integer A timestamp added by the joining homeserver +==================== ======= ============ + +.. TODO-spec + - Why does the protoevent have an event_id, only for the real event to ignore + it and specify a different one? We should definitely pick one or the other. + +This will be a true event, so the joining server should apply the event-signing +algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` +fields. + +To complete the join handshake, the joining server must now submit this new +event to an assisting resident, by using the ``send_join`` endpoint. This is +invoked using the room ID and the event ID of the new member event. + +The assisting resident then accepts this event into the room's event graph, and +responds to the joining server with the full set of state for the newly-joined +room. This is returned as a two-element list, whose first element is the +integer 200, and whose second element contains the following keys: + +.. TODO-spec + - This is likely an implementation bug; see SYN-490. This should probably + actually just return the object directly + +============== ===== ============ + Key Type Description +============== ===== ============ +``auth_chain`` List A list of events giving the authorization chain for this + join event +``state`` List A complete list of the prevailing state events at the + instant just before accepting the new ``m.room.member`` + event +============== ===== ============ + +.. TODO-spec + - (paul) I don't really understand why the full auth_chain events are given + here. What purpose does it serve expanding them out in full, when surely + they'll appear in the state anyway? + - (paul) the state seems to be entirely ignored by synapse, so I'm not really + sure what ought to be there. + Backfilling ----------- .. NOTE:: @@ -763,6 +907,7 @@ Querying directory information:: servers: list of strings giving the join candidates The list of join candidates is a list of server names that are likely to hold -the given room; these are servers that the requesting server may wish to try -joining with. This list may or may not include the server answering the query. +the given room; these are servers that the requesting server may wish to use as +assisting resident servers as part of the remote join handshake. This list may +or may not include the server answering the query. From cddfc110edc9018e44ce1a3690f3f6431a0ea9b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 29 Oct 2015 12:48:04 +0000 Subject: [PATCH 020/177] Review comments --- specification/modules/third_party_invites.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 140bab86..6d3f8f6a 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -22,7 +22,7 @@ and public key to the inviting homeserver. When the invitee's homeserver receives the notification of the binding, it should insert an ``m.room.member`` event into the room's graph for that user, with ``content.membership`` = ``invite``, as well as a -``content.third_party_invite`` property whichi contains proof that the invitee +``content.third_party_invite`` property which contains proof that the invitee does indeed own that third party identifier. Events @@ -94,8 +94,8 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, their homeserver, H3, - is notified, and attempts to issue an ``m.room.member`` event to participate + When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate in the room. H3 validates the signature given to it by the identity server. From d067e50af5990177199a6f3adb0617c2a8ee8836 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:38:33 +0000 Subject: [PATCH 021/177] Document the differences in event formatting between the v1 and v2 client APIs --- specification/events.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/specification/events.rst b/specification/events.rst index 7d64882b..d2428b6b 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -12,6 +12,41 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} +Differences between /v1 and /v2 events +-------------------------------------- + +There are a few differences between how events are formatted for sending +between servers over federation and how they are formatted for sending between +a server and its clients. + +Additionally there are a few differences between the format of events in the +responses to client APIs with a /v1 prefix and responses APIs with a /v2 +prefix. + +Events in responses for APIs with the /v2 prefix are generated from an event +formatted for federation by: + +* Removing the following keys: + ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in + milliseconds that has ellapsed since the event was sent. +* Adding a ``prev_content`` to the ``unsigned`` object if the event is + a ``state event`` which gives previous content of that state key. +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was + redacted which gives the event that redacted it. +* Adding a ``transaction_id`` if the event was sent by the client requesting it. + +Events in responses for APIs with the /v1 prefix are generated from an event +formatted for the /v2 prefix by: + +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. +* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` + then adding a ``invite_room_state`` key to the top level event object. + + Size limits ----------- From 451801bf38159aacc5ee2e05d0230a5b89382712 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:40:05 +0000 Subject: [PATCH 022/177] Add an example of ``prev_content`` in ``unsigned`` to v2 /sync --- api/client-server/v2_alpha/sync.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 01308405..a2d5a2b8 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -241,6 +241,9 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "join"}, + "unsigned": { + "prev_content": {"membership": "invite"} + }, "origin_server_ts": 1417731086795 }, "$74686972643033:example.com": { From d297d83151cf44b785ca5d671ddf4bc8de5f4974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:45:53 +0000 Subject: [PATCH 023/177] Mention that sender is renamed to user_id in v1 --- specification/events.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/events.rst b/specification/events.rst index d2428b6b..a1aece1c 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -43,6 +43,7 @@ formatted for the /v2 prefix by: * Moving the folling keys from the ``unsigned`` object to the top level event object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. * Removing the ``unsigned`` object. +* Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From 91eb25b76d12b7936583e7a3c482d5b3c066245e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 15:45:46 +0000 Subject: [PATCH 024/177] Include the full schema for an http API in the docs by resolving references to other files --- templating/matrix_templates/units.py | 66 +++++++++++++++++++++------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index be113b9c..74aee26c 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -28,7 +28,25 @@ ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" -def get_json_schema_object_fields(obj, enforce_title=False): +def resolve_references(path, schema): + if isinstance(schema, dict): + result = {} + for key, value in schema.items(): + if key == "$ref": + path = os.path.join(os.path.dirname(path), value) + with open(path) as f: + schema = json.load(f) + return resolve_references(path, schema) + else: + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + +def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) if obj.get("type") != "object": @@ -36,9 +54,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): "get_json_schema_object_fields: Object %s isn't an object." % obj ) if enforce_title and not obj.get("title"): - raise Exception( - "get_json_schema_object_fields: Nested object %s doesn't have a title." % obj - ) + # Force a default titile of "NO_TITLE" to make it obvious in the + # specification output which parts of the schema are missing a title + obj["title"] = 'NO_TITLE' required_keys = obj.get("required") if not required_keys: @@ -73,9 +91,15 @@ def get_json_schema_object_fields(obj, enforce_title=False): "Object %s has no properties or parents." % obj ) if not props: # parents only + if include_parents: + if obj["title"] == "NO_TITLE" and parents[0].get("title"): + obj["title"] = parents[0].get("title") + props = parents[0].get("properties") + + if not props: return [{ "title": obj["title"], - "parent": parents[0]["$ref"], + "parent": parents[0].get("$ref"), "no-table": True }] @@ -91,7 +115,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): if prop_val == "object": nested_object = get_json_schema_object_fields( props[key_name]["additionalProperties"], - enforce_title=True + enforce_title=True, + include_parents=include_parents, ) key = props[key_name]["additionalProperties"].get( "x-pattern", "string" @@ -103,8 +128,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True + props[key_name], + enforce_title=True, + include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] @@ -114,8 +140,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": nested_object = get_json_schema_object_fields( - props[key_name]["items"], - enforce_title=True + props[key_name]["items"], + enforce_title=True, + include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] tables += nested_object @@ -159,7 +186,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): class MatrixUnits(Units): - def _load_swagger_meta(self, api, group_name): + def _load_swagger_meta(self, filepath, api, group_name): endpoints = [] for path in api["paths"]: for method in api["paths"][path]: @@ -262,7 +289,10 @@ class MatrixUnits(Units): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields(req_obj) + req_tables = get_json_schema_object_fields( + resolve_references(filepath, req_obj), + include_parents=True, + ) if req_tables > 1: for table in req_tables[1:]: @@ -379,7 +409,10 @@ class MatrixUnits(Units): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields(schema) + res_tables = get_json_schema_object_fields( + resolve_references(filepath, schema), + include_parents=True, + ) for table in res_tables: if "no-table" not in table: endpoint["res_tables"].append(table) @@ -445,13 +478,16 @@ class MatrixUnits(Units): if not filename.endswith(".yaml"): continue self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") if is_v2: group_name = "v2_" + group_name api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) apis[group_name] = api return apis From 9f4d81308d3eca3870f54c196335d2bb5176bd44 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 30 Oct 2015 16:21:50 +0000 Subject: [PATCH 025/177] Pull out separate invite_room_state example --- .../v1/m.room.member#invite_room_state | 30 +++++++++++++++++++ .../v1/m.room.member#third_party_invite | 16 ---------- 2 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/v1/m.room.member#invite_room_state new file mode 100644 index 00000000..e2ca5668 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#invite_room_state @@ -0,0 +1,30 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid" + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite index 4ef571ec..2457302a 100644 --- a/event-schemas/examples/v1/m.room.member#third_party_invite +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -16,22 +16,6 @@ } } }, - "invite_room_state": [ - { - "type": "m.room.name", - "state_key": "", - "content": { - "name": "Forest of Magic" - } - }, - { - "type": "m.room.join_rules", - "state_key": "", - "content": { - "join_rules": "invite" - } - } - ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", From b49472e3b09c385d6a3071c31bdfe425c16174f4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:52:34 +0000 Subject: [PATCH 026/177] Add private_user_data to v1 /initialSync --- api/client-server/v1/sync.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index d07e9399..8517ae4d 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -245,7 +245,12 @@ paths: "user_id": "@alice:localhost" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + }] } ] } @@ -332,6 +337,16 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to + this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: From ad86426e9506a9d0807267acf50e617497ebbd96 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:55:18 +0000 Subject: [PATCH 027/177] Add private_user_data to v1 room /initialSync --- api/client-server/v1/rooms.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d1..46273603 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -311,7 +311,12 @@ paths: "user_id": "@alice:example.com" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!636q39766251:example.com" + }] } schema: title: RoomInfo @@ -371,6 +376,15 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] 403: description: > From f557e698606a2d33f61100cc63bd15a43d7494ba Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 10:13:47 +0000 Subject: [PATCH 028/177] Note that m.tag events can appear in v1 initialSync and /events as well as v2 sync. Only add the room_id for v1 /events since it is redundant in v1 /initialSync --- api/client-server/v1/sync.yaml | 3 +-- specification/modules/tags.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8517ae4d..0f11a98f 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,8 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + "content": {"tags": ["work"]} }] } ] diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 1a364a43..567e5085 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,13 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the -``private_user_data`` section of a room in v2 sync. +The tags on a room are receieved as single ``m.tag`` event in the +``private_user_data`` section of a room in a v2 /sync. + +The ``m.tag`` can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. {{m_tag_event}} From 52f55e05422024e6701b8d211258e1b6a4011a00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:31:25 +0000 Subject: [PATCH 029/177] Allow room tags to have asssociated content, and return that content in the m.tag events --- api/client-server/v2_alpha/tags.yaml | 17 ++++++++--------- event-schemas/examples/v1/m.tag | 6 +++--- event-schemas/schema/v1/m.tag | 7 ++++--- specification/modules/tags.rst | 3 +++ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 91b81212..009e6000 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -49,16 +49,15 @@ paths: type: object properties: tags: - type: array - items: - type: string + title: Tags + type: object examples: application/json: |- { - "tags": [ - "work", - "pinned" - ] + "tags": { + "work": {"order": 1}, + "pinned": {} + } } "/user/{userId}/rooms/{roomId}/tags/{tag}": put: @@ -94,11 +93,11 @@ paths: name: body required: true description: |- - An empty JSON object. + Extra data for the tag, e.g. ordering. schema: type: object example: |- - {} + {"order": 1} responses: 200: description: diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 28431b73..1fd3158e 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -1,8 +1,8 @@ { "type": "m.tag", "content": { - "tags": [ - "work" - ] + "tags": { + "work": {} + } } } diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 235d442d..2ec1a55f 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -11,9 +11,10 @@ "type": "object", "properties": { "tags": { - "type": "array", - "items": { - "type": "string" + "type": "object", + "additionalProperties": { + "title": "Tag", + "type": "object" } } } diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 567e5085..c2b65b3e 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -18,6 +18,9 @@ The ``m.tag`` can also be received in a v1 /events response or in the events appearing in v1 /events will have a ``room_id`` with the room the tags are for. +Each tag has an associated JSON object with information about the tag, e.g how +to order the tags. + {{m_tag_event}} Client Behaviour From 149890227aa62cfd0c01b94b0c532b7a5f2a1b36 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:35:11 +0000 Subject: [PATCH 030/177] Fix wording --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index c2b65b3e..e9b4435e 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -19,7 +19,7 @@ events appearing in v1 /events will have a ``room_id`` with the room the tags are for. Each tag has an associated JSON object with information about the tag, e.g how -to order the tags. +to order the rooms with a given tag. {{m_tag_event}} From 40fa339cf7790837f550cf1ac7fa580a8442ad80 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:00:18 +0000 Subject: [PATCH 031/177] Draw a pretty (well at least I think it's pretty) ASCII diagram of the remote join handshake --- specification/server_server_api.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f0239128..66633376 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -578,8 +578,23 @@ construction, though in principle any valid candidate may be used at each time. Thus, any join handshake can potentially involve anywhere from two to four homeservers, though most in practice will use just two. -.. TODO-doc - - Consider drawing a request/response diagram here +:: + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response The first part of the handshake involves using the directory server to request the room ID and join candidates. This is covered in more detail on the From f6c55979e0cdf6881ca4211745b9fc5cd6f6fbb7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:17:18 +0000 Subject: [PATCH 032/177] Remove TODO comment about SYN-490 as it's unlikely to matter for v1; we'll fix it in v2 --- specification/server_server_api.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66633376..d26cc26e 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -671,10 +671,6 @@ responds to the joining server with the full set of state for the newly-joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: -.. TODO-spec - - This is likely an implementation bug; see SYN-490. This should probably - actually just return the object directly - ============== ===== ============ Key Type Description ============== ===== ============ From 2e3a0b4e00798b672fe3282e24cc1e1c34cd8b0b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 2 Nov 2015 15:26:06 +0000 Subject: [PATCH 033/177] Specify guest accounts --- api/client-server/v2_alpha/registration.yaml | 17 +++++++++++++++++ templating/build.py | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 2bd86e73..681654ae 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -17,7 +17,24 @@ paths: summary: Register for an account on this homeserver. description: |- Register for an account on this homeserver. + + There are two kinds of user account: + + - `user` accounts. These accounts may use the full API described in this specification. + + - `guest` accounts. These accounts may have limited permissions and may not be supported by all servers. + parameters: + - in: query + name: kind + type: string + x-example: guest + required: false + default: user + enum: + - guest + - user + description: The kind of account to register. Defaults to `user`. - in: body name: body schema: diff --git a/templating/build.py b/templating/build.py index b91e1da2..a35d8a08 100755 --- a/templating/build.py +++ b/templating/build.py @@ -84,6 +84,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) output_lines = [wrapper.fill(line) for line in input_lines] + + for i in range(len(output_lines)): + line = output_lines[i] + in_bullet = line.startswith("- ") + if in_bullet: + output_lines[i] = line.replace("\n", "\n " + initial_indent) + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters From e9d361841b8be0f3f4edc2e042439cc77c57225f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:42:58 +0000 Subject: [PATCH 034/177] Fix tag examples --- api/client-server/v1/rooms.yaml | 3 +-- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 46273603..e7b93ad9 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,8 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!636q39766251:example.com" + "content": {"tags": {"work": {}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 0f11a98f..89b9400e 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 3a1d79db..b5465601 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} } ] } From bcb8fac53c3b6beb47777148a387604d11c71a7d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:46:01 +0000 Subject: [PATCH 035/177] Add a description for the tag event --- event-schemas/schema/v1/m.tag | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 2ec1a55f..a36c6a45 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -12,13 +12,14 @@ "properties": { "tags": { "type": "object", + "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag", + "title": "Tag Contents", "type": "object" } } } } }, - "required": ["type", "content"] + "required": ["type", "content"] } From fba3c04e42113e4d3a079b9945abe737f68dc154 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:48:30 +0000 Subject: [PATCH 036/177] Apparently the spec generator breaks if the title in a schema is too long --- event-schemas/schema/v1/m.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index a36c6a45..4c5b4fa5 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -14,7 +14,7 @@ "type": "object", "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag Contents", + "title": "Tag", "type": "object" } } From 3953006792cca87800af18076e715715322c830b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:54:33 +0000 Subject: [PATCH 037/177] Fix spelling --- specification/modules/tags.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index e9b4435e..3d45975c 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -4,13 +4,13 @@ Room Tagging .. _module:tagging: Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "familly". A room may have multiple tags. Tags are only visible to the +"work", "family". A room may have multiple tags. Tags are only visible to the user that set them but are shared across all their devices. Events ------ -The tags on a room are receieved as single ``m.tag`` event in the +The tags on a room are received as single ``m.tag`` event in the ``private_user_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the From d53814097f11ca5f24bb6b1af0b1b9863243f173 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:57:06 +0000 Subject: [PATCH 038/177] Add example content to the tags in example tag events --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- event-schemas/examples/v1/m.tag | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index e7b93ad9..9b8b930f 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 89b9400e..b52908a9 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index b5465601..4c1a7601 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} } ] } diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 1fd3158e..00e81060 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -2,7 +2,7 @@ "type": "m.tag", "content": { "tags": { - "work": {} + "work": {"order": 1} } } } From 30a4f17420f1c6b21648c7e5f729fb4d84468d54 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 3 Nov 2015 16:15:36 +0000 Subject: [PATCH 039/177] Don't mark 'aliases' on m.room.canonical_alias as required, because it isn't. --- event-schemas/schema/v1/m.room.canonical_alias | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias index 25cd00c0..49e0e669 100644 --- a/event-schemas/schema/v1/m.room.canonical_alias +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -13,8 +13,7 @@ "type": "string", "description": "The canonical alias." } - }, - "required": ["alias"] + } }, "state_key": { "type": "string", From ebc02371095be8789051ac0a47de99803f961606 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:35:44 +0000 Subject: [PATCH 040/177] Add the missing titles to the schema --- .../v1/definitions/push_rule.json | 3 +- .../v1/definitions/push_ruleset.json | 37 +++++++++++-------- .../v2_alpha/definitions/event_batch.json | 1 + .../schema/v1/core-event-schema/event.json | 1 + .../v1/core-event-schema/state_event.json | 1 + event-schemas/schema/v1/m.room.member | 1 + 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/v1/definitions/push_rule.json index 4df93f67..d10f11f8 100644 --- a/api/client-server/v1/definitions/push_rule.json +++ b/api/client-server/v1/definitions/push_rule.json @@ -1,4 +1,5 @@ { + "title": "PushRule", "type": "object", "properties": { "default": { @@ -17,4 +18,4 @@ "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/v1/definitions/push_ruleset.json index e0372701..529ebeed 100644 --- a/api/client-server/v1/definitions/push_ruleset.json +++ b/api/client-server/v1/definitions/push_ruleset.json @@ -1,60 +1,65 @@ { - "type": "object", + "type": "object", "properties": { "content": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "override": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "sender": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "underride": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "room": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 75762d75..395aed13 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,6 +5,7 @@ "type": "array", "description": "List of events", "items": { + "title": "Event", "type": "object" } } diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/v1/core-event-schema/event.json index e73aec80..f9103715 100644 --- a/event-schemas/schema/v1/core-event-schema/event.json +++ b/event-schemas/schema/v1/core-event-schema/event.json @@ -5,6 +5,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." }, "type": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json index 88b4900a..5809cf7f 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -11,6 +11,7 @@ "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'." }, "prev_content": { + "title": "EventContent", "type": "object", "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." } diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70..0bd8cd1e 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -8,6 +8,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "properties": { "membership": { "type": "string", From 8322151661f8c8a7cb6cfa3fe1d22ab83ce55642 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:42:49 +0000 Subject: [PATCH 041/177] Don't put a space when appending the "Must be" strings to the desciption if there isn't a description, otherwise it will mess up the indent --- templating/matrix_templates/units.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 74aee26c..9aa9489a 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -164,12 +164,16 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" + if desc: + desc += " " desc += ( - " One of: %s" % json.dumps(props[key_name]["enum"]) + "One of: %s" % json.dumps(props[key_name]["enum"]) ) else: + if desc: + desc += " " desc += ( - " Must be '%s'." % props[key_name]["enum"][0] + "Must be '%s'." % props[key_name]["enum"][0] ) if isinstance(value_type, list): value_type = " or ".join(value_type) From e49ea9015f4ee5f86b2d401190ec39517c5ae49b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:39:36 +0000 Subject: [PATCH 042/177] Deduplicate tables with the same title --- templating/matrix_templates/units.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9aa9489a..9cc68f49 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -185,7 +185,17 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) - return tables + + titles = set() + filtered = [] + for table in tables: + if table.get("title") in titles: + continue + + titles.add(table.get("title")) + filtered.append(table) + + return filtered class MatrixUnits(Units): From 8070489080afdc08179087c64b1f6a4a261c224d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:44:20 +0000 Subject: [PATCH 043/177] Handle lists of types in arrays --- templating/matrix_templates/units.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9cc68f49..cec5b83e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -147,7 +147,10 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals value_type = "[%s]" % nested_object[0]["title"] tables += nested_object else: - value_type = "[%s]" % props[key_name]["items"]["type"] + value_type = props[key_name]["items"]["type"] + if isinstance(value_type, list): + value_type = " or ".join(value_type) + value_type = "[%s]" % value_type array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: From c7199463704709ca4c0dc8a3b60572210f464c84 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:51:50 +0000 Subject: [PATCH 044/177] Enable syntax highlighting for message type examples --- templating/matrix_templates/templates/msgtypes.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index f7862451..18d3492b 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -18,6 +18,8 @@ ================== ================= =========================================== {% endfor %} -Example:: +Example: + +.. code:: json {{example | jsonify(4, 4)}} From 2fd5fc39a032dbce80a3aba4361f79f553cc5b85 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 10:55:11 +0000 Subject: [PATCH 045/177] Add spec for calculating display names for rooms and users Merged from https://github.com/matrix-org/matrix-doc/pull/145 --- specification/modules/instant_messaging.rst | 125 +++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 09fcb843..7a2142dd 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -116,12 +116,58 @@ of the client-server API will resolve this by attaching the transaction ID of th sending request to the event itself. +Calculating the display name for a user +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show the human-readable display name of a room member as +part of a membership list, or when they send a message. However, different +members may have conflicting display names. Display names MUST be disambiguated +before showing them to the user, in order to prevent spoofing of other users. + +To ensure this is done consistently across clients, clients SHOULD use the +following algorithm to calculate a disambiguated display name for a given user: + +1. Inspect the ``m.room.member`` state event for the relevant user id. +2. If the ``m.room.member`` state event has no ``displayname`` field, or if + that field has a ``null`` value, use the raw user id as the display + name. Otherwise: +3. If the ``m.room.member`` event has a ``displayname`` which is unique among + members of the room with ``membership: join`` or ``membership: invite``, use + the given ``displayname`` as the user-visible display name. Otherwise: +4. The ``m.room.member`` event has a non-unique ``displayname``. This should be + disambiguated using the user id, for example "display name + (@id:homeserver.org)". + +Developers should take note of the following when implementing the above +algorithm: + +* The user-visible display name of one member can be affected by changes in the + state of another member. For example, if ``@user1:matrix.org`` is present in + a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins + the room, also with ``displayname: Alice``, *both* users must be given + disambiguated display names. Similarly, when one of the users then changes + their display name, there is no longer a clash, and *both* users can be given + their chosen display name. Clients should be alert to this possibility and + ensure that all affected users are correctly renamed. + +* The display name of a room may also be affected by changes in the membership + list. This is due to the room name sometimes being based on user display + names (see `Calculating the display name for a room`_). + +* If the entire membership list is searched for clashing display names, this + leads to an O(N^2) implementation for building the list of room members. This + will be very inefficient for rooms with large numbers of members. It is + recommended that client implementations maintain a hash table mapping from + ``displayname`` to a list of room members using that name. Such a table can + then be used for efficient calculation of whether disambiguation is needed. + + Displaying membership information with messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Clients may wish to show the display name and avatar URL of the room member who sent a message. This can be achieved by inspecting the ``m.room.member`` state -event for that user ID. +event for that user ID (see `Calculating the display name for a user`_). When a user paginates the message history, clients may wish to show the **historical** display name and avatar URL for a room member. This is possible @@ -133,6 +179,83 @@ events update the old state. When paginated events are processed sequentially, the old state represents the state of the room *at the time the event was sent*. This can then be used to set the historical display name and avatar URL. + +Calculating the display name for a room +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show a human-readable name for a room. There are a number +of possibilities for choosing a useful name. To ensure that rooms are named +consistently across clients, clients SHOULD use the following algorithm to +choose a name: + +1. If the room has an `m.room.name`_ state event, use the name given by that + event. + +#. If the room has an `m.room.canonical_alias`_ state event, use the alias + given by that event. + +#. If neither of the above events are present, a name should be composed based + on the members of the room. Clients should consider `m.room.member`_ events + for users other than the logged-in user, with ``membership: join`` or + ``membership: invite``. + + .. _active_members: + + i. If there is only one such event, the display name for the room should be + the `disambiguated display name`_ of the corresponding user. + + #. If there are two such events, they should be lexicographically sorted by + their ``state_key`` (i.e. the corresponding user IDs), and the display + name for the room should be the `disambiguated display name`_ of both + users: " and ", or a localised variant thereof. + + #. If there are three or more such events, the display name for the room + should be based on the disambiguated display name of the user + corresponding to the first such event, under a lexicographical sorting + according to their ``state_key``. The display name should be in the + format " and others" (or a localised variant thereof), where N + is the number of `m.room.member`_ events with ``membership: join`` or + ``membership: invite``, excluding the logged-in user and "user1". + + For example, if Alice joins a room, where Bob (whose user id is + ``@superuser:example.com``), Carol (user id ``@carol:example.com``) and + Dan (user id ``@dan:matrix.org``) are in conversation, Alice's + client should show the room name as "Carol and 2 others". + + .. TODO-spec + Sorting by user_id certainly isn't ideal, as IDs at the start of the + alphabet will end up dominating room names: they will all be called + "Arathorn and 15 others". Furthermore - user_ids are not necessarily + ASCII, which means we need to either specify a collation order, or specify + how to choose one. + + Ideally we might sort by the time when the user was first invited to, or + first joined the room. But we don't have this information. + +#. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and + no active members other than the current user, clients should consider + ``m.room.member`` events with ``membership: leave``. If such events exist, a + display name such as "Empty room (was and others)" (or a + localised variant thereof) should be used, following similar rules as for + active members (see `above `_). + +#. A complete absence of ``m.room.name``, ``m.room.canonical_alias``, and + ``m.room.member`` events is likely to indicate a problem with creating the + room or synchronising the state table; however clients should still handle + this situation. A display name such as "Empty room" (or a localised variant + thereof) should be used in this situation. + +.. _`disambiguated display name`: `Calculating the display name for a user`_ + +Clients SHOULD NOT use `m.room.aliases`_ events as a source for room names, as +it is difficult for clients to agree on the best alias to use, and aliases can +change unexpectedly. + +.. TODO-spec + How can we make this less painful for clients to implement, without forcing + an English-language implementation on them all? + + Server behaviour ---------------- From 8cba11b1cd6f70906aa3f8ce03d4332d469af13f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 11:06:31 +0000 Subject: [PATCH 046/177] Add some links to spec bugs for display names Just added a couple of TODO comments to useful jira bugs --- specification/modules/instant_messaging.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 7a2142dd..a58c762f 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -138,6 +138,11 @@ following algorithm to calculate a disambiguated display name for a given user: disambiguated using the user id, for example "display name (@id:homeserver.org)". + .. TODO-spec + what does it mean for a ``displayname`` to be 'unique'? Are we + case-sensitive? Do we care about homograph attacks? See + https://matrix.org/jira/browse/SPEC-221. + Developers should take note of the following when implementing the above algorithm: @@ -232,6 +237,8 @@ choose a name: Ideally we might sort by the time when the user was first invited to, or first joined the room. But we don't have this information. + See https://matrix.org/jira/browse/SPEC-267 for further discussion. + #. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and no active members other than the current user, clients should consider ``m.room.member`` events with ``membership: leave``. If such events exist, a From 161441fa3ae83db985c51c9f1c8a4cb96494bd45 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 18:11:20 +0000 Subject: [PATCH 047/177] Update 3pid spec based on new implementation --- api/client-server/v1/membership.yaml | 110 +-------------- .../v1/third_party_membership.yaml | 127 ++++++++++++++++++ event-schemas/examples/v1/m.room.member | 17 +-- event-schemas/schema/v1/m.room.member | 20 +-- specification/modules/anonymous_access.rst | 50 +++++++ specification/modules/third_party_invites.rst | 85 ++++++++---- 6 files changed, 246 insertions(+), 163 deletions(-) create mode 100644 api/client-server/v1/third_party_membership.yaml create mode 100644 specification/modules/anonymous_access.rst diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index f8dfdea5..c089e154 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -73,9 +73,12 @@ paths: post: summary: Invite a user to participate in a particular room. description: |- + .. _invite-by-user-id-endpoint: + *Note that there are two forms of this API, which are documented separately. This version of the API requires that the inviter knows the Matrix - identifier of the invitee.* + identifier of the invitee. The other is documented in the* + `third party invites section`_. This API invites a user to participate in a particular room. They do not start participating in the room until they actually join the @@ -86,6 +89,8 @@ paths: If the user was invited to the room, the home server will append a ``m.room.member`` event to the room. + + .. _third party invites section: `invite-by-third-party-id-endpoint`_ security: - accessToken: [] parameters: @@ -132,106 +137,3 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" - - "/rooms/{roomId}/invite": - post: - summary: Invite a user to participate in a particular room. - description: |- - *Note that there are two forms of this API, which are documented separately. - This version of the API does not require that the inviter know the Matrix - identifier of the invitee, and instead relies on third party identifiers. - The homeserver uses an identity server to perform the mapping from - third party identifier to a Matrix identifier.* - - This API invites a user to participate in a particular room. - They do not start participating in the room until they actually join the - room. - - Only users currently in a particular room can invite other users to - join that room. - - If the identity server did know the Matrix user identifier for the - third party identifier, the home server will append a ``m.room.member`` - event to the room. - - If the identity server does not know a Matrix user identifier for the - passed third party identifier, the homeserver will issue an invitation - which can be accepted upon providing proof of ownership of the third - party identifier. This is achieved by the identity server generating a - token, which it gives to the inviting homeserver. The homeserver will - add an ``m.room.third_party_invite`` event into the graph for the room, - containing that token. - - When the invitee binds the invited third party identifier to a Matrix - user ID, the identity server will give the user a list of pending - invitations, each containing: - - - The room ID to which they were invited - - - The token given to the homeserver - - - A signature of the token, signed with the identity server's private key - - - The matrix user ID who invited them to the room - - If a token is requested from the identity server, the home server will - append a ``m.room.third_party_invite`` event to the room. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier (not alias) to which to invite the user. - required: true - x-example: "!d41d8cd:matrix.org" - - in: body - name: body - required: true - schema: - type: object - example: |- - { - "id_server": "matrix.org", - "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" - } - properties: - id_server: - type: string - description: The hostname+port of the identity server which should be used for third party identifier lookups. - medium: - type: string - # TODO: Link to identity service spec when it eixsts - description: The kind of address being passed in the address field, for example ``email``. - address: - type: string - description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] - responses: - 200: - description: The user has been invited to join the room. - examples: - application/json: |- - {} - schema: - type: object - 403: - description: |- - You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: - - - The invitee has been banned from the room. - - The invitee is already a member of the room. - - The inviter is not currently in the room. - - The inviter's power level is insufficient to invite users to the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml new file mode 100644 index 00000000..90b167b4 --- /dev/null +++ b/api/client-server/v1/third_party_membership.yaml @@ -0,0 +1,127 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API for third party identifiers" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + description: |- + .. _invite-by-third-party-id-endpoint: + + *Note that there are two forms of this API, which are documented separately. + This version of the API does not require that the inviter know the Matrix + identifier of the invitee, and instead relies on third party identifiers. + The homeserver uses an identity server to perform the mapping from + third party identifier to a Matrix identifier. The other is documented in the* + `joining rooms section`_. + + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + Only users currently in a particular room can invite other users to + join that room. + + If the identity server did know the Matrix user identifier for the + third party identifier, the home server will append a ``m.room.member`` + event to the room. + + If the identity server does not know a Matrix user identifier for the + passed third party identifier, the homeserver will issue an invitation + which can be accepted upon providing proof of ownership of the third + party identifier. This is achieved by the identity server generating a + token, which it gives to the inviting homeserver. The homeserver will + add an ``m.room.third_party_invite`` event into the graph for the room, + containing that token. + + When the invitee binds the invited third party identifier to a Matrix + user ID, the identity server will give the user a list of pending + invitations, each containing: + + - The room ID to which they were invited + + - The token given to the homeserver + + - A signature of the token, signed with the identity server's private key + + - The matrix user ID who invited them to the room + + If a token is requested from the identity server, the home server will + append a ``m.room.third_party_invite`` event to the room. + + .. _joining rooms section: `invite-by-user-id-endpoint`_ + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "id_server": "matrix.org", + "medium": "email", + "address": "cheeky@monkey.com", + "display_name": "A very cheeky monkey" + } + properties: + id_server: + type: string + description: The hostname+port of the identity server which should be used for third party identifier lookups. + medium: + type: string + # TODO: Link to identity service spec when it eixsts + description: The kind of address being passed in the address field, for example ``email``. + address: + type: string + description: The invitee's third party identifier. + display_name: + type: string + description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. + required: ["id_server", "medium", "address", "display_name"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a..e2ca5668 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,22 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } + "displayname": "Alice Margatroid" }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 0bd8cd1e..81057049 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -27,18 +27,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -58,13 +46,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/anonymous_access.rst b/specification/modules/anonymous_access.rst new file mode 100644 index 00000000..5a421187 --- /dev/null +++ b/specification/modules/anonymous_access.rst @@ -0,0 +1,50 @@ +Guest access +================ + +.. _module:guest-access: + +It may be desirable to allow users without a fully registered user account to +ephemerally access Matrix rooms. This module specifies limited ways of doing so. + +Note that this is not currently a complete anonymous access solution; in +particular, it only allows servers to provided anonymous access to rooms in +which they are already participating, and relies on individual homeservers to +adhere to the conventions which this module sets, rather than allowing all +participating homeservers to enforce them. + +Events +------ + +{{m_room_guest_accessibility}} + +Client behaviour +---------------- +A client can register for guest access using the FOO endpoint. From that point +on, they can interact with a limited subset of the existing client-server API, +as if they were a fully registered user, using the access token granted to them +by the server. + +These users are only allowed to make calls in relation to rooms which have the +``m.room.history_visibility`` event set to ``world_readable``. + +The APIs they are allowed to hit are: + +/rooms/{roomId}/messages +/rooms/{roomId}/state +/rooms/{roomId}/state/{eventType}/{stateKey} +/events + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. Federation-specific logic should be included +here. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31..4d268631 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -1,27 +1,30 @@ Third party invites =================== -.. _module:third_party_invites: +.. _module:third-party-invites: This module adds in support for inviting new members to a room where their Matrix user ID is not known, instead addressing them by a third party identifier such as an email address. - There are two flows here; one if a Matrix user ID is known for the third party identifier, and one if not. Either way, the client calls ``/invite`` with the details of the third party identifier. The homeserver asks the identity server whether a Matrix user ID is known for -that identifier. If it is, an invite is simply issued for that user. +that identifier: + +- If it is, an invite is simply issued for that user. -If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets -a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +- If it is not, the homeserver asks the identity server to record the details of + the invitation, and to notify the invitee's homeserver of this pending invitation if it gets + a binding for this identifier in the future. The identity server returns a token + and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property which contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -33,15 +36,18 @@ Client behaviour A client asks a server to invite a user by their third party identifier. +{{third_party_membership_http_api}} + Server behaviour ---------------- All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -84,24 +90,24 @@ membership is questionable. For example: - If room R has two participating homeservers, H1, H2 +#. Room R has two participating homeservers, H1, H2 - And user A on H1 invites a third party identifier to room R +#. User A on H1 invites a third party identifier to room R - H1 asks the identity server for a binding to a Matrix user ID, and has none, - so issues an ``m.room.third_party_invite`` event to the room. +#. H1 asks the identity server for a binding to a Matrix user ID, and has none, + so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. +#. When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. +#. H3 validates the signature given to it by the identity server. - H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` - property *and* check ``key_validity_url``. +#. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` + property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 - begins participating in the room. H2 *must* accept this event. +#. Having validated these things, H1 writes the invite event to the room, and H3 + begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking ``key_validity_url`` is that we must ensure event acceptance is deterministic. @@ -112,3 +118,32 @@ This relies on participating servers trusting each other, but that trust is already implied by the server-server protocol. Also, the public key signature verification must still be performed, so the attack surface here is minimized. +Security considerations +----------------------- + +There are a number of privary and trust implications to this module. + +It is important for user privacy that leaking the mapping between a matrix user +ID and a third party identifier is hard. In particular, being able to look up +all third party identifiers from a matrix user ID (and accordingly, being able +to link each third party identifier) should be avoided wherever possible. +To this end, when implementing this API care should be taken to avoid +adding links between these two identifiers as room events. This mapping can be +unintentionally created by specifying the third party identifier in the +``display_name`` field of the ``m.room.third_party_invite`` event, and then +observing which matrix user ID joins the room using that invite. Clients SHOULD +set ``display_name`` to a value other than the third party identifier, e.g. the +invitee's common name. + +Homeservers are not required to trust any particular identity server(s). It is +generally a client's responsibility to decide which identity servers it trusts, +not a homeserver's. Accordingly, this API takes identity servers as input from +end users, and doesn't have any specific trusted set. It is possible some +homeservers may want to supply defaults, or reject some identity servers for +*its* users, but no homeserver is allowed to dictate which identity servers +*other* homeservers' users trust. + +There is some risk of denial of service attacks by flooding homeservers or +identity servers with many requests, or much state to store. Defending against +these is left to the implementer's discretion. + From 559747e77a1bac4cd104f9526bfcaa84217651fb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:18:28 +0000 Subject: [PATCH 048/177] speculator: Sent Content-Type: text/html header Go is auto-detecting that this is XML (because for some reason we generate XHTML), and serving it with a Content-Type header text/xml. This causes the browser to render it as XHTML, which gives interesting quirks like extra newlines. This forces the browser to interpret it as HTML. What we should probably do instead of stop generating XHTML and start generating HTML. But in the mean time, this will fix the rendering issues. --- scripts/speculator/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 85fb2596..ef37b931 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -384,9 +384,9 @@ func main() { log.Fatal(err) } s := server{masterCloneDir} - http.HandleFunc("/spec/", s.serveSpec) - http.HandleFunc("/diff/rst/", s.serveRSTDiff) - http.HandleFunc("/diff/html/", s.serveHTMLDiff) + http.HandleFunc("/spec/", forceHTML(s.serveSpec)) + http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) @@ -394,6 +394,13 @@ func main() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } +func forceHTML(h func(w http.ResponseWriter, req *http.Request)) func(w http.ResponseWriter, req *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/html") + h(w, req) + } +} + func serveText(s string) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, req *http.Request) { io.WriteString(w, s) From 937ff046d866b96d068c3e1424689ebb58de99f5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:21:16 +0000 Subject: [PATCH 049/177] Force / to be HTML too --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index ef37b931..7f86bd62 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -388,7 +388,7 @@ func main() { http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) - http.HandleFunc("/", listPulls) + http.HandleFunc("/", forceHTML(listPulls)) fmt.Printf("Listening on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From 7f6eafdce52972021643512d09e46eaef88c810d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 14:46:55 +0000 Subject: [PATCH 050/177] continuserv: set Content-Type header --- scripts/continuserv/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95..425a1af9 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -113,6 +113,7 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() b := toServe.Load().([]byte) + w.Header().Set("Content-Type", "text/html") w.Write(b) } From 1be5b856bd2447aa557ada4fcde8b8ddf63a3f70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:05:07 +0000 Subject: [PATCH 051/177] Preserve text/plain for errors Newlines are nice --- scripts/continuserv/main.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 425a1af9..4613437a 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,7 +25,7 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + toServe atomic.Value // Always contains valid bytesOrErr to serve. May be stale unless wg is zero. wg sync.WaitGroup // Indicates how many updates are pending. mu sync.Mutex // Prevent multiple updates in parallel. ) @@ -112,9 +112,14 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() - b := toServe.Load().([]byte) - w.Header().Set("Content-Type", "text/html") - w.Write(b) + b := toServe.Load().(bytesOrErr) + if b.err != nil { + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(b.err.Error())) + } else { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(b.bytes)) + } } func populateOnce(dir string) { @@ -127,15 +132,15 @@ func populateOnce(dir string) { cmd.Stderr = &b err := cmd.Run() if err != nil { - toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) if err != nil { - toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) return } - toServe.Store(specBytes) + toServe.Store(bytesOrErr{specBytes, nil}) } func doPopulate(ch chan struct{}, dir string) { @@ -160,3 +165,8 @@ func exists(path string) bool { _, err := os.Stat(path) return !os.IsNotExist(err) } + +type bytesOrErr struct { + bytes []byte + err error +} From bbf9e229a7735c1c1ac295169a9f81b83874cc91 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:09:09 +0000 Subject: [PATCH 052/177] continuserv: guard concurrent accesses to wg --- scripts/continuserv/main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95..2c6072a3 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,9 +25,11 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. - wg sync.WaitGroup // Indicates how many updates are pending. - mu sync.Mutex // Prevent multiple updates in parallel. + mu sync.Mutex // Prevent multiple updates in parallel. + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + + wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. + wg sync.WaitGroup // Indicates how many updates are pending. ) func main() { @@ -111,7 +113,9 @@ func filter(e fsnotify.Event) bool { } func serve(w http.ResponseWriter, req *http.Request) { + wgMu.Lock() wg.Wait() + wgMu.Unlock() b := toServe.Load().([]byte) w.Write(b) } @@ -143,7 +147,9 @@ func doPopulate(ch chan struct{}, dir string) { select { case <-ch: if pending == 0 { + wgMu.Lock() wg.Add(1) + wgMu.Unlock() } pending++ case <-time.After(10 * time.Millisecond): From e72151f2c3f8e2825eb2719b3e4598672c134a03 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 18:15:21 +0000 Subject: [PATCH 053/177] Specify guest room access This was reviewed as PR #150 and merged from daniel/anonymousaccess --- api/client-server/v1/guest_events.yaml | 103 ++++++++++++++++++ event-schemas/examples/v1/m.room.guest_access | 12 ++ event-schemas/schema/v1/m.room.guest_access | 30 +++++ specification/modules/guest_access.rst | 81 ++++++++++++++ specification/targets.yaml | 1 + templating/matrix_templates/units.py | 9 +- 6 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 api/client-server/v1/guest_events.yaml create mode 100644 event-schemas/examples/v1/m.room.guest_access create mode 100644 event-schemas/schema/v1/m.room.guest_access create mode 100644 specification/modules/guest_access.rst diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/v1/guest_events.yaml new file mode 100644 index 00000000..bbb5799a --- /dev/null +++ b/api/client-server/v1/guest_events.yaml @@ -0,0 +1,103 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Sync Guest API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/events": + get: + summary: Listen on the event stream. + description: |- + This will listen for new events related to a particular room and return + them to the caller. This will block until an event is received, or until + the ``timeout`` is reached. + + This API is the same as the non-guest /events endpoint, but can be + called by guest users. + security: + - accessToken: [] + parameters: + - in: query + type: string + name: from + description: |- + The token to stream from. This token is either from a previous + request to this API or from the initial sync API. + required: false + x-example: "s3456_9_0" + - in: query + type: integer + name: timeout + description: The maximum time in milliseconds to wait for an event. + required: false + x-example: "35000" + - in: query + type: array + items: + type: string + name: room_id + description: |- + The room IDs for which events should be returned. + x-example: + - "!somewhere:over" + - "!the:rainbow" + responses: + 200: + description: "The events received, which may be none." + examples: + application/json: |- + { + "start": "s3456_9_0", + "end": "s3457_9_0", + "chunk": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + schema: + type: object + properties: + start: + type: string + description: |- + A token which correlates to the first value in ``chunk``. This + is usually the same token supplied to ``from=``. + end: + type: string + description: |- + A token which correlates to the last value in ``chunk``. This + token should be used in the next request to ``/events``. + chunk: + type: array + description: "An array of events." + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" + 400: + description: "Bad pagination ``from`` parameter." diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/v1/m.room.guest_access new file mode 100644 index 00000000..8ad4d294 --- /dev/null +++ b/event-schemas/examples/v1/m.room.guest_access @@ -0,0 +1,12 @@ +{ + "age": 242353, + "content": { + "guest_access": "can_join" + }, + "state_key": "", + "origin_server_ts": 1431961217938, + "event_id": "$WLGTSEFSEG:localhost", + "type": "m.room.guest_access", + "room_id": "!Cuyf34gef24u:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/v1/m.room.guest_access new file mode 100644 index 00000000..a0141f37 --- /dev/null +++ b/event-schemas/schema/v1/m.room.guest_access @@ -0,0 +1,30 @@ +{ + "type": "object", + "title": "Controls whether guest users are allowed to join rooms.", + "description": "This event controls whether guest users are allowed to join rooms. If this event is absent, servers should act as if it is present and has the guest_access value \"forbidden\".", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "guest_access": { + "type": "string", + "description": "Whether guests can join the room.", + "enum": ["can_join", "forbidden"] + } + }, + "required": ["guest_access"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.guest_access"] + } + } +} diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst new file mode 100644 index 00000000..ac373af9 --- /dev/null +++ b/specification/modules/guest_access.rst @@ -0,0 +1,81 @@ +Guest access +============ + +.. _module:guest-access: + +There are times when it is desirable for clients to be able to interact with +rooms without having to fully register for an account on a homeserver or join +the room. This module specifies how these clients should interact with servers +in order to participate in rooms as guests. + +Guest users retrieve access tokens from a homeserver using the ordinary +`register endpoint <#post-matrix-client-api-v2-alpha-register>`_, specifying +the ``kind`` parameter as ``guest``. They may then interact with the +client-server API as any other user would, but will only have access to a subset +of the API as described the Client behaviour subsection below. +Homeservers may choose not to allow this access at all to their local users, but +have no information about whether users on other homeservers are guests or not. + +This module does not fully factor in federation; it relies on individual +homeservers properly adhering to the rules set out in this module, rather than +allowing all homeservers to enforce the rules on each other. + +Events +------ +{{m_room_guest_access_event}} + +Client behaviour +---------------- +The following API endpoints are allowed to be accessed by guest accounts for +retrieving events: + +* `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ +* `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ +* `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ + +There is also a special version of the +`GET /events <#get-matrix-client-api-v1-events>`_ endpoint: + +{{guest_events_http_api}} + +They will only return events which happened while the room state had the +``m.room.history_visibility`` state event present with ``history_visibility`` +value ``world_readable``. Guest clients do not need to join rooms in order to +receive events for them. + +The following API endpoints are allowed to be accessed by guest accounts for +sending events: + +* `POST /rooms/:room_id/join <#post-matrix-client-api-v1-rooms-roomid-join>`_ +* `PUT /rooms/:room_id/send/m.room.message/:txn_id <#put-matrix-client-api-v1-rooms-roomid-send-eventtype-txnid>`_ + +Guest clients *do* need to join rooms in order to send events to them. + +Server behaviour +---------------- +Servers are required to only return events to guest accounts for rooms where +the room state at the event had the ``m.room.history_visibility`` state event +present with ``history_visibility`` value ``world_readable``. These events may +be returned even if the anonymous user is not joined to the room. + +Servers MUST only allow guest users to join rooms if the ``m.room.guest_access`` +state event is present on the room, and has the ``guest_access`` value +``can_join``. If the ``m.room.guest_access`` event is changed to stop this from +being the case, the server MUST set those users' ``m.room.member`` state to +``leave``. + +Security considerations +----------------------- +Each homeserver manages its own guest accounts itself, and whether an account +is a guest account or not is not information passed from server to server. +Accordingly, any server participating in a room is trusted to properly enforce +the permissions outlined in this section. + +Clients may wish to display to their users that rooms which are +``world_readable`` *may* be showing messages to non-joined users. There is no +way using this module to find out whether any non-joined guest users *do* see +events in the room, or to list or count any guest users. + +Homeservers may want to enable protections such as captchas for guest +registration to prevent spam, denial of service, and similar attacks. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfd..8e6a2ce0 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/guest_access.rst title_styles: ["=", "-", "~", "+", "^", "`"] diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index cec5b83e..adb7f427 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -369,7 +369,7 @@ class MatrixUnits(Units): ] if len(params_missing_examples) == 0: path_template = api.get("basePath", "").rstrip("/") + path - qps = {} + qps = [] body = "" for param in single_api.get("parameters", []): if param["in"] == "path": @@ -381,7 +381,12 @@ class MatrixUnits(Units): elif param["in"] == "body": body = param["schema"]["example"] elif param["in"] == "query": - qps[param["name"]] = param["x-example"] + example = param["x-example"] + if type(example) == list: + for value in example: + qps.append((param["name"], value)) + else: + qps.append((param["name"], example)) query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) if body: endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( From 436a35e9f647fcaa5ef27b48ef26b683440fcb40 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 16:04:31 +0000 Subject: [PATCH 054/177] Document macaroon type=login --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 93622c3d..a7c1b036 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -25,10 +25,13 @@ Specified caveats: | gen | Generation of the macaroon caveat spec. | 1 | | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| refresh - only used to authorize a token refresh | +| | | refresh - only used to authorize a token refresh | +| | | login - issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | -| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | -| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | -| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| -| Note that exact equality of time is largely meaningless. | +| | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| | | Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| | | Note that exact equality of time is largely meaningless. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ From 24c2036a35738ebf9c1ebe3b01cef1943a087702 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 17:30:18 +0000 Subject: [PATCH 055/177] 3pid invites: remove mentions of display_name --- api/client-server/v1/third_party_membership.yaml | 8 ++------ specification/modules/third_party_invites.rst | 11 ++++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml index 90b167b4..a10d6167 100644 --- a/api/client-server/v1/third_party_membership.yaml +++ b/api/client-server/v1/third_party_membership.yaml @@ -84,8 +84,7 @@ paths: { "id_server": "matrix.org", "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" + "address": "cheeky@monkey.com" } properties: id_server: @@ -98,10 +97,7 @@ paths: address: type: string description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] + required: ["id_server", "medium", "address"] responses: 200: description: The user has been invited to join the room. diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 4d268631..d8e3d4d9 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -127,13 +127,10 @@ It is important for user privacy that leaking the mapping between a matrix user ID and a third party identifier is hard. In particular, being able to look up all third party identifiers from a matrix user ID (and accordingly, being able to link each third party identifier) should be avoided wherever possible. -To this end, when implementing this API care should be taken to avoid -adding links between these two identifiers as room events. This mapping can be -unintentionally created by specifying the third party identifier in the -``display_name`` field of the ``m.room.third_party_invite`` event, and then -observing which matrix user ID joins the room using that invite. Clients SHOULD -set ``display_name`` to a value other than the third party identifier, e.g. the -invitee's common name. +To this end, the third party identifier is not put in any event, rather an +opaque display name provided by the identity server is put into the events. +Clients should not remember or display third party identifiers from invites, +other than for the use of the inviter themself. Homeservers are not required to trust any particular identity server(s). It is generally a client's responsibility to decide which identity servers it trusts, From c1866ebebc708d0e8a404edbc3f0bb9b0f3d4297 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:26:06 +0000 Subject: [PATCH 056/177] Fix table formatting --- drafts/macaroons_caveats.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index a7c1b036..71c4784e 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -21,14 +21,17 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | Caveat name | Description | Legal Values | -+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ ++=============+==================================================+================================================================================================+ | gen | Generation of the macaroon caveat spec. | 1 | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | | | | refresh - only used to authorize a token refresh | | | | login - issued as a very short-lived token by third party login flows; proves that | | | | authentication has happened but doesn't grant any privileges other than being able to be | | | | exchanged for other tokens. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | | | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | From 51fe4a90b6f3e79428f1e1160b7c7df4f906adb9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:28:27 +0000 Subject: [PATCH 057/177] More formatting fixes --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 71c4784e..11e36e59 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -9,12 +9,15 @@ Caveats can only be used for reducing the scope of a token, never for increasing Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. -All caveats must take the form: +All caveats must take the form:: -`key` `operator` `value` -where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] -`operator` is a non-empty string which does not contain whitespace -`value` is a non-empty string + key operator value + +where: + - ``key`` is a non-empty string drawn from the character set [A-Za-z0-9_] + - ``operator`` is a non-empty string which does not contain whitespace + - ``value`` is a non-empty string + And these are joined by single space characters. Specified caveats: From c8f6ed11074f489df5562ce614e3113f48ea77d9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:31:31 +0000 Subject: [PATCH 058/177] More formatting.. --- drafts/macaroons_caveats.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 11e36e59..de5973fa 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -29,11 +29,11 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ -| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| | | refresh - only used to authorize a token refresh | -| | | login - issued as a very short-lived token by third party login flows; proves that | -| | | authentication has happened but doesn't grant any privileges other than being able to be | -| | | exchanged for other tokens. | +| type | The purpose of this macaroon. | - ``access``: used to authorize any action except token refresh | +| | | - ``refresh``: only used to authorize a token refresh | +| | | - ``login``: issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | From c00abe9f2faa80f6cfcf7fba028de870eca676ac Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:26:51 +0000 Subject: [PATCH 059/177] Fix msgtype display --- templating/matrix_templates/sections.py | 2 +- templating/matrix_templates/units.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 81b2bb3c..78aabca7 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -136,7 +136,7 @@ class MatrixSections(Sections): if not event_name.startswith("m.room.message#m."): continue sections.append(template.render( - example=examples[event_name], + example=examples[event_name][0], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 5ae117ca..94435c52 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -562,7 +562,7 @@ class MatrixUnits(Units): event_name = filename.split("#")[0] example = json.loads(f.read()) - examples[filename] = examples.get(event_name, []) + examples[filename] = examples.get(filename, []) examples[filename].append(example) if filename != event_name: examples[event_name] = examples.get(event_name, []) From 27ffe7bacd15ae559cdfc7bce03a42d00699d7d4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:34:32 +0000 Subject: [PATCH 060/177] Don't serve rst diffs as HTML --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 7f86bd62..e1898d7e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -385,7 +385,7 @@ func main() { } s := server{masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) - http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", forceHTML(listPulls)) From d7357ef9b75893631e4a8b1e5e40273885cd6709 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:39:40 +0000 Subject: [PATCH 061/177] Specify /publicRooms --- api/client-server/v1/list_public_rooms.yaml | 85 +++++++++++++++++++ specification/client_server_api.rst | 5 ++ .../matrix_templates/templates/http-api.tmpl | 5 +- 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 api/client-server/v1/list_public_rooms.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml new file mode 100644 index 00000000..2f189d2e --- /dev/null +++ b/api/client-server/v1/list_public_rooms.yaml @@ -0,0 +1,85 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +paths: + "/publicRooms": + get: + summary: Lists the public rooms on the server. + description: |- + Lists the public rooms on the server. + + This API returns paginated responses. + responses: + 200: + description: A list of the rooms on the server. + schema: + type: object + description: A list of the rooms on the server. + properties: + chunk: + title: "PublicRoomsChunks" + type: array + description: |- + A paginated chunk of public rooms. + items: + type: object + title: "PublicRoomsChunk" + properties: + aliases: + type: array + description: |- + Aliases of the room. May be empty. + items: + type: string + name: + type: string + description: |- + The name of the room, if any. May be null. + num_joined_members: + type: number + description: |- + The number of members joined to the room. + room_id: + type: string + description: |- + The ID of the room. + topic: + type: string + description: |- + The topic of the room, if any. May be null. + start: + type: string + description: |- + A pagination token for the response. + end: + type: string + description: |- + A pagination token for the response. + examples: + application/json: |- + { + "chunk": [ + { + "aliases": ["#murrays:cheese.bar"], + "name": "CHEESE", + "num_joined_members": 37, + "room_id": "!ol19s:bleecker.street", + "topic": "Tasty tasty cheese" + } + ], + "start": "p190q", + "end": "p1902" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index b02dbf28..05b6ff3c 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -931,6 +931,11 @@ member's state, by making a request to "membership": "ban" } +Listing rooms +~~~~~~~~~~~~~ + +{{list_public_rooms_http_api}} + Profiles -------- diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 86eacb14..d7258e98 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -12,7 +12,7 @@ {{":Requires auth: Yes." if endpoint.requires_auth else "" }} Request format: - +{% if (endpoint.req_param_by_loc | length) %} =========================================== ================= =========================================== Parameter Value Description =========================================== ================= =========================================== @@ -24,6 +24,9 @@ Request format: {% endfor -%} {% endfor -%} =========================================== ================= =========================================== +{% else %} +`No parameters` +{% endif %} {% if endpoint.res_tables|length > 0 -%} Response format: From dcf54e11b10b07fd30cd18be127d9cde9b198e8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:53:31 +0000 Subject: [PATCH 062/177] Specify /publicRooms world_readable and guest_access Depends on https://github.com/matrix-org/matrix-doc/pull/154 --- api/client-server/v1/list_public_rooms.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml index 2f189d2e..23b1c3d0 100644 --- a/api/client-server/v1/list_public_rooms.yaml +++ b/api/client-server/v1/list_public_rooms.yaml @@ -57,6 +57,16 @@ paths: type: string description: |- The topic of the room, if any. May be null. + world_readable: + type: boolean + description: |- + Whether the room may be viewed by guest users without joining. + guest_can_join: + type: boolean + description: |- + Whether guest users may join the room and participate in it. + If they can, they will be subject to ordinary power level + rules like any other user. start: type: string description: |- @@ -71,10 +81,12 @@ paths: "chunk": [ { "aliases": ["#murrays:cheese.bar"], + "guest_can_join": false, "name": "CHEESE", "num_joined_members": 37, "room_id": "!ol19s:bleecker.street", - "topic": "Tasty tasty cheese" + "topic": "Tasty tasty cheese", + "world_readable": true } ], "start": "p190q", From 740cc66a7cccc366ca34eac1ebf65fd7c9af90b3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 14:01:45 +0000 Subject: [PATCH 063/177] speculator: Fetch before deciding head is fresh --- scripts/speculator/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e1898d7e..e467018e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -178,6 +178,10 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + if err := gitFetch(s.matrixDocCloneURL); err != nil { + writeError(w, 500, err) + return + } originHead, err := s.getSHAOf("origin/master") if err != nil { writeError(w, 500, err) From 0f0359d9c1049c60dbea365344b40cadc1a50c16 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 15:13:34 +0000 Subject: [PATCH 064/177] speculator: Nessle up some more if statements --- scripts/speculator/main.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e467018e..97e67c8c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -74,8 +74,7 @@ func gitClone(url string, shared bool) (string, error) { cmd.Args = append(cmd.Args, "--shared") } - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error cloning repo: %v", err) } return directory, nil @@ -92,8 +91,7 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) } return nil @@ -126,8 +124,7 @@ func generate(dir string) error { cmd.Dir = path.Join(dir, "scripts") var b bytes.Buffer cmd.Stderr = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return nil @@ -167,8 +164,7 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil From 061105c9dc25d52a01ba260afc9c90bd0158931b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 17:18:58 +0000 Subject: [PATCH 065/177] Guest users are allowed room initialSync --- api/client-server/v1/rooms.yaml | 2 +- specification/modules/guest_access.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d1..90b51290 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -371,7 +371,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - required: ["room_id", "membership"] + required: ["room_id"] 403: description: > You aren't a member of the room and weren't previously a diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af9..ae8d156e 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -32,6 +32,7 @@ retrieving events: * `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ * `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ * `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ +* `GET /rooms/:room_id/initialSync <#get-matrix-client-api-v1-rooms-roomid-initialsync>`_ There is also a special version of the `GET /events <#get-matrix-client-api-v1-events>`_ endpoint: From 4d3175fc8b5c01a41afb802c2e4bc0e191b822db Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 10:45:57 +0000 Subject: [PATCH 066/177] Note that invite_room_state is optional --- event-schemas/schema/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 81057049..25b30109 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], From 29d9c8eec6a0a7da49b78c4885b168662fc04f99 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 12:05:02 +0000 Subject: [PATCH 067/177] Guests are allowed to set displaynames --- specification/modules/guest_access.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af9..d7359f23 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -51,6 +51,11 @@ sending events: Guest clients *do* need to join rooms in order to send events to them. +The following API endpoints are allowed to be accessed by guest accounts for +their own account maintenance: + +* `PUT /profile/:user_id/displayname <#put-matrix-client-api-v1-profile-userid-displayname>`_ + Server behaviour ---------------- Servers are required to only return events to guest accounts for rooms where From d7c69fae438410e1b91030e60466f48f28e27360 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:33:34 +0000 Subject: [PATCH 068/177] Fix typo 'process' --- specification/server_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d26cc26e..1c3e761d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -555,8 +555,8 @@ sign, and emit a new ``m.room.member`` state event adding the user into that room. When the homeserver does not yet know about the room it cannot do this directly. Instead, it must take a longer multi-stage handshaking process by which it first selects a remote homeserver which is already participating in -that room, and uses it to assist in the joining procss. This is the remote join -handshake. +that room, and uses it to assist in the joining process. This is the remote +join handshake. This handshake involves the homeserver of the new member wishing to join (referred to here as the "joining" server), the directory server hosting the From aac45295ee9dc615cfc7290cc928c011ca0d4e79 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:34:22 +0000 Subject: [PATCH 069/177] Remark that the directory server step could be skipped in an invite case --- specification/server_server_api.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 1c3e761d..e158ae23 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -596,9 +596,12 @@ homeservers, though most in practice will use just two. | <---------- join response -The first part of the handshake involves using the directory server to request -the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. +The first part of the handshake usually involves using the directory server to +request the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver could +optimise this step away by picking the origin server of that invite message as +the join candidate. Once the joining server has the room ID and the join candidates, it then needs to obtain enough of the current state of the room to fill in the required From db5a90edcd7fcc33385d2145b2ad1ed93313182f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:36:26 +0000 Subject: [PATCH 070/177] Avoid using the words 'current state' when talking about the result of the /make_join request --- specification/server_server_api.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e158ae23..6dfbd86f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -566,9 +566,8 @@ room members are already present (referred to as the "resident" server). In summary, the remote join handshake consists of the joining server querying the directory server for information about the room alias; receiving a room ID and a list of join candidates. The joining server then requests information -about the current state of the room from one of the residents. It uses this -information to construct a ``m.room.member`` event which it finally sends to -a resident server. +about the room from one of the residents. It uses this information to construct +a ``m.room.member`` event which it finally sends to a resident server. Conceptually these are three different roles of homeserver. In practice the directory server is likely to be resident in the room, and so may be selected @@ -604,11 +603,11 @@ optimise this step away by picking the origin server of that invite message as the join candidate. Once the joining server has the room ID and the join candidates, it then needs -to obtain enough of the current state of the room to fill in the required -fields of the ``m.room.member`` event. It obtains this by selecting a resident -from the candidate list, and requesting the ``make_join`` endpoint using a -``GET`` request, specifying the room ID and the user ID of the new member who -is attempting to join. +to obtain enough information about the room to fill in the required fields of +the ``m.room.member`` event. It obtains this by selecting a resident from the +candidate list, and requesting the ``make_join`` endpoint using a ``GET`` +request, specifying the room ID and the user ID of the new member who is +attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some From 885dd1e86cc8d6ee818ab32ab246eeec9a1f5ede Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:39:31 +0000 Subject: [PATCH 071/177] Explain the 'prev_events' join protoevent key --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6dfbd86f..ebf55eb4 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -629,7 +629,8 @@ the assisting resident. The required fields are: resident ``origin`` String The name of the assisting resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver -``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_events`` List An event-reference list containing the immediate + predecessor events ``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member From 988d77347651ce2504ab1e52196166358dffe7f7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 17:49:22 +0000 Subject: [PATCH 072/177] /make_join protoevent no longer needs the pointless 'prev_state' key (SYN-517) --- specification/server_server_api.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index ebf55eb4..9cc10141 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -631,7 +631,6 @@ the assisting resident. The required fields are: ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events -``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member ``state_key`` String The user ID of the joining member @@ -688,8 +687,6 @@ integer 200, and whose second element contains the following keys: - (paul) I don't really understand why the full auth_chain events are given here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? - - (paul) the state seems to be entirely ignored by synapse, so I'm not really - sure what ought to be there. Backfilling ----------- From cc8ef691fb20331d135a39993e843d42588e0761 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 16:49:22 +0000 Subject: [PATCH 073/177] Rewrite the table templates Allow columns to stretch if they end up with wide content. Apart from the hassle of having to manually update the calculations, having the columns wide enough to hold the widest thing they might ever have leads to tables with lots of whitespace in the results. --- templating/build.py | 22 ++++ .../templates/common-event-fields.tmpl | 15 +-- .../matrix_templates/templates/events.tmpl | 15 +-- .../matrix_templates/templates/http-api.tmpl | 27 +---- .../matrix_templates/templates/msgtypes.tmpl | 15 +-- .../matrix_templates/templates/tables.tmpl | 104 ++++++++++++++++++ 6 files changed, 139 insertions(+), 59 deletions(-) create mode 100644 templating/matrix_templates/templates/tables.tmpl diff --git a/templating/build.py b/templating/build.py index a35d8a08..77fecf91 100755 --- a/templating/build.py +++ b/templating/build.py @@ -93,6 +93,27 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return '\n\n'.join(output_lines) + def fieldwidths(input, keys, defaults=[], default_width=15): + """ + A template filter to help in the generation of tables. + + Given a list of rows, returns a list giving the maximum length of the + values in each column. + + :param list[dict[str, str]] input: a list of rows. Each row should be a + dict with the keys given in ``keys``. + :param list[str] keys: the keys corresponding to the table columns + :param list[int] defaults: for each column, the default column width. + :param int default_width: if ``defaults`` is shorter than ``keys``, this + will be used as a fallback + """ + def colwidth(key, default): + return reduce(max, (len(row[key]) for row in input), + default if default is not None else default_width) + + results = map(colwidth, keys, defaults) + return results + # make Jinja aware of the templates and filters env = Environment( loader=FileSystemLoader(in_mod.exports["templates"]), @@ -102,6 +123,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): env.filters["indent"] = indent env.filters["indent_block"] = indent_block env.filters["wrap"] = wrap + env.filters["fieldwidths"] = fieldwidths # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 3f16be3d..8d8c8f0c 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,17 +1,8 @@ +{% import 'tables.tmpl' as tables -%} + {{common_event.title}} Fields {{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} -================== ================= =========================================== - Key Type Description -================== ================= =========================================== -{% for row in common_event.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc | indent(18 - (row.type|length)) |wrap(43) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440..324a8f3e 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.type}}`` {{(4 + event.type | length) * title_kind}} *{{event.typeof}}* @@ -7,18 +9,7 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ================= =========================================== - {{table.title or "Content"}} Key Type Description -======================= ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} -{% endfor -%} -======================= ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98..1253e66e 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} {% if "alias_for_path" in endpoint -%} @@ -13,17 +15,7 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== -{% for loc in endpoint.req_param_by_loc -%} -*{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- -{% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} -{% endfor -%} -{% endfor -%} -=========================================== ================= =========================================== +{{ tables.split_paramtable(endpoint.req_param_by_loc) }} {% else %} `No parameters` {% endif %} @@ -34,18 +26,7 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ========================= ========================================== - Param Type Description -======================= ========================= ========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (42 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} -{% endfor -%} -======================= ========================= ========================================== +{{ tables.paramtable(table.rows) }} {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 18d3492b..87cf4a19 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,21 +1,12 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.msgtype}}`` {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/tables.tmpl b/templating/matrix_templates/templates/tables.tmpl new file mode 100644 index 00000000..aba6c0c4 --- /dev/null +++ b/templating/matrix_templates/templates/tables.tmpl @@ -0,0 +1,104 @@ +{# + # A set of macros for generating RST tables + #} + + +{# + # write a table for a list of parameters. + # + # 'rows' is the list of parameters. Each row should have the keys + # 'key', 'type', and 'desc'. + #} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} +{{ split_paramtable({None: rows}, titles) }} +{% endmacro %} + + +{# + # write a table for the request parameters, split by location. + # 'rows_by_loc' is a map from location to a list of parameters. + # + # As a special case, if a key of 'rows_by_loc' is 'None', no title row is + # written for that location. This is used by the standard 'paramtable' macro. + #} +{% macro split_paramtable(rows_by_loc, + titles=["Parameter", "Type", "Description"]) -%} + +{% set rowkeys = ['key', 'type', 'desc'] %} +{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %} + +{# We need the rows flattened into a single list. Abuse the 'sum' filter to + # join arrays instead of add numbers. -#} +{% set flatrows = rows_by_loc.values()|sum(start=[]) -%} + +{# Figure out the widths of the columns. The last column is always 50 characters + # wide; the others default to 10, but stretch if there is wider text in the + # column. -#} +{% set fieldwidths = (([titlerow] + flatrows) | + fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} + +{{ tableheader(fieldwidths) }} +{{ tablerow(fieldwidths, titlerow, rowkeys) }} +{{ tableheader(fieldwidths) }} +{% for loc in rows_by_loc -%} + +{% if loc != None -%} +{{ tablespan(fieldwidths, "*" ~ loc ~ " parameters*") }} +{% endif -%} + +{% for row in rows_by_loc[loc] -%} +{{ tablerow(fieldwidths, row, rowkeys) }} +{% endfor -%} +{% endfor -%} + +{{ tableheader(fieldwidths) }} +{% endmacro %} + + + +{# + # Write a table header row, for the given column widths + #} +{% macro tableheader(widths) -%} +{% for arg in widths -%} +{{"="*arg}} {% endfor -%} +{% endmacro %} + + + +{# + # Write a normal table row. Each of 'widths' and 'keys' should be sequences + # of the same length; 'widths' defines the column widths, and 'keys' the + # attributes of 'row' to look up for values to put in the columns. + #} +{% macro tablerow(widths, row, keys) -%} +{% for key in keys -%} +{% set value=row[key] -%} +{% if not loop.last -%} + {# the first few columns need space after them -#} + {{ value }}{{" "*(1+widths[loop.index0]-value|length) -}} +{% else -%} + {# the last column needs wrapping and indenting (by the sum of the widths of + the preceding columns, plus the number of preceding columns (for the + separators)) -#} + {{ value | wrap(widths[loop.index0]) | + indent_block(widths[0:-1]|sum + loop.index0) -}} +{% endif -%} +{% endfor -%} +{% endmacro %} + + + + +{# + # write a tablespan row. This is a single value which spans the entire table. + #} +{% macro tablespan(widths, value) -%} +{{value}} +{# we write a trailing space to stop the separator being misinterpreted + # as a header line. -#} +{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} + + + + From 838af2a23ec623b2a7f2bd18ac0af51a097dd2b5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 17:21:56 +0000 Subject: [PATCH 074/177] Updates to swagger table generation A bunch of related fixes to the code for parsing the state and API yaml files: 1. Some of our objects are {key: {key: value}} - style nested key/value dictionaries. Handle this by refactoring get_json_schema_object_fields so that such objects are handled wherever they appear, rather than when they are just subproperties of a 'proper' object. 2. Fix multi-level inheritance (so an object can have an 'allOf' property which can successfully refer to an object which itself has an 'allOf' property). 3. $ref fields in event schemas weren't being expanded correctly 4. sort type tables breadth-first rather than depth-first so that the ordering in complex structures like the /sync response makes a bit more sense. --- templating/build.py | 4 + templating/matrix_templates/units.py | 200 +++++++++++++++++---------- 2 files changed, 130 insertions(+), 74 deletions(-) diff --git a/templating/build.py b/templating/build.py index a35d8a08..eea6ac7a 100755 --- a/templating/build.py +++ b/templating/build.py @@ -42,6 +42,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, met from argparse import ArgumentParser, FileType import importlib import json +import logging import os import sys from textwrap import TextWrapper @@ -188,6 +189,9 @@ if __name__ == '__main__': ) args = parser.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + if not args.input: raise Exception("Missing [i]nput python module.") diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427..5acbcd56 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -8,6 +8,7 @@ For the actual conversion of data -> RST (including templates), see the sections file instead. """ from batesian.units import Units +import logging import inspect import json import os @@ -27,6 +28,7 @@ TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" +logger = logging.getLogger(__name__) def resolve_references(path, schema): if isinstance(schema, dict): @@ -46,6 +48,32 @@ def resolve_references(path, schema): return schema +def inherit_parents(obj): + """ + Recurse through the 'allOf' declarations in the object + """ + logger.debug("inherit_parents %r" % obj) + parents = obj.get("allOf", []) + if not parents: + return obj + + result = {} + + # settings defined in the child take priority over the parents, so we + # iterate through the parents first, and then overwrite with the settings + # from the child. + for p in map(inherit_parents, parents) + [obj]: + for key in ('title', 'type', 'required'): + if p.get(key): + result[key] = p[key] + + for key in ('properties', 'additionalProperties', 'patternProperties'): + if p.get(key): + result.setdefault(key, {}).update(p[key]) + + return result + + def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -53,22 +81,44 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals raise Exception( "get_json_schema_object_fields: Object %s isn't an object." % obj ) + + obj = inherit_parents(obj) + + logger.debug("Processing object with title '%s'", obj.get("title")) + if enforce_title and not obj.get("title"): # Force a default titile of "NO_TITLE" to make it obvious in the # specification output which parts of the schema are missing a title obj["title"] = 'NO_TITLE' - required_keys = obj.get("required") - if not required_keys: - required_keys = [] + additionalProps = obj.get("additionalProperties") + if additionalProps: + # not "really" an object, just a KV store + logger.debug("%s is a pseudo-object", obj.get("title")) + + key_type = additionalProps.get("x-pattern", "string") + + value_type = additionalProps["type"] + if value_type == "object": + nested_objects = get_json_schema_object_fields( + additionalProps, + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + tables = [x for x in nested_objects if not x.get("no-table")] + else: + key_type = "string" + tables = [] - fields = { - "title": obj.get("title"), - "rows": [] - } - tables = [fields] + tables = [{ + "title": "{%s: %s}" % (key_type, value_type), + "no-table": True + }]+tables + + logger.debug("%s done: returning %s", obj.get("title"), tables) + return tables - parents = obj.get("allOf") props = obj.get("properties") if not props: props = obj.get("patternProperties") @@ -79,73 +129,60 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if pretty_key: props[pretty_key] = props[key_name] del props[key_name] - if not props and not parents: - # Sometimes you just want to specify that a thing is an object without - # doing all the keys. Allow people to do that if they set a 'title'. - if obj.get("title"): - parents = [{ - "$ref": obj.get("title") - }] - if not props and not parents: - raise Exception( - "Object %s has no properties or parents." % obj - ) - if not props: # parents only - if include_parents: - if obj["title"] == "NO_TITLE" and parents[0].get("title"): - obj["title"] = parents[0].get("title") - props = parents[0].get("properties") - if not props: + # Sometimes you just want to specify that a thing is an object without + # doing all the keys. Allow people to do that if they set a 'title'. + if not props and obj.get("title"): return [{ "title": obj["title"], - "parent": parents[0].get("$ref"), "no-table": True }] + if not props: + raise Exception( + "Object %s has no properties and no title" % obj + ) + + required_keys = set(obj.get("required", [])) + + fields = { + "title": obj.get("title"), + "rows": [] + } + + tables = [fields] + for key_name in sorted(props): + logger.debug("Processing property %s.%s", obj.get('title'), key_name) value_type = None required = key_name in required_keys desc = props[key_name].get("description", "") - - if props[key_name]["type"] == "object": - if props[key_name].get("additionalProperties"): - # not "really" an object, just a KV store - prop_val = props[key_name]["additionalProperties"]["type"] - if prop_val == "object": - nested_object = get_json_schema_object_fields( - props[key_name]["additionalProperties"], - enforce_title=True, - include_parents=include_parents, - ) - key = props[key_name]["additionalProperties"].get( - "x-pattern", "string" - ) - value_type = "{%s: %s}" % (key, nested_object[0]["title"]) - if not nested_object[0].get("no-table"): - tables += nested_object - else: - value_type = "{string: %s}" % prop_val - else: - nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True, - include_parents=include_parents, - ) - value_type = "{%s}" % nested_object[0]["title"] - - if not nested_object[0].get("no-table"): - tables += nested_object - elif props[key_name]["type"] == "array": + prop_type = props[key_name].get('type') + + if prop_type is None: + raise KeyError("Property '%s' of object '%s' missing 'type' field" + % (key_name, obj)) + logger.debug("%s is a %s", key_name, prop_type) + + if prop_type == "object": + nested_objects = get_json_schema_object_fields( + props[key_name], + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + + tables += [x for x in nested_objects if not x.get("no-table")] + elif prop_type == "array": # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": - nested_object = get_json_schema_object_fields( + nested_objects = get_json_schema_object_fields( props[key_name]["items"], enforce_title=True, include_parents=include_parents, ) - value_type = "[%s]" % nested_object[0]["title"] - tables += nested_object + value_type = "[%s]" % nested_objects[0]["title"] + tables += nested_objects else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): @@ -163,7 +200,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals " Must be '%s'." % array_enums[0] ) else: - value_type = props[key_name]["type"] + value_type = prop_type if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -188,15 +225,32 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) + logger.debug("Done property %s" % key_name) + + return tables + + +def get_tables_for_schema(path, schema, include_parents=False): + resolved_schema = resolve_references(path, schema) + tables = get_json_schema_object_fields(resolved_schema, + include_parents=include_parents, + ) + + # the result may contain duplicates, if objects are referred to more than + # once. Filter them out. + # + # Go through the tables backwards so that we end up with a breadth-first + # rather than depth-first ordering. titles = set() filtered = [] - for table in tables: + for table in reversed(tables): if table.get("title") in titles: continue titles.add(table.get("title")) filtered.append(table) + filtered.reverse() return filtered @@ -306,16 +360,14 @@ class MatrixUnits(Units): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields( - resolve_references(filepath, req_obj), - include_parents=True, - ) + req_tables = get_tables_for_schema( + filepath, req_obj, include_parents=True) if req_tables > 1: for table in req_tables[1:]: nested_key_name = [ s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) + s["type"] == ("%s" % (table["title"],)) ][0] for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) @@ -431,8 +483,7 @@ class MatrixUnits(Units): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields( - resolve_references(filepath, schema), + res_tables = get_tables_for_schema(filepath, schema, include_parents=True, ) for table in res_tables: @@ -571,8 +622,9 @@ class MatrixUnits(Units): for filename in os.listdir(path): if not filename.startswith("m."): continue - self.log("Reading %s" % os.path.join(path, filename)) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + self.log("Reading %s" % filepath) + with open(filepath, "r") as f: json_schema = json.loads(f.read()) schema = { "typeof": None, @@ -614,15 +666,15 @@ class MatrixUnits(Units): schema["desc"] = json_schema.get("description", "") # walk the object for field info - schema["content_fields"] = get_json_schema_object_fields( + schema["content_fields"] = get_tables_for_schema(filepath, Units.prop(json_schema, "properties/content") ) # This is horrible because we're special casing a key on m.room.member. # We need to do this because we want to document a non-content object. if schema["type"] == "m.room.member": - invite_room_state = get_json_schema_object_fields( - json_schema["properties"]["invite_room_state"]["items"] + invite_room_state = get_tables_for_schema(filepath, + json_schema["properties"]["invite_room_state"]["items"], ) schema["content_fields"].extend(invite_room_state) From 3f0262081c7a900cd1e5bcd7ca9a2991facb6ef9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 11 Nov 2015 19:22:38 +0000 Subject: [PATCH 075/177] Update sync API defn to reflect SPEC-254 changes State now corresponds to the start of the timeline, not the end. --- api/client-server/v2_alpha/sync.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index a2d5a2b8..9e921610 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -119,7 +119,11 @@ paths: title: State type: object description: |- - The state updates for the room. + Updates to the state, between the time indicated by + the ``since`` parameter, and the start of the + ``timeline`` (or all state up to the start of the + ``timeline``, if ``since`` is not given, or + ``full_state`` is true). allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -193,8 +197,7 @@ paths: title: State type: object description: |- - The state updates for the room up to the point when - the user left. + The state updates for the room up to the start of the timeline. allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -260,7 +263,6 @@ paths: "state": { "events": [ "$66697273743031:example.com", - "$7365636s6r6432:example.com" ] }, "timeline": { From 57995a815a83ec7496283eae5511ebd89ddeb747 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:30:29 +0000 Subject: [PATCH 076/177] Fix a number of untruths in the documentation about /sync. Attempts to make the /sync documentation better represent fact as it currently stands - in particular document the structure of the returned events. --- api/client-server/v2_alpha/core-event-schema | 1 - .../v2_alpha/definitions/event.json | 53 +++++++++++++++++++ .../v2_alpha/definitions/event_batch.json | 4 +- .../v2_alpha/definitions/timeline_batch.json | 4 +- api/client-server/v2_alpha/sync.yaml | 16 +++--- specification/events.rst | 5 +- 6 files changed, 68 insertions(+), 15 deletions(-) delete mode 120000 api/client-server/v2_alpha/core-event-schema create mode 100644 api/client-server/v2_alpha/definitions/event.json diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema deleted file mode 120000 index b020e6da..00000000 --- a/api/client-server/v2_alpha/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json new file mode 100644 index 00000000..6a01c688 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "title": "Event", + "properties": { + "content": { + "type": "object", + "title": "EventContent", + "description": "The content of this event. The fields in this object will vary depending on the type of event." + }, + "origin_server_ts": { + "type": "integer", + "format": "int64", + "description": "Timestamp in milliseconds on originating homeserver when this event was sent." + }, + "sender": { + "type": "string", + "description": "The MXID of the user who sent this event." + }, + "state_key": { + "type": "string", + "description": "Optional. This key will only be present for state events. A unique key which defines the overwriting semantics for this piece of room state." + }, + "type": { + "type": "string", + "description": "The type of event." + }, + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 395aed13..7f489423 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,8 +5,8 @@ "type": "array", "description": "List of events", "items": { - "title": "Event", - "type": "object" + "type": "object", + "allOf": [{"$ref": "event.json" }] } } } diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index ddf8d341..15ca1ead 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -4,11 +4,11 @@ "properties": { "limited": { "type": "boolean", - "description": "Whether there are more events on the server" + "description": "True if the number of events returned was limited by the ``limit`` on the filter" }, "prev_batch": { "type": "string", - "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve earlier events" } } } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 9e921610..06763a08 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -96,8 +96,10 @@ paths: Updates to rooms. properties: joined: - title: Joined + title: Joined Rooms type: object + description: |- + The rooms that the user has joined. additionalProperties: title: Joined Room type: object @@ -114,7 +116,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -144,7 +146,7 @@ paths: allOf: - $ref: "definitions/event_batch.json" invited: - title: Invited + title: Invited Rooms type: object description: |- The rooms that the user has been invited to. @@ -171,11 +173,10 @@ paths: allOf: - $ref: "definitions/event_batch.json" archived: - title: Archived + title: Archived rooms type: object description: |- - The rooms that the user has left or been banned from. The - entries in the room_map will lack an ``ephemeral`` key. + The rooms that the user has left or been banned from. additionalProperties: title: Archived Room type: object @@ -192,7 +193,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -276,7 +277,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } diff --git a/specification/events.rst b/specification/events.rst index a1aece1c..7bf08012 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -30,12 +30,13 @@ formatted for federation by: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, ``origin``, ``prev_state``. * Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has ellapsed since the event was sent. + milliseconds that has elapsed since the event was sent. * Adding a ``prev_content`` to the ``unsigned`` object if the event is a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``transaction_id`` if the event was sent by the client requesting it. +* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the + client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From b41d771c1586a06e387222030f923ed5fc95ecf0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:58:04 +0000 Subject: [PATCH 077/177] Fix typos in /sync example response --- api/client-server/v2_alpha/sync.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 06763a08..2290b070 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, + "unsigned": {"age": 124524, "txn_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" @@ -263,7 +263,7 @@ paths: }, "state": { "events": [ - "$66697273743031:example.com", + "$66697273743031:example.com" ] }, "timeline": { From 29bd4d45ee9155bd80e61ce50ab34745be1fd8a4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:40:41 +0000 Subject: [PATCH 078/177] Flatten the response to /sync Now that we don't expect there to be duplication betwen the 'timeline' and 'state' results, there's no point in having the state map. (That does mean the events themselves need event_id fields though). Also: - move the contents of the 'unsigned' dictionary into the events themselves - replace the state list with two layers of dictionary keyed on type and state_key - rename the children of the 'rooms' dict from "joined/invited/archived" to "join/invite/leave" to match the membership states --- .../v2_alpha/definitions/event.json | 45 +++--- .../definitions/room_event_batch.json | 12 -- .../v2_alpha/definitions/timeline_batch.json | 2 +- api/client-server/v2_alpha/sync.yaml | 134 +++++++----------- specification/events.rst | 17 +-- 5 files changed, 84 insertions(+), 126 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/room_event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c688..3a15357a 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,16 +2,34 @@ "type": "object", "title": "Event", "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, + "event_id": { + "type": "string", + "description": "Globally unique identifier for this event." + }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "prev_sender": { + "type": "string", + "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." + }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -24,30 +42,9 @@ "type": "string", "description": "The type of event." }, - "unsigned": { - "type": "object", - "title": "Unsigned", - "description": "Information about this event which was not sent by the originating homeserver", - "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "replaces_state": { - "type": "string", - "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." - } - } + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json deleted file mode 100644 index fcf148f3..00000000 --- a/api/client-server/v2_alpha/definitions/room_event_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "events": { - "type": "array", - "description": "List of event ids", - "items": { - "type": "string" - } - } - } -} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index 15ca1ead..f27a5746 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "allOf": [{"$ref":"definitions/event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b070..c63f3432 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -95,7 +95,7 @@ paths: description: |- Updates to rooms. properties: - joined: + join: title: Joined Rooms type: object description: |- @@ -104,19 +104,6 @@ paths: title: Joined Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object @@ -127,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -145,7 +132,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - invited: + invite: title: Invited Rooms type: object description: |- @@ -171,36 +158,23 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/event_batch.json" - archived: - title: Archived rooms + - $ref: "definitions/state_batch.json" + leave: + title: Left rooms type: object description: |- The rooms that the user has left or been banned from. additionalProperties: - title: Archived Room + title: Left Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -230,46 +204,43 @@ paths: ] }, "rooms": { - "joined": { + "join": { "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "unsigned": { - "prev_content": {"membership": "invite"} - }, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, - "content": { - "body": "I am a fish", - "msgtype": "m.text" - }, - "origin_server_ts": 1417731086797 - } - }, "state": { - "events": [ - "$66697273743031:example.com" - ] + "m.room.member": { + "@alice:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795, + "event_id": "$66697273743031:example.com" + } + } }, "timeline": { "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" + { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "prev_content": {"membership": "invite"}, + "origin_server_ts": 1417731086795, + "event_id": "$7365636s6r6432:example.com": + }, + { + "sender": "@alice:example.com", + "type": "m.room.message", + "age": 124524, + "txn_id": "1234", + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797, + "event_id": "$74686972643033:example.com" + } ], "limited": true, "prev_batch": "t34-23535_0_0" @@ -277,6 +248,7 @@ paths: "ephemeral": { "events": [ { + "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -284,26 +256,30 @@ paths: } } }, - "invited": { + "invite": { "!696r7674:example.com": { "invite_state": { - "events": [ - { + "m.room.name": { + "": { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"} - }, - { + "content": {"name": "My Room Name"}, + "event_id": "$asdkgjrsfg2314375:example.com", + + } + }, + "m.room.member": { + "@bob:example.com": { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"} - } - ] + "content": {"membership": "invite"}, + "event_id": "$257kasjdg315324akhg:example.com", + } } } }, - "archived": {} + "leave": {} } } diff --git a/specification/events.rst b/specification/events.rst index 7bf08012..3d069be8 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,22 +28,19 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``. -* Adding an ``age`` to the ``unsigned`` object which gives the time in + ``origin``, ``prev_state``, ``unsigned``. +* Adding an ``age`` to the event object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding a ``prev_content`` to the ``unsigned`` object if the event is - a ``state event`` which gives previous content of that state key. -* Adding a ``redacted_because`` to the ``unsigned`` object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the event object if the event + is a ``state event``, which give the previous content and previous sender of + that state key +* Adding a ``redacted_because`` to event object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``txn_id`` if the event was sent by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: -* Moving the folling keys from the ``unsigned`` object to the top level event - object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. -* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From e1b12a753ecd96227effef3612f70d9adf48c0a9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 00:17:22 +0000 Subject: [PATCH 079/177] Fix typos and missing file --- .../v2_alpha/definitions/state_batch.json | 12 ++++++++++++ api/client-server/v2_alpha/sync.yaml | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json new file mode 100644 index 00000000..45728f40 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/state_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "additionalProperties": { + "type": "object", + "x-pattern": "$EVENT_TYPE", + "additionalProperties": { + "type": "object", + "x-pattern": "$STATE_KEY", + "allOf": [{"$ref": "event.json" }] + } + } +} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c63f3432..198f4833 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -227,7 +227,7 @@ paths: "content": {"membership": "join"}, "prev_content": {"membership": "invite"}, "origin_server_ts": 1417731086795, - "event_id": "$7365636s6r6432:example.com": + "event_id": "$7365636s6r6432:example.com" }, { "sender": "@alice:example.com", @@ -265,8 +265,7 @@ paths: "type": "m.room.name", "state_key": "", "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com", - + "event_id": "$asdkgjrsfg2314375:example.com" } }, "m.room.member": { @@ -275,7 +274,8 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com", + "event_id": "$257kasjdg315324akhg:example.com" + } } } } From 96be7ff24140ec80937db48da300c6f4e05473e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Nov 2015 15:00:31 +0000 Subject: [PATCH 080/177] Support more nesting --- templating/matrix_templates/units.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427..e5a7c319 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -122,10 +122,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "x-pattern", "string" ) value_type = "{%s: %s}" % (key, nested_object[0]["title"]) + value_id = "%s: %s" % (key, nested_object[0]["title"]) if not nested_object[0].get("no-table"): tables += nested_object else: - value_type = "{string: %s}" % prop_val + value_type = "{string: %s}" % (prop_val,) + value_id = "string: %s" % (prop_val,) else: nested_object = get_json_schema_object_fields( props[key_name], @@ -133,6 +135,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) if not nested_object[0].get("no-table"): tables += nested_object @@ -145,12 +148,14 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) tables += nested_object else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): value_type = " or ".join(value_type) value_type = "[%s]" % value_type + value_id = "%s" % (value_type,) array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: @@ -164,6 +169,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals ) else: value_type = props[key_name]["type"] + value_id = props[key_name]["type"] if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -184,6 +190,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals fields["rows"].append({ "key": key_name, "type": value_type, + "id": value_id, "required": required, "desc": desc, "req_str": "**Required.** " if required else "" @@ -313,10 +320,16 @@ class MatrixUnits(Units): if req_tables > 1: for table in req_tables[1:]: - nested_key_name = [ - s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) - ][0] + nested_key_name = { + "key": s["key"] + for rtable in req_tables + for s in rtable["rows"] + if s["id"] == table["title"] + }.get("key", None) + + if nested_key_name is None: + raise Exception("Failed to find table for %r" % (table["title"],)) + for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) From 8648f86032bea6c7576fa92a4bd8ea8e0fdd9793 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Nov 2015 15:08:37 +0000 Subject: [PATCH 081/177] Moar spaces --- templating/matrix_templates/templates/http-api.tmpl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98..4a9491f3 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -13,17 +13,17 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== +=============================================================== ================= =========================================== + Parameter Value Description +=============================================================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- +----------------------------------------------------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} +{{param.key}}{{param.type|indent(64-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(82)}} {% endfor -%} {% endfor -%} -=========================================== ================= =========================================== +=============================================================== ================= =========================================== {% else %} `No parameters` {% endif %} From 36af793f05992d0bef5ef325c303a753f16f24e0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:30:22 +0000 Subject: [PATCH 082/177] s/full object/full event/ --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 9cc10141..127a3572 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -612,7 +612,7 @@ attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this -object is not a full object; notably it does not need to be hashed or signed by +object is not a full event; notably it does not need to be hashed or signed by the assisting resident. The required fields are: ==================== ======== ============ From 923f05e5541f354c2003acf77dbe4e2c3d6f90ec Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:34:11 +0000 Subject: [PATCH 083/177] More consistency around 'resident homeserver' --- specification/server_server_api.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 127a3572..4c315edc 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -613,7 +613,7 @@ The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this object is not a full event; notably it does not need to be hashed or signed by -the assisting resident. The required fields are: +the resident homeserver. The required fields are: ==================== ======== ============ Key Type Description @@ -625,9 +625,9 @@ the assisting resident. The required fields are: ``content`` Object The event content ``depth`` Integer (this field must be present but is ignored; it may be 0) -``event_id`` String A new event ID specified by the assisting - resident -``origin`` String The name of the assisting resident homeserver +``event_id`` String A new event ID specified by the resident + homeserver +``origin`` String The name of the resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events @@ -665,12 +665,12 @@ algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` fields. To complete the join handshake, the joining server must now submit this new -event to an assisting resident, by using the ``send_join`` endpoint. This is +event to an resident homeserver, by using the ``send_join`` endpoint. This is invoked using the room ID and the event ID of the new member event. -The assisting resident then accepts this event into the room's event graph, and -responds to the joining server with the full set of state for the newly-joined -room. This is returned as a two-element list, whose first element is the +The resident homeserver then accepts this event into the room's event graph, +and responds to the joining server with the full set of state for the newly- +joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: ============== ===== ============ @@ -919,6 +919,6 @@ Querying directory information:: The list of join candidates is a list of server names that are likely to hold the given room; these are servers that the requesting server may wish to use as -assisting resident servers as part of the remote join handshake. This list may -or may not include the server answering the query. +resident servers as part of the remote join handshake. This list may or may not +include the server answering the query. From 122c082fcf0ef14cb93e09d1b8d2b7469866c786 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:29:43 +0000 Subject: [PATCH 084/177] Comment about origin servers of invites having subsequently left the room --- specification/server_server_api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4c315edc..20026890 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -600,7 +600,9 @@ request the room ID and join candidates. This is covered in more detail on the directory server documentation, below. In the case of a new user joining a room as a result of a received invite, the joining user's homeserver could optimise this step away by picking the origin server of that invite message as -the join candidate. +the join candidate. However, the joining server should be aware that the origin +server of the invite might since have left the room, so should be prepared to +fall back on the regular join flow if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of From 6cbfba70113e302985c4f93af181141e7b0c84f1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:33:26 +0000 Subject: [PATCH 085/177] 'auth_events' is a List, not a String --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 20026890..6c66b3d9 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -621,7 +621,7 @@ the resident homeserver. The required fields are: Key Type Description ==================== ======== ============ ``type`` String The value ``m.room.member`` -``auth_events`` String An event-reference list containing the +``auth_events`` List An event-reference list containing the authorization events that would allow this member to join ``content`` Object The event content From 769c5285ab75243d247afd2b6bdb806224d0309d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:33:57 +0000 Subject: [PATCH 086/177] Add API for setting client config --- api/client-server/v2_alpha/client-config.yaml | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 api/client-server/v2_alpha/client-config.yaml diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml new file mode 100644 index 00000000..91be1686 --- /dev/null +++ b/api/client-server/v2_alpha/client-config.yaml @@ -0,0 +1,105 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server Client Config API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client. This config is only visible to the user + that set the config. The config will be synced to clients in the + top-level ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. + "/user/{userId}/rooms/{roomId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client on a given room. This config is only + visible to the user that set the config. The config will be synced to + clients in the per-room ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to set config on. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.room.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. From 22b3159a3983a98762961ff5374deb3eff6c6d99 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:34:39 +0000 Subject: [PATCH 087/177] Add examples of v1 initialSync and v2 /sync returning the client config --- api/client-server/v1/sync.yaml | 24 ++++++++++++++++++++---- api/client-server/v2_alpha/sync.yaml | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a9..57a15583 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,6 +131,14 @@ paths: "type": "m.presence" } ], + "private_user_data": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ], "rooms": [ { "membership": "join", @@ -246,10 +254,18 @@ paths: } ], "visibility": "private", - "private_user_data": [{ - "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} - }] + "private_user_data": [ + { + "type": "m.tag", + "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] } ] } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 4c1a7601..e180c5ff 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -215,6 +215,16 @@ paths: } ] }, + "private_user_data": { + "events": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] + }, "rooms": { "joined": { "!726s6s6q:example.com": { @@ -271,6 +281,12 @@ paths: { "type": "m.tags", "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } } ] } From c3769ef75ca039ad2bfe2532a17ad979c631c71c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:49:32 +0000 Subject: [PATCH 088/177] Add a module for client config to the specification --- specification/modules/client_config.rst | 26 +++++++++++++++++++++++++ specification/targets.yaml | 1 + 2 files changed, 27 insertions(+) create mode 100644 specification/modules/client_config.rst diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst new file mode 100644 index 00000000..837f174d --- /dev/null +++ b/specification/modules/client_config.rst @@ -0,0 +1,26 @@ +Client Config +============= + +.. _module:client_config: + +Clients can store their config on their homeserver. This config will be synced +between different devices and can persist across installations on a particular +device. + +The config may be either global or scoped to a particular rooms. + +Events +------ + +The client recieves the config as a event in the ``private_user_data`` sections +of a v2 /sync. + +These events can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. + +Client Behaviour +---------------- + +{{v2_client_config_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 2c6a8f20..b6ac6a65 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -26,6 +26,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/third_party_invites.rst - modules/search.rst - modules/tags.rst + - modules/client_config.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 83168813939ea1dc4c5a778781cb12e1f8eb9d87 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:50:54 +0000 Subject: [PATCH 089/177] txn_id field in events is called transaction_id --- api/client-server/v2_alpha/definitions/event.json | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c688..5a8f52f6 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -43,7 +43,7 @@ "type": "string", "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." }, - "txn_id": { + "transaction_id": { "type": "string", "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b070..d23929df 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, + "unsigned": {"age": 124524, "transaction_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" From 233e8486bc41e4ad87fc8e06a57bbcd4b8579974 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:50:58 +0000 Subject: [PATCH 090/177] Wording fix - objects contain keys, not list elements directly --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6c66b3d9..cc9426e2 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -673,7 +673,8 @@ invoked using the room ID and the event ID of the new member event. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the newly- joined room. This is returned as a two-element list, whose first element is the -integer 200, and whose second element contains the following keys: +integer 200, and whose second element is an object which contains the +following keys: ============== ===== ============ Key Type Description From 0db055b4ea22f4d29064728eaaaf2c37a2f9d926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:54:57 +0000 Subject: [PATCH 091/177] Fix another reference to 'txn_id' --- specification/events.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/events.rst b/specification/events.rst index 7bf08012..9d0be55f 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -35,8 +35,8 @@ formatted for federation by: a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From 299af673da3d4a698dc943aaabe3c2e961f2da49 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:04:19 +0000 Subject: [PATCH 092/177] Specify how ordering of tags is supposed to work --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- api/client-server/v2_alpha/tags.yaml | 4 ++-- specification/modules/tags.rst | 5 +++++ 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 1cce76f3..bafcb98b 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a9..8050ede3 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index ec34c085..7a597f9e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -291,7 +291,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} } ] } diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 009e6000..91945dca 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -55,7 +55,7 @@ paths: application/json: |- { "tags": { - "work": {"order": 1}, + "work": {"order": "1"}, "pinned": {} } } @@ -97,7 +97,7 @@ paths: schema: type: object example: |- - {"order": 1} + {"order": "1"} responses: 200: description: diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 3d45975c..15982a0a 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -21,6 +21,11 @@ the tags are for. Each tag has an associated JSON object with information about the tag, e.g how to order the rooms with a given tag. +Ordering information is given under the ``order`` key as a string. The string +are compared lexicographically by unicode codepoint to determine which should +displayed first. So a tag with an ``order`` key of ``"apples"`` would appear +before a tag with an ``order`` key of ``"oranges"``. + {{m_tag_event}} Client Behaviour From 48f35e15cba959e28aeb897e3339e5d6dfbfddc7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:08:59 +0000 Subject: [PATCH 093/177] describe how to order rooms that don't have an order in their tags --- specification/modules/tags.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 15982a0a..56d216e7 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -23,10 +23,13 @@ to order the rooms with a given tag. Ordering information is given under the ``order`` key as a string. The string are compared lexicographically by unicode codepoint to determine which should -displayed first. So a tag with an ``order`` key of ``"apples"`` would appear -before a tag with an ``order`` key of ``"oranges"``. +displayed first. So a room with a tag with an ``order`` key of ``"apples"`` +would appear before a room with a tag with an ``order`` key of ``"oranges"``. +If a room has a tag without an ``order`` key then it should appear after the +rooms with that tag that have an ``order`` key. -{{m_tag_event}} + +{{m_tag_event} Client Behaviour ---------------- From e7fbe6f13b12797817eed767cd7306498dcddcec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:13:02 +0000 Subject: [PATCH 094/177] Limit the size of a tag --- specification/modules/tags.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 56d216e7..4d88c771 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -28,6 +28,7 @@ would appear before a room with a tag with an ``order`` key of ``"oranges"``. If a room has a tag without an ``order`` key then it should appear after the rooms with that tag that have an ``order`` key. +The name of a tag MUST not exceed 255 bytes. {{m_tag_event} From 25769493b11cd95ef73b5cfe936971a8bc7565ea Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:14:41 +0000 Subject: [PATCH 095/177] Fix template --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 4d88c771..5fe6345f 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,7 +30,7 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -{{m_tag_event} +{{m_tag_event}} Client Behaviour ---------------- From c77b22778fab74d7bae2ebc25b1d635d17e8afb3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:48:41 +0000 Subject: [PATCH 096/177] Add some documentation on names of tags --- specification/modules/tags.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 5fe6345f..681d33d7 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,6 +30,16 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. +The name of a tag should be human readable. When displaying tags for a room a +client should display this human readable name. When adding a tag for a room +a client may offer a list to choose from that includes all the tags that the +user has previously set on any of their rooms. + +Two special names are listed in the specification: + +* ``m.favourite`` +* ``m.lowpriority`` + {{m_tag_event}} Client Behaviour From 03a0377c76861eb1c9af503b718b31508dcdd7dc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 20:07:42 +0000 Subject: [PATCH 097/177] Plans for end-to-end in matrix --- drafts/markjh_end_to_end.rst | 142 +++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 drafts/markjh_end_to_end.rst diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst new file mode 100644 index 00000000..1699f79b --- /dev/null +++ b/drafts/markjh_end_to_end.rst @@ -0,0 +1,142 @@ +Goals of Key-Distribution in Matrix +=================================== + +* No Central Authority: Users should not need to trust a central authority + when determining the authenticity of keys. + +* Easy to Add New Devices: It should be easy for a user to start using a + new device. + +* Possible to discover MITM: It should be possible for a user to determine if + they are being MITM. + +* Lost Devices: It should be possible for a user to recover if they lose all + their devices. + +* No Copying Keys: Keys should be per device and shouldn't leave the device + they were created on. + +A Possible Mechanism for Key Distribution +========================================= + +Basic API for setting up keys on a server: + +https://github.com/matrix-org/matrix-doc/pull/24 + +Client shouldn't trust the keys unless they have been verified, e.g by +comparing fingerprints. + +If a user adds a new device it should some yet to be specified protocol +communicate with an old device and obtain a cross-signature from the old +device for its public key. + +The new device can then present the cross-signed key to all the devices +that the user is in conversations with. Those devices should then include +the new device into those conversations. + +If the user cannot cross-sign the new key, e.g. because their old device +is lost or stolen. Then they will need to reauthenticate their conversations +out of band, e.g by comparing fingerprints. + + +Goals of End-to-end encryption in Matrix +======================================== + +* Access to Chat History: Users should be able to see the history of a + conversation on a new device. User should be able to control who can + see their chat history and how much of the chat history they can see. + +* Forward Secrecy of Discarded Chat History: Users should be able to discard + history from their device, once they have discarded the history it should be + impossible for an adversary to recover that history. + +* Forward Secrecy of Future Messages: Users should be able to recover from + disclosure of the chat history on their device. + +* Deniablity of Chat History: It should not be possible to prove to a third + party that a given user sent a message. + +* Authenticity of Chat History: It should be possible to prove amoungst + the members of a chat that a message sent by a user was authored by that + user. + + +Bonus Goals: + +* Traffic Analysis: It would be nice if the protocol was resilient to traffic + or metadata analysis. However it's not something we want to persue if it + harms the usability of the protocol. It might be cool if there was a + way for the user to could specify the trade off between performance and + resilience to traffic analysis that they wanted. + + +A Possible Design for Group Chat using Olm +========================================== + +Protecting the secrecy of history +--------------------------------- + +Each message sent by a client has a 32-bit counter. This counter increments +by one for each message sent by the client. This counter is used to advance a +ratchet. The ratchet is split into a vector four 256-bit values, +:math:`R_{n,j}` for :math:`j \in {0,1,2,3}`. The ratchet can be advanced as +follows: + +.. math:: + \begin{align} + R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ + R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_1\left(R_{(i-1),3}\right) + \end{align} + +Where :math:`H_1` and :math:`H_2` are different hash functions. For example +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and +:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. + +So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. +Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. +Every :math:`2^8` iterations :math:`R_{n,3}` is reseeded from :math:`R_{n,2}`. + +This scheme allows the ratchet to be advanced an arbitrary amount forwards +while needing only 1024 hash computations. + +This the value of the ratchet is hashed to generate the keys used to encrypt +each mesage. + +A client can decrypt chat history onwards from the earliest value of the +ratchet it is aware of. But cannot decrypt history from before that point +without reversing the hash function. + +This allows a client to share its ability to decrypt chat history with another +from a point in the conversation onwards by giving a copy of the ratchet at +that point in the conversation. + +A client can discard history by advancing a ratchet to beyond the last message +they want to discard and then forgetting all previous values of the ratchet. + +Proving and denying the authenticity of history +----------------------------------------------- + +Client sign the messages they send using a Ed25519 key generated per +conversation. That key, along with the ratchet key, is distributed +to other clients using 1:1 olm ratchets. Those 1:1 ratchets are started using +Triple Diffie-Hellman which provides authenticity of the messages to the +participants and deniability of the messages to third parties. Therefore +any keys shared over those keys inherit the same levels of deniability and +authenticity. + +Protecting the secrecy of future messages +----------------------------------------- + +A client would need to generate new keys if it wanted to prevent access to +messages beyond a given point in the conversation. It must generate new keys +whenever someone leaves the room. It should generate new keys periodically +anyway. + +The frequency of key generation in a large room may need to be restricted to +keep the frequency of messages broadcast over the individual 1:1 channels +low. From 8201eaa042875feb69389f14262356fd3cfa73ed Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 15:31:10 +0000 Subject: [PATCH 098/177] Swaggerify /rooms/:room_id/leave --- .../v1/{membership.yaml => inviting.yaml} | 51 +------------- api/client-server/v1/joining.yaml | 68 +++++++++++++++++++ api/client-server/v1/leaving.yaml | 57 ++++++++++++++++ specification/client_server_api.rst | 32 ++------- 4 files changed, 132 insertions(+), 76 deletions(-) rename api/client-server/v1/{membership.yaml => inviting.yaml} (61%) create mode 100644 api/client-server/v1/joining.yaml create mode 100644 api/client-server/v1/leaving.yaml diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/inviting.yaml similarity index 61% rename from api/client-server/v1/membership.yaml rename to api/client-server/v1/inviting.yaml index c089e154..0a028710 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/inviting.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - title: "Matrix Client-Server v1 Room Membership API" + title: "Matrix Client-Server v1 Room Joining API" version: "1.0.0" host: localhost:8008 schemes: @@ -18,55 +18,6 @@ securityDefinitions: name: access_token in: query paths: - "/rooms/{roomId}/join": - post: - summary: Start the requesting user participating in a particular room. - description: |- - This API starts a user participating in a particular room, if that user - is allowed to participate in that room. After this call, the client is - allowed to see all current state events in the room, and all subsequent - events associated with the room until the user leaves the room. - - After a user has joined a room, the room will appear as an entry in the - response of the |initialSync| API. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier or room alias to join. - required: true - x-example: "#monkeys:matrix.org" - responses: - 200: - description: |- - The room has been joined. - - The joined room ID must be returned in the ``room_id`` field. - examples: - application/json: |- - {"room_id": "!d41d8cd:matrix.org"} - schema: - type: object - 403: - description: |- - You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - - The room is invite-only and the user was not invited. - - The user has been banned from the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" - x-alias: - canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" - aliases: - - /join/{roomId} - # With an extra " " to disambiguate from the 3pid invite endpoint # The extra space makes it sort first for what I'm sure is a good reason. "/rooms/{roomId}/invite ": diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml new file mode 100644 index 00000000..20fdbd65 --- /dev/null +++ b/api/client-server/v1/joining.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Inviting API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml new file mode 100644 index 00000000..376b2060 --- /dev/null +++ b/api/client-server/v1/leaving.yaml @@ -0,0 +1,57 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Leaving API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/leave": + post: + summary: Stop the requesting user participating in a particular room. + description: |- + This API stops a user participating in a particular room. + + If the user was already in the room, they will no longer be able to see + new events in the room. If the room requires an invite to join, they + will need to be re-invited before they can re-join. + + If the user was invited to the room, but had not joined, this call + serves to reject the invite. + + The user will still be allowed to retrieve history from the room which + they were previously allowed to see. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to leave. + required: true + x-example: "!nkl290a:matrix.org" + responses: + 200: + description: |- + The room has been left. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 05b6ff3c..ba5578bc 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -870,42 +870,22 @@ following values: ``invite`` This room can only be joined if you were invited. -{{membership_http_api}} +{{inviting_http_api}} + +{{joining_http_api}} Leaving rooms ~~~~~~~~~~~~~ -.. TODO-spec - HS deleting rooms they are no longer a part of. Not implemented. - - This is actually Very Tricky. If all clients a HS is serving leave a room, - the HS will no longer get any new events for that room, because the servers - who get the events are determined on the *membership list*. There should - probably be a way for a HS to lurk on a room even if there are 0 of their - members in the room. - - Grace period before deletion? - - Under what conditions should a room NOT be purged? - - A user can leave a room to stop receiving events for that room. A user must have been invited to or have joined the room before they are eligible to leave the room. Leaving a room to which the user has been invited rejects the invite. +Once a user leaves a room, it will no longer appear on the |initialSync|_ API. Whether or not they actually joined the room, if the room is an "invite-only" room they will need to be re-invited before they can re-join -the room. To leave a room, a request should be made to -|/rooms//leave|_ with:: - - {} - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "leave" - } +the room. -See the `Room events`_ section for more information on ``m.room.member``. Once a -user has left a room, that room will no longer appear on the |initialSync|_ API. -If all members in a room leave, that room becomes eligible for deletion. +{{leaving_http_api}} Banning users in a room ~~~~~~~~~~~~~~~~~~~~~~~ From 6763317e64645945224b7c68ac385ed055fe80c3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 10:33:46 -0500 Subject: [PATCH 099/177] Specify /rooms/:room_id/forget --- api/client-server/v1/leaving.yaml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml index 376b2060..e81d812b 100644 --- a/api/client-server/v1/leaving.yaml +++ b/api/client-server/v1/leaving.yaml @@ -55,3 +55,38 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" + "/rooms/{roomId}/forget": + post: + summary: Stop the requesting user remembering about a particular room. + description: |- + This API stops a user remembering about a particular room. + + In general, history is a first class citizen in Matrix. After this API + is called, however, a user will no longer be able to retrieve history + for this room. If all users on a homeserver forget a room, the room is + eligible for deletion from that homeserver. + + If the user is currently joined to the room, they will implicitly leave + the room as part of this API call. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to forget. + required: true + x-example: "!au1ba7o:matrix.org" + responses: + 200: + description: |- + The room has been forgotten. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" From 6653362f315d1379d1dda248dfd299c8c3c8530d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 15:15:21 +0000 Subject: [PATCH 100/177] Unflatten 'unsigned' It turns out that flattening 'unsigned' comes with too many downsides. Let's stick with the status quo. --- .../v2_alpha/definitions/event.json | 45 ++++++++++--------- specification/events.rst | 19 ++++---- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 3a15357a..5a8f52f6 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,34 +2,16 @@ "type": "object", "title": "Event", "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, - "event_id": { - "type": "string", - "description": "Globally unique identifier for this event." - }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "prev_sender": { - "type": "string", - "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." - }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -42,9 +24,30 @@ "type": "string", "description": "The type of event." }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "transaction_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } } } } diff --git a/specification/events.rst b/specification/events.rst index eb48ca17..5a003115 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,20 +28,23 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``, ``unsigned``. -* Adding an ``age`` to the event object which gives the time in + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding ``prev_content`` and ``prev_sender`` to the event object if the event - is a ``state event``, which give the previous content and previous sender of - that state key -* Adding a ``redacted_because`` to event object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the + event is a ``state event``, which give the previous content and previous + sender of that state key +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the event object if the event was sent by the client - requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From fcbb985073e82368894a41285dab86feb53daa21 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:27:26 +0000 Subject: [PATCH 101/177] s/private_user_data/account_data/ --- api/client-server/v1/rooms.yaml | 4 ++-- api/client-server/v1/sync.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 4 ++-- specification/modules/tags.rst | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index bafcb98b..ff5587a9 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -312,7 +312,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -375,7 +375,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to this room. diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8050ede3..973ccc6e 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -246,7 +246,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -336,7 +336,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7a597f9e..28358f55 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -139,7 +139,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - private_user_data: + account_data: title: Private User Data type: object description: |- @@ -287,7 +287,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "m.tags", diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 681d33d7..bb467bc0 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -11,10 +11,10 @@ Events ------ The tags on a room are received as single ``m.tag`` event in the -``private_user_data`` section of a room in a v2 /sync. +``account_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From 40f7eab73f1a3055c3c4874b8d122f6b9c57e167 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:44:18 +0000 Subject: [PATCH 102/177] s/private_user_data/account_data/ --- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/client-config.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 2 +- specification/modules/client_config.rst | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b8608171..58cd5544 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,7 +131,7 @@ paths: "type": "m.presence" } ], - "private_user_data": [ + "account_data": [ { "type": "org.example.custom.config", "content": { diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml index 91be1686..76b6219e 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/client-config.yaml @@ -24,7 +24,7 @@ paths: description: |- Set some config for the client. This config is only visible to the user that set the config. The config will be synced to clients in the - top-level ``private_user_data``. + top-level ``account_data``. security: - accessToken: [] parameters: @@ -63,7 +63,7 @@ paths: description: |- Set some config for the client on a given room. This config is only visible to the user that set the config. The config will be synced to - clients in the per-room ``private_user_data``. + clients in the per-room ``account_data``. security: - accessToken: [] parameters: diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index e2016d68..f5c99508 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -233,7 +233,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "org.example.custom.config", diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst index 837f174d..581a5828 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/client_config.rst @@ -12,11 +12,11 @@ The config may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``private_user_data`` sections +The client recieves the config as a event in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From d7d59d78e1dca06407dea20ad875f64569844ad3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 16:17:29 +0000 Subject: [PATCH 103/177] /sync: Put state dict back to being a list Turning the state into a dict-of-dicts caused more pain than it solved. Put it back to a list. --- .../v2_alpha/definitions/state_batch.json | 12 ------- api/client-server/v2_alpha/sync.yaml | 31 ++++++++----------- 2 files changed, 13 insertions(+), 30 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json deleted file mode 100644 index 45728f40..00000000 --- a/api/client-server/v2_alpha/definitions/state_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "additionalProperties": { - "type": "object", - "x-pattern": "$EVENT_TYPE", - "additionalProperties": { - "type": "object", - "x-pattern": "$STATE_KEY", - "allOf": [{"$ref": "event.json" }] - } - } -} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 198f4833..0c9d6186 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -114,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -158,7 +158,7 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" leave: title: Left rooms type: object @@ -174,7 +174,7 @@ paths: description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -207,8 +207,8 @@ paths: "join": { "!726s6s6q:example.com": { "state": { - "m.room.member": { - "@alice:example.com": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@alice:example.com", @@ -216,7 +216,7 @@ paths: "origin_server_ts": 1417731086795, "event_id": "$66697273743031:example.com" } - } + ] }, "timeline": { "events": [ @@ -248,7 +248,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -259,24 +258,20 @@ paths: "invite": { "!696r7674:example.com": { "invite_state": { - "m.room.name": { - "": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com" - } - }, - "m.room.member": { - "@bob:example.com": { + "content": {"name": "My Room Name"} + }, + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com" + "content": {"membership": "invite"} } - } + ] } } }, From 5aeaa42a5039bd0435f66be0a9787cb3efc24985 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:41:25 -0500 Subject: [PATCH 104/177] Changelog for 0.3.0 --- CHANGELOG.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e3198ef..4916c771 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,35 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.3.0 (2015-11-18) +============================================ + +Many sections have been filled in, or added to reflect long-standing status +quos. Several API endpoints have also been converted to Swagger. Several +presentation improvements (e.g. syntax highlighting) have also been added. + +Additions: + - Extensive narrative on instant messaging + - Documentation of event sending + - Documentation of state event retrieval + - Redaction algorithm + - Algorithm for calculating display names for rooms and users + - New endpoint: ``/publicRooms`` + - New event: ``m.room.avatar`` + - invite_room_state on ``m.room.member`` events + - New section: Third party invitations + - Ability to reject invitations + - New API: Search + - Guest access to rooms + - Descriptions of server-server API endpoints for joining rooms + - Sample expected crypto output + - Several security warnings + + Modifications: + - Fixed errors in parameters of /login + - Significant clarification and improvements to description of pagination + - Changes to v2_alpha/sync API (Note: this API is still not stable) + Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -38,4 +67,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file +- Restructure the format of the Client-Server HTTP APIs. From 1a1a7d87dcd1f8ada30592cae7b4da86b8b2468f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:53:20 -0500 Subject: [PATCH 105/177] Revert "Changelog for 0.3.0" This reverts commit 5aeaa42a5039bd0435f66be0a9787cb3efc24985. --- CHANGELOG.rst | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4916c771..6e3198ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,35 +6,6 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. -Specification changes in v0.3.0 (2015-11-18) -============================================ - -Many sections have been filled in, or added to reflect long-standing status -quos. Several API endpoints have also been converted to Swagger. Several -presentation improvements (e.g. syntax highlighting) have also been added. - -Additions: - - Extensive narrative on instant messaging - - Documentation of event sending - - Documentation of state event retrieval - - Redaction algorithm - - Algorithm for calculating display names for rooms and users - - New endpoint: ``/publicRooms`` - - New event: ``m.room.avatar`` - - invite_room_state on ``m.room.member`` events - - New section: Third party invitations - - Ability to reject invitations - - New API: Search - - Guest access to rooms - - Descriptions of server-server API endpoints for joining rooms - - Sample expected crypto output - - Several security warnings - - Modifications: - - Fixed errors in parameters of /login - - Significant clarification and improvements to description of pagination - - Changes to v2_alpha/sync API (Note: this API is still not stable) - Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -67,4 +38,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. +- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file From 05c00926641a7a1156b2f239efc153b3aa39e9eb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:36:27 +0000 Subject: [PATCH 106/177] s/config/account_data/ --- .../{client-config.yaml => account-data.yaml} | 44 +++++++++---------- .../{client_config.rst => account_data.rst} | 4 +- specification/targets.yaml | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) rename api/client-server/v2_alpha/{client-config.yaml => account-data.yaml} (60%) rename specification/modules/{client_config.rst => account_data.rst} (91%) diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/account-data.yaml similarity index 60% rename from api/client-server/v2_alpha/client-config.yaml rename to api/client-server/v2_alpha/account-data.yaml index 76b6219e..98134d6f 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -18,12 +18,12 @@ securityDefinitions: name: access_token in: query paths: - "/user/{userId}/config/{configType}": + "/user/{userId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client. This config is only visible to the user - that set the config. The config will be synced to clients in the + Set some account_data for the client. This config is only visible to the user + that set the account_data. The config will be synced to clients in the top-level ``account_data``. security: - accessToken: [] @@ -33,36 +33,36 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to + The type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_config_value"} responses: 200: description: - The config was successfully added. - "/user/{userId}/rooms/{roomId}/config/{configType}": + The account_data was successfully added. + "/user/{userId}/rooms/{roomId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client on a given room. This config is only - visible to the user that set the config. The config will be synced to + Set some account_data for the client on a given room. This config is only + visible to the user that set the account_data. The config will be synced to clients in the per-room ``account_data``. security: - accessToken: [] @@ -72,7 +72,7 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path @@ -80,26 +80,26 @@ paths: name: roomId required: true description: |- - The id of the room to set config on. + The id of the room to set account_data on. x-example: "!726s6s6q:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to - avoid clashes. + The type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_account_data_value"} responses: 200: description: - The config was successfully added. + The account_data was successfully added. diff --git a/specification/modules/client_config.rst b/specification/modules/account_data.rst similarity index 91% rename from specification/modules/client_config.rst rename to specification/modules/account_data.rst index 581a5828..fc0e2e68 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/account_data.rst @@ -1,7 +1,7 @@ Client Config ============= -.. _module:client_config: +.. _module:account_data: Clients can store their config on their homeserver. This config will be synced between different devices and can persist across installations on a particular @@ -23,4 +23,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_client_config_http_api}} +{{v2_account_data_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 418515c3..afb7f8d9 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -27,7 +27,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/search.rst - modules/guest_access.rst - modules/tags.rst - - modules/client_config.rst + - modules/account_data.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 032ee75537fd8d67db0e7e930ada4cea6d428faf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:42:22 +0000 Subject: [PATCH 107/177] Update specification wording to match s/config/account_data/ --- specification/modules/account_data.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index fc0e2e68..784bc874 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -3,16 +3,17 @@ Client Config .. _module:account_data: -Clients can store their config on their homeserver. This config will be synced -between different devices and can persist across installations on a particular -device. +Clients can store custom config data for their account on their homeserver. +This account data will be synced between different devices and can persist +across installations on a particular device. Users may only view the account +data for their own account -The config may be either global or scoped to a particular rooms. +The account_data may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``account_data`` sections +The client recieves the account data as events in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the From 9ad64b02d183e72b6548d7a36f9b96949f624e70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 15:41:15 -0500 Subject: [PATCH 108/177] speculator: guard against concurrent git commands --- scripts/speculator/main.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 97e67c8c..4472164c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -136,17 +137,26 @@ func writeError(w http.ResponseWriter, code int, err error) { } type server struct { + mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL matrixDocCloneURL string } +func (s *server) updateBase() error { + s.mu.Lock() + defer s.mu.Unlock() + return gitFetch(s.matrixDocCloneURL) +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = s.updateBase() if err != nil { return } + s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) + s.mu.Unlock() if err != nil { return } @@ -164,7 +174,10 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - if err := cmd.Run(); err != nil { + s.mu.Lock() + err := cmd.Run() + s.mu.Unlock() + if err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil @@ -174,7 +187,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := gitFetch(s.matrixDocCloneURL); err != nil { + if err := s.updateBase(); err != nil { writeError(w, 500, err) return } @@ -383,7 +396,7 @@ func main() { if err != nil { log.Fatal(err) } - s := server{masterCloneDir} + s := server{matrixDocCloneURL: masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) From 757b0bcd12269b0a9905efee7520e61abc81ec7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:08:37 -0500 Subject: [PATCH 109/177] Try to build continuserv and speculator --- jenkins.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins.sh b/jenkins.sh index 0b217e58..b2aa8489 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -7,3 +7,8 @@ set -ex (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) + +if which go >/dev/null 2>/dev/null; then + (cd scripts/continuserv && go build) + (cd scripts/speculator && go build) +fi From dd53847211c57553180efca9d843c1c8741d6de3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:11:19 -0500 Subject: [PATCH 110/177] Include command stderr in error text --- scripts/speculator/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4472164c..00a59eea 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -70,13 +70,15 @@ const ( func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, directory) + if err := os.MkdirAll(directory, 0755); err != nil { + return "", fmt.Errorf("error making directory %s: %v", directory, err) + } + args := []string{"clone", url, directory} if shared { - cmd.Args = append(cmd.Args, "--shared") + args = append(args, "--shared") } - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("error cloning repo: %v", err) + if err := runGitCommand(directory, args); err != nil { + return "", err } return directory, nil } @@ -92,8 +94,10 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path + var b bytes.Buffer + cmd.Stderr = &b if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String()) } return nil } From 8872e17f9355a47f9b7a2822cfd1303d8e56e2bc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:14:22 -0500 Subject: [PATCH 111/177] Fall back to last known HEAD sha if fetch fails --- scripts/speculator/main.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 00a59eea..380cd2bc 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -87,10 +87,6 @@ func gitCheckout(path, sha string) error { return runGitCommand(path, []string{"checkout", sha}) } -func gitFetch(path string) error { - return runGitCommand(path, []string{"fetch"}) -} - func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path @@ -148,7 +144,7 @@ type server struct { func (s *server) updateBase() error { s.mu.Lock() defer s.mu.Unlock() - return gitFetch(s.matrixDocCloneURL) + return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } // generateAt generates spec from repo at sha. @@ -191,16 +187,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := s.updateBase(); err != nil { + if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return + } else { + sha = headSha } - originHead, err := s.getSHAOf("origin/master") - if err != nil { - writeError(w, 500, err) - return - } - sha = originHead } else { pr, err := lookupPullRequest(*req.URL, "/spec") if err != nil { @@ -237,6 +229,25 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { specCache.Add(sha, b) } +// lookupHeadSHA looks up what origin/master's HEAD SHA is. +// It attempts to `git fetch` before doing so. +// If this fails, it may still return a stale sha, but will also return an error. +func (s *server) lookupHeadSHA() (sha string, retErr error) { + retErr = s.updateBase() + if retErr != nil { + log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr) + } + originHead, err := s.getSHAOf("origin/master") + if err != nil { + retErr = err + } + sha = originHead + if retErr != nil && originHead != "" { + log.Printf("Successfully fell back to possibly stale sha: %s", sha) + } + return +} + func checkAuth(pr *PullRequest) error { if !pr.User.IsTrusted() { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login) From 6f1d00097be403998e5490845835ecb23d2c741c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:15:13 -0500 Subject: [PATCH 112/177] Only bother trying to fetch if we need to --- scripts/speculator/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 380cd2bc..4972210e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -147,12 +147,20 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } +func (s *server) knowsAbout(sha string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = s.updateBase() - if err != nil { - return + if !s.knowsAbout(sha) { + err = s.updateBase() + if err != nil { + return + } } s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) From 858674477111bfc80adb299ad73715a98099c296 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:41:58 -0500 Subject: [PATCH 113/177] Add anchors to spec This is currently done by a script on the prod serving machine. We might as well keep the matrix.org spec and dev spec as similar as possible. --- scripts/gendoc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed36a5a7..28a11528 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -238,6 +238,18 @@ def rst2html(i, o): ) +def addAnchors(path): + with open(path, "r") as f: + lines = f.readlines() + + replacement = replacement = r'

\n\1' + with open(path, "w") as f: + for line in lines: + line = re.sub(r'()', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write(line + "\n") + + def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: @@ -387,6 +399,7 @@ def main(target_name, keep_intermediates): shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") + addAnchors("gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() From e0410330484257e6d0c8da481d8d7e31a2b05053 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:17:50 -0500 Subject: [PATCH 114/177] Rename file --- scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} (100%) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/add-matrix-org-stylings.sh similarity index 100% rename from scripts/matrix-org-gendoc.sh rename to scripts/add-matrix-org-stylings.sh From ca3a9e3562492964e314d7f9173d320281fbddfd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:18:33 -0500 Subject: [PATCH 115/177] exec gendoc outside of script --- scripts/add-matrix-org-stylings.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 5716786d..b02cb306 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -35,8 +35,6 @@ if [ ! -f $FOOTER ]; then exit 1 fi -python gendoc.py - perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header #' gen/specification.html gen/howtos.html From da93317a7874096c2c13974542e2f2272e5d2a60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:13:40 -0500 Subject: [PATCH 116/177] Take dir not files as args --- scripts/add-matrix-org-stylings.sh | 43 +++++++++--------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index b02cb306..83888346 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -1,39 +1,20 @@ -#! /bin/bash +#!/bin/bash -eu -if [ -z "$1" ]; then - echo "Expected /includes/head.html file as 1st arg." - exit 1 -fi - -if [ -z "$2" ]; then - echo "Expected /includes/nav.html file as 2nd arg." - exit 1 +if [[ $# != 1 || ! -d $1 ]]; then + echo >&2 "Usage: $0 include_dir" + exit 1 fi -if [ -z "$3" ]; then - echo "Expected /includes/footer.html file as 3rd arg." - exit 1 -fi - - -HEADER=$1 -NAV_BAR=$2 -FOOTER=$3 +HEADER="$1/head.html" +NAV_BAR="$1/nav.html" +FOOTER="$1/footer.html" -if [ ! -f $HEADER ]; then - echo $HEADER " does not exist" +for f in "${HEADER}"/{head,nav,footer}.html; do + if [[ ! -e "${f}" ]]; then + echo >&2 "Need ${f} to exist" exit 1 -fi - -if [ ! -f $NAV_BAR ]; then - echo $NAV_BAR " does not exist" - exit 1 -fi - -if [ ! -f $FOOTER ]; then - echo $FOOTER " does not exist" - exit 1 -fi + fi +done perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header From 4ac85997f52303b5fa75241f9bc50cdc227af3fa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:16:02 -0500 Subject: [PATCH 117/177] Fix check --- scripts/add-matrix-org-stylings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 83888346..ef4e1014 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -9,7 +9,7 @@ HEADER="$1/head.html" NAV_BAR="$1/nav.html" FOOTER="$1/footer.html" -for f in "${HEADER}"/{head,nav,footer}.html; do +for f in "$1"/{head,nav,footer}.html; do if [[ ! -e "${f}" ]]; then echo >&2 "Need ${f} to exist" exit 1 From 181d3f976d4f236c34ba0e86f58f729690d82e34 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 13:01:20 +0000 Subject: [PATCH 118/177] Initial proposal for websockets support. --- drafts/websockets.rst | 285 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 drafts/websockets.rst diff --git a/drafts/websockets.rst b/drafts/websockets.rst new file mode 100644 index 00000000..bd0ff081 --- /dev/null +++ b/drafts/websockets.rst @@ -0,0 +1,285 @@ +WebSockets API +============== + +Introduction +------------ +This document is a proposal for a WebSockets-based client-server API. It is not +intended to replace the REST API, but rather to complement it and provide an +alternative interface for certain operations. + +The primary goal is to offer a more efficient interface than the REST API: by +using a bidirectional protocol such as WebSockets we can avoid the overheads +involved in long-polling (SSL negotiation, HTTP headers, etc). In doing so we +will reduce the latency between server and client by allowing the server to +send events as soon as they arrive, rather than having to wait for a poll from +the client. + +Handshake +--------- +1. Instead of calling ``/sync``, the client makes a websocket request to + ``/_matrix/client/rN/stream``, passing the query parameters ``access_token`` + and ``since``, and optionally ``filter`` - all of which have the same + meaning as for ``/sync``. + + * The client sets the ``Sec-WebSocket-Protocol`` to ``m.json``. (Servers may + offer alternative encodings; at present only the JSON encoding is + specified but in future we will specify alternative encodings.) + +#. The server returns the websocket handshake; the socket is then connected. + +If the server does not return a valid websocket handshake, this indicates that +the server or an intermediate proxy does not support WebSockets. In this case, +the client should fall back to polling the ``/sync`` REST endpoint. + +Example +~~~~~~~ + +Client request: + +.. code:: http + + GET /_matrix/client/v2_alpha/stream?access_token=123456&since=s72594_4483_1934 HTTP/1.1 + Host: matrix.org + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: m.json + Sec-WebSocket-Version: 13 + Origin: https://matrix.org + +Server response: + +.. code:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: m.json + + +Update Notifications +-------------------- +Once the socket is connected, the server begins streaming updates over the +websocket. The server sends Update notifications about new messages or state +changes. To make it easy for clients to parse, Update notifications have the +same structure as the response to ``/sync``: an object with the following +members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +next_batch string The batch token to supply in the ``since`` param of + the next /sync request. This is not required for + streaming of events over the WebSocket, but is + provided so that clients can reconnect if the + socket is disconnected. +presence Presence The updates to the presence status of other users. +rooms Rooms Updates to rooms. +============= ========== =================================================== + +Example +~~~~~~~ +Message from the server: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [] + }, + "rooms": { + "join": {}, + "invite": {}, + "leave": {} + } + } + + +Client-initiated operations +--------------------------- + +The client can perform certain operations by sending a websocket message to +the server. Such a "Request" message should be a JSON-encoded object with +the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string A unique identifier for this request +method string Specifies the name of the operation to be + performed; see below for available operations +param object The parameters for the requested operation. +============= ========== =================================================== + +The server responds to a client Request with a Response message. This is a +JSON-encoded object with the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string The same as the value in the corresponding Request + object. The presence of the ``id`` field + distinguishes a Response message from an Update + notification. +result object On success, the results of the request. +error object On error, an object giving the resons for the + error. This has the same structure as the "standard + error response" for the Matrix API: an object with + the fields ``errcode`` and ``error``. +============= ========== =================================================== + +Request methods +~~~~~~~~~~~~~~~ +It is not intended that all operations which are available via the REST API +will be available via the WebSockets API, but a few simple, common operations +will be exposed. The initial operations will be as follows. + +``ping`` +^^^^^^^^ +This is a no-op which clients may use to keep their connection alive. + +The request ``params`` and the response ``result`` should be empty. + +``send`` +^^^^^^^^ +Send a message event to a room. The parameters are as follows: + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to send the event to +event_type string **Required.** The type of event to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + +The ``id`` from the Request message is used as the transaction ID by the +server. + +``state`` +^^^^^^^^^ +Update the state on a room. + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to set the state in +event_type string **Required.** The type of event to send. +state_key string **Required.** The state_key for the state to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + + +Example +~~~~~~~ +Client request: + +.. code:: json + + { + "id": "12345", + "method": "send", + "params": { + "room_id": "!d41d8cd:matrix.org", + "event_type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "hello" + } + } + } + +Server response: + +.. code:: json + + { + "id": "12345", + "result": { + "event_id": "$66697273743031:matrix.org" + } + } + +Alternative server response, in case of error: + +.. code:: json + + { + "id": "12345", + "error": { + "errcode": "M_MISSING_PARAM", + "error": "Missing parameter: event_type" + } + } + + +Rationale +--------- +Alternatives to WebSockets include HTTP/2, CoAP, and simply rolling our own +protocol over raw TCP sockets. However, the need to implement browser-based +clients essentially reduces our choice to WebSockets. HTTP/2 streams will +probably provide an interesting alternative in the future, but current browsers +do not appear to give javascript applications low-level access to the protocol. + +Concerning the continued use of the JSON encoding: we prefer to focus on the +transition to WebSockets initially. Replacing JSON with a compact +representation such as CBOR, MessagePack, or even just compressed JSON will be +a likely extension for the future. The support for negotiation of subprotocols +within WebSockets should make this a simple transition once time permits. + +The number of methods available for client requests is deliberately limited, as +each method requires code to be written to map it onto the equivalent REST +implementation. Some REST methods - for instance, user registration and login - +would be pointless to expose via WebSockets. It is likely, however, that we +will increate the number of methods available via the WebSockets API as it +becomes clear which would be most useful. + +Open questions +-------------- + +Throttling +~~~~~~~~~~ +At least in v2 sync, clients are inherently self-throttling - if they do not +poll quickly enough, events will be dropped from the next result. This proposal +raises the possibility that events will be produced more quickly than they can +be sent to the client; backlogs will build up on the server and/or in the +intermediate network, which will not only lead to high latency on events being +delivered, but will lead to responses to client requests also being delayed. + +We may need to implement some sort of throttling mechanism by which the server +can start to drop events. The difficulty is in knowing when to start dropping +events. A few ideas: + +* Use websocket pings to measure the RTT; if it starts to increase, start + dropping events. But this requires knowledge of the base RTT, and a useful + model of what constitutes an excessive increase. + +* Have the client acknowledge each batch of events, and use a window to ensure + the number of outstanding batches is limited. This is annoying as it requires + the client to have to acknowledge batches - and it's not clear what the right + window size is: we want a big window for long fat networks (think of mobile + clients), but a small one for one with lower latency. + +* Start dropping events if the server's TCP buffer starts filling up. This has + the advantage of delegating the congestion-detection to TCP (which already + has a number of algorithms to deal with it, to greater or lesser + effectiveness), but relies on homeservers being hosted on OSes which use + sensible TCP congestion-avoidance algorithms, and more critically, an ability + to read the fill level of the TCP send buffer. From 5ccc39b850a13874774782f2454f5f136f6e9e1d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 20 Nov 2015 18:45:09 +0000 Subject: [PATCH 119/177] Say that type is an event type --- api/client-server/v2_alpha/account-data.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/v2_alpha/account-data.yaml index 98134d6f..b634dddc 100644 --- a/api/client-server/v2_alpha/account-data.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -41,8 +41,8 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be namespaced to - avoid clashes. + The event type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content @@ -87,7 +87,7 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be + The event type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body From e045f28b44137303acccc833ffb486008c46be06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:20:54 +0000 Subject: [PATCH 120/177] Pull out constant for permissions Also, drop permissions from 0755 to 0700 --- scripts/speculator/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4972210e..cfe0f4ee 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -64,13 +64,14 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" - matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + permissionsOwnerFull = 0700 ) func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - if err := os.MkdirAll(directory, 0755); err != nil { + if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil { return "", fmt.Errorf("error making directory %s: %v", directory, err) } args := []string{"clone", url, directory} From 866fa582763279063856ea89c3459e4eeb5ae2b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:22:53 +0000 Subject: [PATCH 121/177] Rename --- scripts/speculator/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index cfe0f4ee..e1874383 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -148,7 +148,8 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } -func (s *server) knowsAbout(sha string) bool { +// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL. +func (s *server) canCheckout(sha string) bool { s.mu.Lock() defer s.mu.Unlock() return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil @@ -157,7 +158,7 @@ func (s *server) knowsAbout(sha string) bool { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - if !s.knowsAbout(sha) { + if !s.canCheckout(sha) { err = s.updateBase() if err != nil { return From c432396079645052c8a9c60eb5be822889d25117 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:26:32 +0000 Subject: [PATCH 122/177] Add comment --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e1874383..0c9a2bfa 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -197,6 +197,8 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. + // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return From 6a6cbd9d24a7cbef9db04a57b6d273e09d51ddf1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:28:58 +0000 Subject: [PATCH 123/177] Always try to build continuserv & speculator on jenkins --- jenkins.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index b2aa8489..f5ed3b15 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -8,7 +8,11 @@ set -ex (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) -if which go >/dev/null 2>/dev/null; then - (cd scripts/continuserv && go build) - (cd scripts/speculator && go build) -fi +: ${GOPATH:=${WORKSPACE}/.gopath} +mkdir -p "${GOPATH}" +export GOPATH +go get github.com/hashicorp/golang-lru +go get gopkg.in/fsnotify.v1 + +(cd scripts/continuserv && go build) +(cd scripts/speculator && go build) From 4e42aab245514a0d7373e23b282c1a284b100dec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 25 Nov 2015 15:13:00 +0000 Subject: [PATCH 124/177] Fix up the multi-hash ratchet thing --- drafts/markjh_end_to_end.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst index 1699f79b..07390b5e 100644 --- a/drafts/markjh_end_to_end.rst +++ b/drafts/markjh_end_to_end.rst @@ -84,18 +84,22 @@ follows: .. math:: \begin{align} - R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ - R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\ R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ - R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ - R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ - R_{i,3} &= H_1\left(R_{(i-1),3}\right) + R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_3\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_3\left(R_{(i-1),3}\right) \end{align} -Where :math:`H_1` and :math:`H_2` are different hash functions. For example -:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and -:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. +Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3` +are different hash functions. For example +:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`. So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. From ec31c0f5184fcf932981b854d770e90011e8fd24 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 12:04:37 +0000 Subject: [PATCH 125/177] speculator: allow styling like matrix.org --- scripts/speculator/main.go | 47 ++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 0c9a2bfa..74731ff6 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -54,9 +54,11 @@ type User struct { } var ( - port = flag.Int("port", 9000, "Port on which to listen for HTTP") - allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") + allowedMembers map[string]bool + specCache *lru.Cache // string -> []byte + styledSpecCache *lru.Cache // string -> []byte ) func (u *User) IsTrusted() bool { @@ -196,6 +198,13 @@ func (s *server) getSHAOf(ref string) (string, error) { func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string + var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != "" + + if styleLikeMatrixDotOrg && *includesDir == "" { + writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified")) + return + } + if strings.ToLower(req.URL.Path) == "/spec/head" { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. @@ -220,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { } sha = pr.Head.SHA } - if cached, ok := specCache.Get(sha); ok { + + var cache = specCache + if styleLikeMatrixDotOrg { + cache = styledSpecCache + } + + if cached, ok := cache.Get(sha); ok { w.Write(cached.([]byte)) return } @@ -232,13 +247,24 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) if err != nil { writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) return } w.Write(b) - specCache.Add(sha, b) + cache.Add(sha, b) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -384,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) { pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } s += `` + if *includesDir != "" { + s += `` + } + io.WriteString(w, s) } @@ -448,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) { } func initCache() error { - c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) - specCache = c + c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + specCache = c1 + + c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + styledSpecCache = c2 return err } From c25a806cef3f5129141449d21fbb888fffa6b266 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 15:03:39 +0000 Subject: [PATCH 126/177] Fix and include /directory api docs --- api/client-server/v1/directory.yaml | 48 ++++++++++++++++++++++++++--- specification/client_server_api.rst | 24 +++------------ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/api/client-server/v1/directory.yaml b/api/client-server/v1/directory.yaml index c70b9f6b..4966a920 100644 --- a/api/client-server/v1/directory.yaml +++ b/api/client-server/v1/directory.yaml @@ -29,6 +29,7 @@ paths: name: roomAlias description: The room alias to set. required: true + x-example: "#monkeys:matrix.org" - in: body name: roomInfo description: Information about this room alias. @@ -39,11 +40,18 @@ paths: room_id: type: string description: The room ID to set. + example: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com" + } responses: 200: description: The mapping was created. + examples: + application/json: |- + {} schema: - type: object # empty json object + type: object get: summary: Get the room ID corresponding to this room alias. parameters: @@ -52,6 +60,7 @@ paths: name: roomAlias description: The room alias. required: true + x-example: "#monkeys:matrix.org" responses: 200: description: The room ID and other information for this alias. @@ -67,10 +76,38 @@ paths: items: type: string description: A server which is aware of this room ID. + examples: + application/json: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com", + "servers": [ + "capuchins.com", + "matrix.org", + "another.com" + ] + } 404: description: There is no mapped room ID for this room alias. + examples: + application/json: |- + { + "errcode": "M_NOT_FOUND", + "error": "Room ID !abnjk1jdasj98:capuchins.com not found." + } + 409: + description: A room alias with that name already exists. + examples: + application/json: |- + { + "errcode": "M_UNKNOWN", + "error": "Room alias #monkeys:matrix.org already exists." + } delete: summary: Remove a mapping of room alias to room ID. + description: |- + Remove a mapping of room alias to room ID. + + Servers may choose to implement additional access control checks here, for instance that room aliases can only be deleted by their creator or a server administrator. security: - accessToken: [] parameters: @@ -79,9 +116,12 @@ paths: name: roomAlias description: The room alias to remove. required: true + x-example: "#monkeys:matrix.org" responses: 200: - description: The mapping was removed. + description: The mapping was deleted. + examples: + application/json: |- + {} schema: - type: object # empty json object - + type: object diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bc..245a9eab 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -783,19 +783,9 @@ client has to use the the following API. Room aliases ~~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - -Room aliases can be created by sending a ``PUT /directory/room/``:: - - { - "room_id": - } -They can be deleted by sending a ``DELETE /directory/room/`` with -no content. Only some privileged users may be able to delete room aliases, e.g. -server admins, the creator of the room alias, etc. This specification does not -outline the privilege level required for deleting room aliases. +Servers may host aliases for rooms with human-friendly names. Aliases take the +form ``#friendlyname:server.name``. As room aliases are scoped to a particular home server domain name, it is likely that a home server will reject attempts to maintain aliases on other @@ -812,17 +802,11 @@ appears to have a room alias of ``#alias:example.com``, this SHOULD be checked to make sure that the room's ID matches the ``room_id`` returned from the request. -Room aliases can be checked in the same way they are resolved; by sending a -``GET /directory/room/``:: - - { - "room_id": , - "servers": [ , , ] - } - Home servers can respond to resolve requests for aliases on other domains than their own by using the federation API to ask other domain name home servers. +{{directory_http_api}} + Permissions ~~~~~~~~~~~ From 5e30b5b8d74cdfbd764175fd735b3c39d652453e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 16:46:29 +0000 Subject: [PATCH 127/177] Remove POST version of /send PUT should always be used. --- api/client-server/v1/room_send.yaml | 47 --------------------- specification/modules/instant_messaging.rst | 8 +--- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml index 9c9273df..3ab1ac40 100644 --- a/api/client-server/v1/room_send.yaml +++ b/api/client-server/v1/room_send.yaml @@ -76,50 +76,3 @@ paths: type: string description: |- A unique identifier for the event. - "/rooms/{roomId}/send/{eventType}": - post: - summary: Send a message event to the given room. - description: |- - This endpoint can be used to send a message event to a room; however - the lack of a transaction ID means that it is possible to cause message - duplication if events are resent on error, so it is preferable to use - `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room to send the event to. - required: true - x-example: "!636q39766251:example.com" - - in: path - type: string - name: eventType - description: The type of event to send. - required: true - x-example: "m.room.message" - - in: body - name: body - schema: - type: object - example: |- - { - "msgtype": "m.text", - "body": "hello" - } - responses: - 200: - description: "An ID for the sent event." - examples: - application/json: |- - { - "event_id": "YUwRidLecu" - } - schema: - type: object - properties: - event_id: - type: string - description: |- - A unique identifier for the event. diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index a58c762f..cd385001 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key. Recommendations when sending messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use -``PUT`` requests with `transaction IDs`_ to make requests idempotent. This -ensures that messages are sent exactly once even under poor network conditions. -Clients SHOULD retry requests using an exponential-backoff algorithm for a +In the event of send failure, clients SHOULD retry requests using an +exponential-backoff algorithm for a certain amount of time T. It is recommended that T is no longer than 5 minutes. After this time, the client should stop retrying and mark the message as "unsent". Users should be able to manually resend unsent messages. @@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room rather than a global queue, as ordering is only relevant within a single room rather than between rooms. -.. _`transaction IDs`: `sect:txn_ids`_ - Local echo ~~~~~~~~~~ From d39494b6dfda679bc4d38a96efb1715428f35279 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 26 Nov 2015 16:55:12 +0000 Subject: [PATCH 128/177] Fix typo in sync example --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c3d120b4..6f2f1412 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -264,7 +264,7 @@ paths: "account_data": { "events": [ { - "type": "m.tags", + "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} } ] From 8d415367573bce44a3a9f2f663157fa08e054bf9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 11:37:24 +0000 Subject: [PATCH 129/177] Address kegan's comments Minor fixes to the e2e spec as raiseds by kegan --- specification/41_end_to_end_encryption.rst | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 023a5684..c5fa4528 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -67,32 +67,35 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with "m." are reserved for algorithms +specification. Algorithm names that start with `m.` are reserved for algorithms defined by this specification. Implementations wanting to experiment with new algorithms are encouraged to pick algorithm names that start with their domain to reduce the risk of collisions. -The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +Algorithm names should be short and meaningful, and should list the primitives +used by the algorithm so that it is easier to see if the algorithm is using a +broken primitive. + +The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -Algorithm names should be short and meaningful. A name of "m.olm.v1" is too -short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" -is too long despite giving a more precise description of the algorithm. - -Algorithm names should list the primitives used by the algorithm so that it -easier to see if the algorithm is using a broken primitive. +A name of `m.olm.v1` is too short: it gives no information about the primitives +in use, and is difficult to extend for different primitives. However a name of +`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +is too long despite giving a more precise description of the algorithm: it adds +to the data transfer overhead and sacrifices clarity for human readers without +adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour @@ -149,7 +152,7 @@ Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be a JSON object per device per user. If one of the user's +will be one JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver must synthesise a JSON object without any device keys for that device. From 2aa4773cc15acfdfa86be396267b1745cbe73ee6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:01:03 +0000 Subject: [PATCH 130/177] Make the speculator serve up errors as plain text ... so that they are legible. --- scripts/speculator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 74731ff6..abe77c74 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -135,6 +135,7 @@ func generate(dir string) error { } func writeError(w http.ResponseWriter, code int, err error) { + w.Header().Set("Content-Type", "text/plain") w.WriteHeader(code) io.WriteString(w, fmt.Sprintf("%v\n", err)) } From 2dbb8ba56c35bf4e914d0c8105bf8db14e4c09ff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:53:03 +0000 Subject: [PATCH 131/177] Fix title levels make the title decoration consistent with the rest of the spec --- specification/modules/end_to_end_encryption.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index cb770680..3ec9368a 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -285,7 +285,7 @@ Encrypted messages are sent in the form. Using Olm -######### ++++++++++ Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their list of supported chat algorithms, must list a Curve25519 device key, and From 0b1ba70a320a7f4fe90e34edf4e3352d5e770433 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 13:33:58 +0000 Subject: [PATCH 132/177] fix rst markup `` > ` --- .../modules/end_to_end_encryption.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 3ec9368a..8b38ab93 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -69,24 +69,24 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with `m.` are reserved for algorithms -defined by this specification. Implementations wanting to experiment with new -algorithms are encouraged to pick algorithm names that start with their -domain to reduce the risk of collisions. +specification. Algorithm names that start with ``m.`` are reserved for +algorithms defined by this specification. Implementations wanting to experiment +with new algorithms are encouraged to pick algorithm names that start with +their domain to reduce the risk of collisions. Algorithm names should be short and meaningful, and should list the primitives used by the algorithm so that it is easier to see if the algorithm is using a broken primitive. -The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm +The name ``m.olm.v1.curve25519-aes-sha2`` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -A name of `m.olm.v1` is too short: it gives no information about the primitives +A name of ``m.olm.v1`` is too short: it gives no information about the primitives in use, and is difficult to extend for different primitives. However a name of -`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +``m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256`` is too long despite giving a more precise description of the algorithm: it adds to the data transfer overhead and sacrifices clarity for human readers without adding any useful extra information. @@ -94,10 +94,10 @@ adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is +The name ``ed25519`` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is +The name ``curve25519`` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour From ad4d8ae7a611bdd13167c358c0447e8fad089fd8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 13:50:24 +0000 Subject: [PATCH 133/177] Move client-server intro to client-server section --- specification/client_server_api.rst | 109 ++++++++++++++++++++++++++++ specification/intro.rst | 109 ---------------------------- 2 files changed, 109 insertions(+), 109 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bc..402ce753 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,115 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +API Standards +------------- + +.. TODO + Need to specify any HMAC or access_token lifetime/ratcheting tricks + We need to specify capability negotiation for extensible transports + +The mandatory baseline for communication in Matrix is exchanging JSON objects +over HTTP APIs. HTTPS is mandated as the baseline for server-server +(federation) communication. HTTPS is recommended for client-server +communication, although HTTP may be supported as a fallback to support basic +HTTP clients. More efficient optional transports for client-server +communication will in future be supported as optional extensions - e.g. a +packed binary encoding over stream-cipher encrypted TCP socket for +low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all +API calls use a Content-Type of ``application/json``. In addition, all strings +MUST be encoded as UTF-8. Clients are authenticated using opaque +``access_token`` strings (see `Client Authentication`_ for details), passed as a +query string parameter on all requests. + +Any errors which occur at the Matrix API level MUST return a "standard error +response". This is a JSON object which looks like:: + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error +codes should have their namespace first in ALL CAPS, followed by a single _ to +ease separating the namespace from the error code. For example, if there was a +custom namespace ``com.mydomain.here``, and a +``FORBIDDEN`` code, the error code should look like +``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the +error, but the keys ``error`` and ``errcode`` MUST always be present. + +Some standard error codes are below: + +:``M_FORBIDDEN``: + Forbidden access, e.g. joining a room without permission, failed login. + +:``M_UNKNOWN_TOKEN``: + The access token specified was not recognised. + +:``M_BAD_JSON``: + Request contained valid JSON, but it was malformed in some way, e.g. missing + required keys, invalid values for keys. + +:``M_NOT_JSON``: + Request did not contain valid JSON. + +:``M_NOT_FOUND``: + No resource was found for this request. + +:``M_LIMIT_EXCEEDED``: + Too many requests have been sent in a short period of time. Wait a while then + try again. + +Some requests have unique error codes: + +:``M_USER_IN_USE``: + Encountered when trying to register a user ID which has been taken. + +:``M_ROOM_IN_USE``: + Encountered when trying to create a room which has been taken. + +:``M_BAD_PAGINATION``: + Encountered when specifying bad pagination query parameters. + +.. _sect:txn_ids: + +The Client-Server API typically uses ``HTTP POST`` to submit requests. This +means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to +make requests idempotent. In order to use a ``PUT``, paths should be suffixed +with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which +identifies the request, and is scoped to a given Client (identified by that +client's ``access_token``). Crucially, it **only** serves to identify new +requests from retransmits. After the request has finished, the ``{txnId}`` +value should be changed (how is not specified; a monotonically increasing +integer is recommended). It is preferable to use ``HTTP PUT`` to make sure +requests to send messages do not get sent more than once should clients need to +retransmit requests. + +Valid requests look like:: + + POST /some/path/here?access_token=secret + { + "key": "This is a post." + } + + PUT /some/path/here/11?access_token=secret + { + "key": "This is a put with a txnId of 11." + } + +In contrast, these are invalid requests:: + + POST /some/path/here/11?access_token=secret + { + "key": "This is a post, but it has a txnId." + } + + PUT /some/path/here?access_token=secret + { + "key": "This is a put but it is missing a txnId." + } + Client Authentication --------------------- Most API endpoints require the user to identify themselves by presenting diff --git a/specification/intro.rst b/specification/intro.rst index 64a21108..6ff7946c 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -358,112 +358,3 @@ dedicated API. The API is symmetrical to managing Profile data. Would it really be overengineered to use the same API for both profile & private user data, but with different ACLs? -API Standards -------------- - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks - We need to specify capability negotiation for extensible transports - -The mandatory baseline for communication in Matrix is exchanging JSON objects -over HTTP APIs. HTTPS is mandated as the baseline for server-server -(federation) communication. HTTPS is recommended for client-server -communication, although HTTP may be supported as a fallback to support basic -HTTP clients. More efficient optional transports for client-server -communication will in future be supported as optional extensions - e.g. a -packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all -API calls use a Content-Type of ``application/json``. In addition, all strings -MUST be encoded as UTF-8. Clients are authenticated using opaque -``access_token`` strings (see `Client Authentication`_ for details), passed as a -query string parameter on all requests. - -Any errors which occur at the Matrix API level MUST return a "standard error -response". This is a JSON object which looks like:: - - { - "errcode": "", - "error": "" - } - -The ``error`` string will be a human-readable error message, usually a sentence -explaining what went wrong. The ``errcode`` string will be a unique string -which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error -codes should have their namespace first in ALL CAPS, followed by a single _ to -ease separating the namespace from the error code. For example, if there was a -custom namespace ``com.mydomain.here``, and a -``FORBIDDEN`` code, the error code should look like -``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the -error, but the keys ``error`` and ``errcode`` MUST always be present. - -Some standard error codes are below: - -:``M_FORBIDDEN``: - Forbidden access, e.g. joining a room without permission, failed login. - -:``M_UNKNOWN_TOKEN``: - The access token specified was not recognised. - -:``M_BAD_JSON``: - Request contained valid JSON, but it was malformed in some way, e.g. missing - required keys, invalid values for keys. - -:``M_NOT_JSON``: - Request did not contain valid JSON. - -:``M_NOT_FOUND``: - No resource was found for this request. - -:``M_LIMIT_EXCEEDED``: - Too many requests have been sent in a short period of time. Wait a while then - try again. - -Some requests have unique error codes: - -:``M_USER_IN_USE``: - Encountered when trying to register a user ID which has been taken. - -:``M_ROOM_IN_USE``: - Encountered when trying to create a room which has been taken. - -:``M_BAD_PAGINATION``: - Encountered when specifying bad pagination query parameters. - -.. _sect:txn_ids: - -The Client-Server API typically uses ``HTTP POST`` to submit requests. This -means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to -make requests idempotent. In order to use a ``PUT``, paths should be suffixed -with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which -identifies the request, and is scoped to a given Client (identified by that -client's ``access_token``). Crucially, it **only** serves to identify new -requests from retransmits. After the request has finished, the ``{txnId}`` -value should be changed (how is not specified; a monotonically increasing -integer is recommended). It is preferable to use ``HTTP PUT`` to make sure -requests to send messages do not get sent more than once should clients need to -retransmit requests. - -Valid requests look like:: - - POST /some/path/here?access_token=secret - { - "key": "This is a post." - } - - PUT /some/path/here/11?access_token=secret - { - "key": "This is a put with a txnId of 11." - } - -In contrast, these are invalid requests:: - - POST /some/path/here/11?access_token=secret - { - "key": "This is a post, but it has a txnId." - } - - PUT /some/path/here?access_token=secret - { - "key": "This is a put but it is missing a txnId." - } - From f0c99a6925a4d5ff38fcc563a50ec280b6895631 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:03:22 +0000 Subject: [PATCH 134/177] Generate API docs from swagger --- api/client-server/api-docs | 50 - api/client-server/api-docs-content | 119 --- api/client-server/api-docs-directory | 101 -- api/client-server/api-docs-events | 247 ----- api/client-server/api-docs-login | 120 --- api/client-server/api-docs-presence | 164 ---- api/client-server/api-docs-profile | 122 --- api/client-server/api-docs-registration | 120 --- api/client-server/api-docs-rooms | 1128 ----------------------- scripts/add-matrix-org-stylings.sh | 8 +- scripts/generate-http-docs.sh | 27 + 11 files changed, 32 insertions(+), 2174 deletions(-) delete mode 100644 api/client-server/api-docs delete mode 100644 api/client-server/api-docs-content delete mode 100644 api/client-server/api-docs-directory delete mode 100644 api/client-server/api-docs-events delete mode 100644 api/client-server/api-docs-login delete mode 100644 api/client-server/api-docs-presence delete mode 100644 api/client-server/api-docs-profile delete mode 100644 api/client-server/api-docs-registration delete mode 100644 api/client-server/api-docs-rooms create mode 100755 scripts/generate-http-docs.sh diff --git a/api/client-server/api-docs b/api/client-server/api-docs deleted file mode 100644 index 9c8c6066..00000000 --- a/api/client-server/api-docs +++ /dev/null @@ -1,50 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "apis": [ - { - "path": "-login", - "description": "Login operations" - }, - { - "path": "-registration", - "description": "Registration operations" - }, - { - "path": "-rooms", - "description": "Room operations" - }, - { - "path": "-profile", - "description": "Profile operations" - }, - { - "path": "-presence", - "description": "Presence operations" - }, - { - "path": "-events", - "description": "Event operations" - }, - { - "path": "-directory", - "description": "Directory operations" - }, - { - "path": "-content", - "description": "Content repository operations" - } - ], - "authorizations": { - "token": { - "scopes": [] - } - }, - "info": { - "title": "Matrix Client-Server API Reference", - "description": "This contains the client-server API for the reference implementation of the home server", - "termsOfServiceUrl": "http://matrix.org", - "license": "Apache 2.0", - "licenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.html" - } -} diff --git a/api/client-server/api-docs-content b/api/client-server/api-docs-content deleted file mode 100644 index a0a31da0..00000000 --- a/api/client-server/api-docs-content +++ /dev/null @@ -1,119 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix", - "resourcePath": "/media/v1/", - "apis": [ - { - "path": "/media/v1/upload", - "operations": [ - { - "method": "POST", - "summary": "Upload some content to the content repository.", - "type": "ContentUploadResponse", - "nickname": "upload_content", - "parameters": [ - { - "name": "body", - "description": "The file to upload.", - "required": true, - "type": "file", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/media/v1/download/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get the content stored at this address.", - "type": "file", - "nickname": "download_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/media/v1/thumbnail/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get a thumbnail of the content stored at this address.", - "type": "file", - "nickname": "thumbnail_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "width", - "description": "The desired width of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "height", - "description": "The desired height of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "method", - "description": "The desired resizing method.", - "enum": [ - "crop", - "scale" - ], - "required": false, - "type": "string", - "paramType": "query" - } - ] - } - ] - } - ], - "models": { - "ContentUploadResponse": { - "id": "ContentUploadResponse", - "properties": { - "content_uri": { - "type": "string", - "description": "The mxc:// URI where this content is stored. This is of the form 'mxc://{serverName}/{mediaId}'", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-directory b/api/client-server/api-docs-directory deleted file mode 100644 index 5dda5806..00000000 --- a/api/client-server/api-docs-directory +++ /dev/null @@ -1,101 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/directory", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/directory/room/{roomAlias}", - "operations": [ - { - "method": "GET", - "summary": "Get the room ID corresponding to this room alias.", - "notes": "Volatile: This API is likely to change.", - "type": "DirectoryResponse", - "nickname": "get_room_id_for_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "PUT", - "summary": "Create a new mapping from room alias to room ID.", - "notes": "Volatile: This API is likely to change.", - "type": "void", - "nickname": "add_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to set.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The room ID to set.", - "required": true, - "type": "RoomAliasRequest", - "paramType": "body" - } - ] - }, - { - "method": "DELETE", - "summary": "Removes a mapping of room alias to room ID.", - "notes": "Only privileged users can perform this action.", - "type": "void", - "nickname": "remove_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to remove.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DirectoryResponse": { - "id": "DirectoryResponse", - "properties": { - "room_id": { - "type": "string", - "description": "The fully-qualified room ID.", - "required": true - }, - "servers": { - "type": "array", - "items": { - "$ref": "string" - }, - "description": "A list of servers that know about this room.", - "required": true - } - } - }, - "RoomAliasRequest": { - "id": "RoomAliasRequest", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID to map the alias to.", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-events b/api/client-server/api-docs-events deleted file mode 100644 index 1bdb9b03..00000000 --- a/api/client-server/api-docs-events +++ /dev/null @@ -1,247 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/events", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/events", - "operations": [ - { - "method": "GET", - "summary": "Listen on the event stream", - "notes": "This can only be done by the logged in user. This will block until an event is received, or until the timeout is reached.", - "type": "PaginationChunk", - "nickname": "get_event_stream", - "parameters": [ - { - "name": "from", - "description": "The token to stream from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "timeout", - "description": "The maximum time in milliseconds to wait for an event.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ], - - "responseMessages": [ - { - "code": 400, - "message": "Bad pagination token." - } - ] - }, - { - "path": "/events/{eventId}", - "operations": [ - { - "method": "GET", - "summary": "Get information about a single event.", - "notes": "Get information about a single event.", - "type": "Event", - "nickname": "get_event", - "parameters": [ - { - "name": "eventId", - "description": "The event ID to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Event not found." - } - ] - } - ] - }, - { - "path": "/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get this user's current state.", - "notes": "Get this user's current state.", - "type": "InitialSyncResponse", - "nickname": "initial_sync", - "parameters": [ - { - "name": "limit", - "description": "The maximum number of messages to return for each room.", - "type": "integer", - "paramType": "query", - "required": false - } - ] - } - ] - }, - { - "path": "/publicRooms", - "operations": [ - { - "method": "GET", - "summary": "Get a list of publicly visible rooms.", - "type": "PublicRoomsPaginationChunk", - "nickname": "get_public_room_list" - } - ] - } - ], - "models": { - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "An array of events.", - "required": true, - "items": { - "$ref": "Event" - } - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred.", - "required": true - } - } - }, - "PublicRoomInfo": { - "id": "PublicRoomInfo", - "properties": { - "aliases": { - "type": "array", - "description": "A list of room aliases for this room.", - "items": { - "$ref": "string" - } - }, - "name": { - "type": "string", - "description": "The name of the room, as given by the m.room.name state event." - }, - "room_id": { - "type": "string", - "description": "The room ID for this public room.", - "required": true - }, - "topic": { - "type": "string", - "description": "The topic of this room, as given by the m.room.topic state event." - } - } - }, - "PublicRoomsPaginationChunk": { - "id": "PublicRoomsPaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "A list of public room data.", - "required": true, - "items": { - "$ref": "PublicRoomInfo" - } - } - } - }, - "InitialSyncResponse": { - "id": "InitialSyncResponse", - "properties": { - "end": { - "type": "string", - "description": "A streaming token which can be used with /events to continue from this snapshot of data.", - "required": true - }, - "presence": { - "type": "array", - "description": "A list of presence events.", - "items": { - "$ref": "Event" - }, - "required": false - }, - "rooms": { - "type": "array", - "description": "A list of initial sync room data.", - "required": false, - "items": { - "$ref": "InitialSyncRoomData" - } - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "PaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/api/client-server/api-docs-login b/api/client-server/api-docs-login deleted file mode 100644 index d6f8d84f..00000000 --- a/api/client-server/api-docs-login +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_login_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when logging in.", - "type": "LoginFlows" - }, - { - "method": "POST", - "nickname": "submit_login", - "notes": "If this is part of a multi-stage login, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A login submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "LoginSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a login action.", - "type": "LoginResult" - } - ], - "path": "/login" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "LoginFlows": { - "id": "LoginFlows", - "properties": { - "flows": { - "description": "A list of valid login flows.", - "type": "array", - "items": { - "$ref": "LoginInfo" - } - } - } - }, - "LoginInfo": { - "id": "LoginInfo", - "properties": { - "stages": { - "description": "Multi-stage login only: An array of all the login types required to login.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The login type that must be used when logging in.", - "type": "string" - } - } - }, - "LoginResult": { - "id": "LoginResult", - "properties": { - "access_token": { - "description": "The access token for this user's login if this is the final stage of the login process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage login only: The next login type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token to send when submitting the next login type.", - "type": "string" - } - } - }, - "LoginSubmission": { - "id": "LoginSubmission", - "properties": { - "type": { - "description": "The type of login being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token from an earlier login stage.", - "type": "string" - }, - "_login_type_defined_keys_": { - "description": "Keys as defined by the specified login type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/login", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-presence b/api/client-server/api-docs-presence deleted file mode 100644 index 6b224460..00000000 --- a/api/client-server/api-docs-presence +++ /dev/null @@ -1,164 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/presence", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/presence/{userId}/status", - "operations": [ - { - "method": "PUT", - "summary": "Update this user's presence state.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "update_presence", - "parameters": [ - { - "name": "body", - "description": "The new presence state", - "required": true, - "type": "PresenceUpdate", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose presence to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get this user's presence state.", - "notes": "Get this user's presence state.", - "type": "PresenceUpdate", - "nickname": "get_presence", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/presence/list/{userId}", - "operations": [ - { - "method": "GET", - "summary": "Retrieve a list of presences for all of this user's friends.", - "notes": "", - "type": "array", - "items": { - "$ref": "Presence" - }, - "nickname": "get_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "POST", - "summary": "Add or remove users from this presence list.", - "notes": "Add or remove users from this presence list.", - "type": "void", - "nickname": "modify_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list is being modified.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The modifications to make to this presence list.", - "required": true, - "type": "PresenceListModifications", - "paramType": "body" - } - ] - } - ] - } - ], - "models": { - "PresenceUpdate": { - "id": "PresenceUpdate", - "properties": { - "presence": { - "type": "string", - "description": "Enum: The presence state.", - "enum": [ - "offline", - "unavailable", - "online", - "free_for_chat" - ] - }, - "status_msg": { - "type": "string", - "description": "The user-defined message associated with this presence state." - } - }, - "subTypes": [ - "Presence" - ] - }, - "Presence": { - "id": "Presence", - "properties": { - "last_active_ago": { - "type": "integer", - "format": "int64", - "description": "The last time this user performed an action on their home server." - }, - "user_id": { - "type": "string", - "description": "The fully qualified user ID" - } - } - }, - "PresenceListModifications": { - "id": "PresenceListModifications", - "properties": { - "invite": { - "type": "array", - "description": "A list of user IDs to add to the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - }, - "drop": { - "type": "array", - "description": "A list of user IDs to remove from the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - } - } - } - } -} diff --git a/api/client-server/api-docs-profile b/api/client-server/api-docs-profile deleted file mode 100644 index d2fccaa6..00000000 --- a/api/client-server/api-docs-profile +++ /dev/null @@ -1,122 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/profile", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/profile/{userId}/displayname", - "operations": [ - { - "method": "PUT", - "summary": "Set a display name.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_display_name", - "parameters": [ - { - "name": "body", - "description": "The new display name for this user.", - "required": true, - "type": "DisplayName", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose display name to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get a display name.", - "notes": "This can be done by anyone.", - "type": "DisplayName", - "nickname": "get_display_name", - "parameters": [ - { - "name": "userId", - "description": "The user whose display name to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/profile/{userId}/avatar_url", - "operations": [ - { - "method": "PUT", - "summary": "Set an avatar URL.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_avatar_url", - "parameters": [ - { - "name": "body", - "description": "The new avatar url for this user.", - "required": true, - "type": "AvatarUrl", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose avatar url to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get an avatar url.", - "notes": "This can be done by anyone.", - "type": "AvatarUrl", - "nickname": "get_avatar_url", - "parameters": [ - { - "name": "userId", - "description": "The user whose avatar url to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DisplayName": { - "id": "DisplayName", - "properties": { - "displayname": { - "type": "string", - "description": "The textual display name" - } - } - }, - "AvatarUrl": { - "id": "AvatarUrl", - "properties": { - "avatar_url": { - "type": "string", - "description": "A url to an image representing an avatar." - } - } - } - } -} diff --git a/api/client-server/api-docs-registration b/api/client-server/api-docs-registration deleted file mode 100644 index 11c170c3..00000000 --- a/api/client-server/api-docs-registration +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_registration_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when registering.", - "type": "RegistrationFlows" - }, - { - "method": "POST", - "nickname": "submit_registration", - "notes": "If this is part of a multi-stage registration, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A registration submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "RegistrationSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a registration action.", - "type": "RegistrationResult" - } - ], - "path": "/register" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "RegistrationFlows": { - "id": "RegistrationFlows", - "properties": { - "flows": { - "description": "A list of valid registration flows.", - "type": "array", - "items": { - "$ref": "RegistrationInfo" - } - } - } - }, - "RegistrationInfo": { - "id": "RegistrationInfo", - "properties": { - "stages": { - "description": "Multi-stage registration only: An array of all the login types required to registration.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The first login type that must be used when logging in.", - "type": "string" - } - } - }, - "RegistrationResult": { - "id": "RegistrationResult", - "properties": { - "access_token": { - "description": "The access token for this user's registration if this is the final stage of the registration process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage registration only: The next registration type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token to send when submitting the next registration type.", - "type": "string" - } - } - }, - "RegistrationSubmission": { - "id": "RegistrationSubmission", - "properties": { - "type": { - "description": "The type of registration being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token from an earlier registration stage.", - "type": "string" - }, - "_registration_type_defined_keys_": { - "description": "Keys as defined by the specified registration type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/register", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-rooms b/api/client-server/api-docs-rooms deleted file mode 100644 index 0d3d9fbc..00000000 --- a/api/client-server/api-docs-rooms +++ /dev/null @@ -1,1128 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/rooms", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "authorizations": { - "token": [] - }, - "apis": [ - { - "path": "/rooms/{roomId}/send/{eventType}", - "operations": [ - { - "method": "POST", - "summary": "Send a generic non-state event to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_non_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/{eventType}/{stateKey}", - "operations": [ - { - "method": "PUT", - "summary": "Send a generic state event to this room.", - "notes": "The state key can be omitted, such that you can PUT to /rooms/{roomId}/state/{eventType}. The state key defaults to a 0 length string in this case.", - "type": "void", - "nickname": "send_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "stateKey", - "description": "An identifier used to specify clobbering semantics. State events with the same (roomId, eventType, stateKey) will be replaced.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message", - "operations": [ - { - "method": "POST", - "summary": "Send a message in this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_message", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The message contents", - "required": true, - "type": "Message", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.topic", - "operations": [ - { - "method": "PUT", - "summary": "Set the topic for this room.", - "notes": "Set the topic for this room.", - "type": "void", - "nickname": "set_topic", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The topic contents", - "required": true, - "type": "Topic", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the topic for this room.", - "notes": "Get the topic for this room.", - "type": "Topic", - "nickname": "get_topic", - "parameters": [ - { - "name": "roomId", - "description": "The room to get topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Topic not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.name", - "operations": [ - { - "method": "PUT", - "summary": "Set the name of this room.", - "notes": "Set the name of this room.", - "type": "void", - "nickname": "set_room_name", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The name contents", - "required": true, - "type": "RoomName", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the room's name.", - "notes": "", - "type": "RoomName", - "nickname": "get_room_name", - "parameters": [ - { - "name": "roomId", - "description": "The room to get the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Name not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.power_levels", - "operations": [ - { - "method": "PUT", - "summary": "Set the power levels for this room.", - "notes": "This has to be set atomically. The levels set will clobber what was previously there.", - "type": "void", - "nickname": "set_room_power_levels", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The power levels", - "required": true, - "type": "RoomPowerLevels", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the power levels for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message.feedback", - "operations": [ - { - "method": "POST", - "summary": "Send feedback to a message.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_feedback", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The feedback contents", - "required": true, - "type": "Feedback", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the feedback in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad feedback type." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/invite", - "operations": [ - { - "method": "POST", - "summary": "Invite a user to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "invite", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to invite.", - "required": true, - "type": "InviteRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/join", - "operations": [ - { - "method": "POST", - "summary": "Join this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "join_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to join.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "JoinRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/leave", - "operations": [ - { - "method": "POST", - "summary": "Leave this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "leave", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to leave.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "LeaveRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/ban", - "operations": [ - { - "method": "POST", - "summary": "Ban a user in the room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}. The caller must have the required power level to do this operation.", - "type": "void", - "nickname": "ban", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has the user to ban.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to ban.", - "required": true, - "type": "BanRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.member/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Change the membership state for a user in a room.", - "notes": "Change the membership state for a user in a room.", - "type": "void", - "nickname": "set_membership", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The new membership state", - "required": true, - "type": "Member", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose membership is being changed.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "No membership key." - }, - { - "code": 400, - "message": "Bad membership value." - }, - { - "code": 403, - "message": "When inviting: You are not in the room." - }, - { - "code": 403, - "message": "When inviting: is already in the room." - }, - { - "code": 403, - "message": "When joining: Cannot force another user to join." - }, - { - "code": 403, - "message": "When joining: You are not invited to this room." - } - ] - }, - { - "method": "GET", - "summary": "Get the membership state of a user in a room.", - "notes": "Get the membership state of a user in a room.", - "type": "Member", - "nickname": "get_membership", - "parameters": [ - { - "name": "userId", - "description": "The user whose membership state you want to get.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Member not found." - } - ] - } - ] - }, - { - "path": "/join/{roomAliasOrId}", - "operations": [ - { - "method": "POST", - "summary": "Join a room via a room alias or room ID.", - "notes": "This endpoint, unlike /rooms/{roomId}/join allows the client to join a room by it's alias. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "JoinRoomInfo", - "nickname": "join", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomAliasOrId", - "description": "The room alias or room ID to join.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad room alias." - } - ] - } - ] - }, - { - "path": "/createRoom", - "operations": [ - { - "method": "POST", - "summary": "Create a room.", - "notes": "Create a room. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "RoomInfo", - "nickname": "create_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The desired configuration for the room.", - "required": true, - "type": "RoomConfig", - "paramType": "body" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Body must be JSON." - }, - { - "code": 400, - "message": "Room alias already taken." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/messages", - "operations": [ - { - "method": "GET", - "summary": "Get a list of messages for this room.", - "notes": "Get a list of messages for this room.", - "type": "MessagePaginationChunk", - "nickname": "get_messages", - "parameters": [ - { - "name": "roomId", - "description": "The room to get messages in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of messages to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/members", - "operations": [ - { - "method": "GET", - "summary": "Get a list of members for this room.", - "notes": "Get a list of members for this room.", - "type": "MemberPaginationChunk", - "nickname": "get_members", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of members from.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of members to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state", - "operations": [ - { - "method": "GET", - "summary": "Get a list of all the current state events for this room.", - "notes": "This is equivalent to the events returned under the 'state' key for this room in /initialSync.", - "type": "array", - "items": { - "$ref": "Event" - }, - "nickname": "get_state_events", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of current state events from.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/typing/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Inform the server that this user is typing.", - "notes": "Only the authorised user can send typing notifications.", - "type": "void", - "nickname": "put_typing", - "parameters": [ - { - "name": "roomId", - "description": "The room to send the m.typing event into.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "userId", - "description": "The user ID of the typer.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The typing information.", - "required": true, - "type": "Typing", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get all the current information for this room, including messages and state events.", - "type": "InitialSyncRoomData", - "nickname": "get_room_sync_data", - "parameters": [ - { - "name": "roomId", - "description": "The room to get information for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "Topic": { - "id": "Topic", - "properties": { - "topic": { - "type": "string", - "description": "The topic text" - } - } - }, - "RoomName": { - "id": "RoomName", - "properties": { - "name": { - "type": "string", - "description": "The human-readable name for the room. Can contain spaces." - } - } - }, - "UserPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__user_id__": { - "type": "integer", - "description": "The power level for __user_id__" - } - } - }, - "EventPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__event_type__": { - "type": "integer", - "description": "The power level required in order to send __event_type__ events." - } - } - }, - "RoomPowerLevels": { - "id": "RoomPowerLevels", - "properties": { - "ban": { - "type": "integer", - "description": "The minimum level required in order to ban someone." - }, - "kick": { - "type": "integer", - "description": "The minimum level required in order to kick someone." - }, - "redact": { - "type": "integer", - "description": "The minimum level required in order to redact a message." - }, - "users_default": { - "type": "integer", - "description": "The default power level of a user who is not in the 'users' list." - }, - "state_default": { - "type": "integer", - "description": "The default power level required in order to send state events." - }, - "events_default": { - "type": "integer", - "description": "The default power level required in order to send non-state events." - }, - "users": { - "type": "UserPowerLevels", - "description": "Mappings of user ID to power level." - }, - "events": { - "type": "EventPowerLevels", - "description": "Mappings of event type to power level." - } - } - }, - "Message": { - "id": "Message", - "properties": { - "msgtype": { - "type": "string", - "description": "The type of message being sent, e.g. \"m.text\"", - "required": true - }, - "_msgtype_defined_keys_": { - "description": "Additional keys as defined by the msgtype, e.g. \"body\"" - } - } - }, - "Feedback": { - "id": "Feedback", - "properties": { - "target_event_id": { - "type": "string", - "description": "The event ID being acknowledged.", - "required": true - }, - "type": { - "type": "string", - "description": "The type of feedback. Either 'delivered' or 'read'.", - "required": true - } - } - }, - "Member": { - "id": "Member", - "properties": { - "membership": { - "type": "string", - "description": "Enum: The membership state of this member.", - "enum": [ - "invite", - "join", - "leave", - "ban" - ] - } - } - }, - "RoomInfo": { - "id": "RoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The allocated room ID.", - "required": true - }, - "room_alias": { - "type": "string", - "description": "The alias for the room.", - "required": false - } - } - }, - "JoinRoomInfo": { - "id": "JoinRoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID joined, if joined via a room alias only.", - "required": true - } - } - }, - "Typing": { - "id": "Typing", - "properties": { - "typing": { - "type": "boolean", - "description": "True if the user is currently typing.", - "required": true - }, - "timeout": { - "type": "integer", - "description": "The length of time until the user should be treated as no longer typing, in milliseconds. Can be omitted if they are no longer typing.", - "required": true - } - } - }, - "RoomConfig": { - "id": "RoomConfig", - "properties": { - "visibility": { - "type": "string", - "description": "Enum: The room visibility. The room_alias_name is required if the visibility is public; without it the room will remain private.", - "required": false, - "enum": [ - "public", - "private" - ] - }, - "room_alias_name": { - "type": "string", - "description": "Localpart of the alias to give the new room. The home server will attach its domain name after this.", - "required": false - }, - "name": { - "type": "string", - "description": "Sets the name of the room. Send a m.room.name event after creating the room with the 'name' key specified.", - "required": false - }, - "topic": { - "type": "string", - "description": "Sets the topic for the room. Send a m.room.topic event after creating the room with the 'topic' key specified.", - "required": false - }, - "invite": { - "type": "array", - "description": "The list of user IDs to invite. Sends m.room.member events after creating the room.", - "items": { - "$ref": "string" - }, - "required": false - } - } - }, - "PaginationRequest": { - "id": "PaginationRequest", - "properties": { - "from": { - "type": "string", - "description": "The token to start getting results from." - }, - "to": { - "type": "string", - "description": "The token to stop getting results at." - }, - "limit": { - "type": "integer", - "description": "The maximum number of entries to return." - } - } - }, - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - } - }, - "subTypes": [ - "MessagePaginationChunk" - ] - }, - "MessagePaginationChunk": { - "id": "MessagePaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of message events.", - "items": { - "$ref": "MessageEvent" - }, - "required": true - } - } - }, - "MemberPaginationChunk": { - "id": "MemberPaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of member events.", - "items": { - "$ref": "MemberEvent" - }, - "required": true - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event. This is automatically set by the server.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred. This is automatically set by the server.", - "required": true - }, - "type": { - "type": "string", - "description": "The event type.", - "required": true - } - }, - "subTypes": [ - "MessageEvent" - ] - }, - "EventId": { - "id": "EventId", - "properties": { - "event_id": { - "type": "string", - "description": "The allocated event ID for this event.", - "required": true - } - } - }, - "EventContent": { - "id": "EventContent", - "properties": { - "__event_content_keys__": { - "type": "string", - "description": "Event-specific content keys and values.", - "required": false - } - } - }, - "MessageEvent": { - "id": "MessageEvent", - "properties": { - "content": { - "type": "Message" - } - } - }, - "MemberEvent": { - "id": "MemberEvent", - "properties": { - "content": { - "type": "Member" - } - } - }, - "InviteRequest": { - "id": "InviteRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - } - } - }, - "JoinRequest": { - "id": "JoinRequest", - "properties": {} - }, - "LeaveRequest": { - "id": "LeaveRequest", - "properties": {} - }, - "BanRequest": { - "id": "BanRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - }, - "reason": { - "type": "string", - "description": "The reason for the ban." - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "MessagePaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - }, - "presence": { - "type": "array", - "description": "A list of m.presence events representing the current presence state of the room members.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index ef4e1014..9a9256b2 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -16,9 +16,11 @@ for f in "$1"/{head,nav,footer}.html; do fi done +files=gen/*.html + perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header -#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s##
@@ -27,7 +29,7 @@ perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s## <
-#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s##
@@ -39,4 +41,4 @@ perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s## $footer
- #' gen/specification.html gen/howtos.html + #' ${files} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh new file mode 100755 index 00000000..0a2bc5f6 --- /dev/null +++ b/scripts/generate-http-docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu + +# This script generates an HTML page containing all of the client-server API docs. +# It takes all of the swagger YAML files for the client-server API, and turns +# them into API docs, with none of the narrative found in the rst files which +# normally wrap these API docs. + +cd "$(dirname $0)" + +mkdir -p tmp gen + +cat >tmp/http_apis <> tmp/http_apis +done + +(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis) +rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html From 7f2813354daecf30bf1a14fa715f88a3511ed5b6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:42:21 +0000 Subject: [PATCH 135/177] Swaggerify /ban --- api/client-server/v1/banning.yaml | 76 +++++++++++++++++++++++++++++ specification/client_server_api.rst | 2 + 2 files changed, 78 insertions(+) create mode 100644 api/client-server/v1/banning.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/v1/banning.yaml new file mode 100644 index 00000000..e841050f --- /dev/null +++ b/api/client-server/v1/banning.yaml @@ -0,0 +1,76 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Banning API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/ban": + post: + summary: Ban a user in the room. + description: |- + Ban a user in the room. If the user is currently in the room, also kick them. + + When a user is banned from a room, they may not join it until they are unbanned. + + The caller must have the required power level in order to perform this operation. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) from which the user should be banned. + required: true + x-example: "!e42d8c:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "reason": "Telling unfunny jokes", + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the user being banned. + reason: + type: string + description: The reason the user has been banned. + required: ["user_id"] + responses: + 200: + description: The user has been kicked and banned from the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to ban the user from the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The banner is not currently in the room. + - The banner's power level is insufficient to ban users from the room. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN", + "error": "You do not have a high enough power level to ban from this room." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 402ce753..58d0f1b2 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -983,6 +983,8 @@ following values: {{joining_http_api}} +{{banning_http_api}} + Leaving rooms ~~~~~~~~~~~~~ A user can leave a room to stop receiving events for that room. A user must From 3951785f19b2b6c38980bcf2748479ce4ba5ef86 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 16:23:31 +0000 Subject: [PATCH 136/177] Fix alias path --- api/client-server/v1/joining.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml index 20fdbd65..b6d2df18 100644 --- a/api/client-server/v1/joining.yaml +++ b/api/client-server/v1/joining.yaml @@ -65,4 +65,4 @@ paths: x-alias: canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" aliases: - - /join/{roomId} + - /_matrix/client/api/v1/join/{roomId} From e171acf01ffecc4a931712928ffca024cbdade8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:21:32 +0000 Subject: [PATCH 137/177] Split spec into page-per-section --- scripts/gendoc.py | 52 +++++++++++++------ specification/intro.rst | 10 ++++ .../modules/typing_notifications.rst | 6 +-- specification/targets.yaml | 11 +++- 4 files changed, 57 insertions(+), 22 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 28a11528..9b87cb30 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -276,6 +276,12 @@ def run_through_template(input, set_verbose): raise +def get_build_targets(targets_listing): + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + return all_targets["targets"].keys() + + """ Extract and resolve groups for the given target in the given targets listing. Args: @@ -386,21 +392,34 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates): prepare_env() - log("Building spec [target=%s]" % target_name) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst", VERBOSE) - fix_relative_titles( - target=target, filename="tmp/templated_spec.rst", - out_filename="tmp/full_spec.rst" - ) - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this - rst2html("tmp/full_spec.rst", "gen/specification.html") - addAnchors("gen/specification.html") - rst2html("tmp/howto.rst", "gen/howtos.html") + log("Building spec [target=%s]" % requested_target_name) + + targets = [requested_target_name] + if requested_target_name == "all": + targets = get_build_targets("../specification/targets.yaml") + + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + rst_file = "tmp/spec_%s.rst" % (target_name,) + html_file = "gen/%s.html" % (target_name,) + + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + run_through_template(templated_file, VERBOSE) + fix_relative_titles( + target=target, filename=templated_file, + out_filename=rst_file, + ) + rst2html(rst_file, html_file) + addAnchors(html_file) + + if requested_target_name == "all": + shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + rst2html("tmp/howto.rst", "gen/howtos.html") + if not keep_intermediates: cleanup_env() @@ -414,8 +433,9 @@ if __name__ == '__main__': help="Do not delete intermediate files. They will be found in tmp/" ) parser.add_argument( - "--target", "-t", default="main", - help="Specify the build target to build from specification/targets.yaml" + "--target", "-t", default="all", + help="Specify the build target to build from specification/targets.yaml. " + + "The value 'all' will build all of the targets therein." ) parser.add_argument( "--verbose", "-v", action="store_true", diff --git a/specification/intro.rst b/specification/intro.rst index 6ff7946c..8c08bf24 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -8,6 +8,16 @@ https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +APIs +~~~~ +The following APIs are documented in this specification: + +- `Client-Server API `_ for writing Matrix clients. +- `Server-Server API `_ for writing servers which can federate with Matrix. +- `Application Service API `_ for writing privileged plugins to servers. + +There are also some `appendices `_. + Changelog ~~~~~~~~~ {{spec_changelog}} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index d2253632..da383e73 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -5,10 +5,8 @@ Typing Notifications Users may wish to be informed when another user is typing in a room. This can be achieved using typing notifications. These are ephemeral events scoped to a -``room_id``. This means they do not form part of the `Event Graph`_ but still -have a ``room_id`` key. - -.. _Event Graph: `sect:event-graph`_ +``room_id``. This means they do not form part of the +`Event Graph `_ but still have a ``room_id`` key. Events ------ diff --git a/specification/targets.yaml b/specification/targets.yaml index 8e6a2ce0..cd0f6f41 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,16 +1,23 @@ targets: - main: # arbitrary name to identify this build target + index: files: # the sort order of files to cat - intro.rst + client_server: + files: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - modules.rst - { 1: feature_profiles.rst } - { 1: "group:modules" } # reference a group of files + application_service: + files: - application_service_api.rst + server_server: + files: - server_server_api.rst - - identity_servers.rst + appendices: + files: - appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: From 30ed918633cf47ceb2d5dac793b1e3a3e9a0d80f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:53:58 +0000 Subject: [PATCH 138/177] speculator: Allow spec viewing for multi-page spec --- scripts/speculator/main.go | 94 +++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index abe77c74..3736672f 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -57,8 +57,8 @@ var ( port = flag.Int("port", 9000, "Port on which to listen for HTTP") includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte - styledSpecCache *lru.Cache // string -> []byte + specCache *lru.Cache // string -> map[string][]byte filename -> contents + styledSpecCache *lru.Cache // string -> map[string][]byte filename -> contents ) func (u *User) IsTrusted() bool { @@ -105,10 +105,7 @@ func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) { if !strings.HasPrefix(url.Path, pathPrefix+"/") { return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) } - prNumber := url.Path[len(pathPrefix)+1:] - if strings.Contains(prNumber, "/") { - return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) - } + prNumber := strings.Split(url.Path[len(pathPrefix)+1:], "/")[0] resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) defer resp.Body.Close() @@ -196,6 +193,17 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } +func extractPath(path, base string) string { + // Assume exactly one flat directory + max := strings.Count(base, "/") + 2 + parts := strings.SplitN(path, "/", max) + + if len(parts) < max || parts[max-1] == "" { + return "index.html" + } + return parts[max-1] +} + func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string @@ -206,7 +214,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } - if strings.ToLower(req.URL.Path) == "/spec/head" { + if strings.HasPrefix(strings.ToLower(req.URL.Path), "/spec/head") { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { @@ -236,36 +244,59 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache = styledSpecCache } + var pathToContent map[string][]byte + if cached, ok := cache.Get(sha); ok { - w.Write(cached.([]byte)) - return - } + pathToContent = cached.(map[string][]byte) + } else { + dst, err := s.generateAt(sha) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, 500, err) + return + } - dst, err := s.generateAt(sha) - defer os.RemoveAll(dst) - if err != nil { - writeError(w, 500, err) - return - } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } - if styleLikeMatrixDotOrg { - cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) - cmd.Dir = path.Join(dst, "scripts") - var b bytes.Buffer - cmd.Stderr = &b - if err := cmd.Run(); err != nil { - writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) - return + fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen")) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading directory: %v", err)) + } + pathToContent = make(map[string][]byte) + for _, fi := range fis { + b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name())) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + return + } + pathToContent[fi.Name()] = b } + cache.Add(sha, pathToContent) } - b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) - if err != nil { - writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + requestedPath := extractPath(req.URL.Path, "/spec/pr") + if b, ok := pathToContent[requestedPath]; ok { + w.Write(b) return } - w.Write(b) - cache.Add(sha, b) + if requestedPath == "index.html" { + // Fall back to single-page spec for old PRs + if b, ok := pathToContent["specification.html"]; ok { + w.Write(b) + return + } + } + w.WriteHeader(404) + w.Write([]byte("Not found")) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -322,7 +353,7 @@ func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) { return } - diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + diffCmd := exec.Command("diff", "-r", "-u", path.Join(base, "scripts", "tmp"), path.Join(head, "scripts", "tmp")) var diff bytes.Buffer diffCmd.Stdout = &diff if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { @@ -366,7 +397,8 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html")) + requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b if err := cmd.Run(); err != nil { From b479b54cd887d0fd49a33458a77a51adaffc45bf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 18:52:58 +0000 Subject: [PATCH 139/177] Add tables of contents to individual specs --- specification/application_service_api.rst | 3 +++ specification/client_server_api.rst | 3 +++ specification/server_server_api.rst | 3 +++ 3 files changed, 9 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index cf2f9d57..bdf8a52e 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -12,6 +12,9 @@ irrespective of the underlying homeserver implementation. Add in Client-Server services? Overview of bots? Seems weird to be in the spec given it is VERY implementation specific. +.. contents:: Table of Contents +.. sectnum:: + Application Services -------------------- Application services are passive and can only observe events from a given diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 36ed2290..134c12c3 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,9 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +.. contents:: Table of Contents +.. sectnum:: + API Standards ------------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index cc9426e2..012c2ac9 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -41,6 +41,9 @@ EDUs and PDUs are further wrapped in an envelope called a Transaction, which is transferred from the origin to the destination home server using an HTTPS PUT request. +.. contents:: Table of Contents +.. sectnum:: + Server Discovery ---------------- From 6c66bfc75589392b61d67e1a37c3f11b66a5cf7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 11:22:57 +0000 Subject: [PATCH 140/177] Flatten out v1 and v2_alpha directories As a side effect, I got rid of all of the horrible symlinks and just put in all of the proper relative paths. Because the horrible symlinks were horrible. --- .../{v2_alpha => }/account-data.yaml | 0 .../{v1 => }/application_service.yaml | 0 api/client-server/{v1 => }/banning.yaml | 0 api/client-server/{v1 => }/content-repo.yaml | 0 api/client-server/{v1 => }/create_room.yaml | 0 .../{v1 => }/definitions/error.yaml | 0 .../{v2_alpha => }/definitions/event.json | 0 .../definitions/event_batch.json | 0 .../definitions/event_filter.json | 0 .../{v1 => }/definitions/push_condition.json | 0 .../{v1 => }/definitions/push_rule.json | 0 .../{v1 => }/definitions/push_ruleset.json | 0 .../definitions/room_event_filter.json | 2 +- .../definitions/sync_filter.json | 8 ++-- .../definitions/timeline_batch.json | 2 +- api/client-server/{v1 => }/directory.yaml | 0 api/client-server/{v2_alpha => }/filter.yaml | 0 api/client-server/{v1 => }/guest_events.yaml | 2 +- api/client-server/{v1 => }/inviting.yaml | 0 api/client-server/{v1 => }/joining.yaml | 0 api/client-server/{v1 => }/leaving.yaml | 0 .../{v1 => }/list_public_rooms.yaml | 0 api/client-server/{v1 => }/login.yaml | 0 .../{v1 => }/message_pagination.yaml | 0 .../{v1/sync.yaml => old_sync.yaml} | 14 +++--- api/client-server/{v1 => }/presence.yaml | 2 +- api/client-server/{v1 => }/profile.yaml | 0 api/client-server/{v1 => }/push_notifier.yaml | 0 api/client-server/{v1 => }/pusher.yaml | 0 api/client-server/{v1 => }/pushrules.yaml | 0 .../{v2_alpha => }/receipts.yaml | 0 .../{v2_alpha => }/registration.yaml | 0 api/client-server/{v1 => }/room_send.yaml | 0 api/client-server/{v1 => }/room_state.yaml | 0 api/client-server/{v1 => }/rooms.yaml | 10 ++--- api/client-server/{v1 => }/search.yaml | 2 +- api/client-server/{v2_alpha => }/sync.yaml | 0 api/client-server/{v2_alpha => }/tags.yaml | 0 .../{v1 => }/third_party_membership.yaml | 0 api/client-server/{v1 => }/typing.yaml | 0 api/client-server/v1/core-event-schema | 1 - api/client-server/v1/v1-event-schema | 1 - .../v2_alpha/definitions/definitions | 1 - .../v2_alpha/definitions/error.yaml | 10 ----- api/client-server/{v1 => }/voip.yaml | 0 event-schemas/README.md | 2 +- event-schemas/check.sh | 6 +-- event-schemas/examples/{v1 => }/m.call.answer | 0 .../examples/{v1 => }/m.call.candidates | 0 event-schemas/examples/{v1 => }/m.call.hangup | 0 event-schemas/examples/{v1 => }/m.call.invite | 0 event-schemas/examples/{v1 => }/m.presence | 0 event-schemas/examples/{v1 => }/m.receipt | 0 .../examples/{v1 => }/m.room.aliases | 0 event-schemas/examples/{v1 => }/m.room.avatar | 0 .../examples/{v1 => }/m.room.canonical_alias | 0 event-schemas/examples/{v1 => }/m.room.create | 0 .../examples/{v1 => }/m.room.guest_access | 0 .../{v1 => }/m.room.history_visibility | 0 .../examples/{v1 => }/m.room.join_rules | 0 event-schemas/examples/{v1 => }/m.room.member | 0 .../{v1 => }/m.room.member#invite_room_state | 0 .../{v1 => }/m.room.member#third_party_invite | 0 .../examples/{v1 => }/m.room.message#m.audio | 0 .../examples/{v1 => }/m.room.message#m.emote | 0 .../examples/{v1 => }/m.room.message#m.file | 0 .../examples/{v1 => }/m.room.message#m.image | 0 .../{v1 => }/m.room.message#m.location | 0 .../examples/{v1 => }/m.room.message#m.notice | 0 .../examples/{v1 => }/m.room.message#m.text | 0 .../examples/{v1 => }/m.room.message#m.video | 0 .../examples/{v1 => }/m.room.message.feedback | 0 event-schemas/examples/{v1 => }/m.room.name | 0 .../examples/{v1 => }/m.room.power_levels | 0 .../examples/{v1 => }/m.room.redaction | 0 .../{v1 => }/m.room.third_party_invite | 0 event-schemas/examples/{v1 => }/m.room.topic | 0 event-schemas/examples/{v1 => }/m.tag | 0 event-schemas/examples/{v1 => }/m.typing | 0 .../{v1 => }/core-event-schema/event.json | 0 .../msgtype_infos/image_info.json | 0 .../core-event-schema/room_event.json | 2 +- .../core-event-schema/state_event.json | 2 +- event-schemas/schema/{v1 => }/m.call.answer | 0 .../schema/{v1 => }/m.call.candidates | 0 event-schemas/schema/{v1 => }/m.call.hangup | 0 event-schemas/schema/{v1 => }/m.call.invite | 0 event-schemas/schema/{v1 => }/m.presence | 0 event-schemas/schema/{v1 => }/m.receipt | 0 event-schemas/schema/{v1 => }/m.room.aliases | 0 event-schemas/schema/{v1 => }/m.room.avatar | 0 .../schema/{v1 => }/m.room.canonical_alias | 0 event-schemas/schema/{v1 => }/m.room.create | 0 .../schema/{v1 => }/m.room.guest_access | 0 .../schema/{v1 => }/m.room.history_visibility | 0 .../schema/{v1 => }/m.room.join_rules | 0 event-schemas/schema/{v1 => }/m.room.member | 0 event-schemas/schema/{v1 => }/m.room.message | 0 .../schema/{v1 => }/m.room.message#m.audio | 0 .../schema/{v1 => }/m.room.message#m.emote | 0 .../schema/{v1 => }/m.room.message#m.file | 0 .../schema/{v1 => }/m.room.message#m.image | 0 .../schema/{v1 => }/m.room.message#m.location | 0 .../schema/{v1 => }/m.room.message#m.notice | 0 .../schema/{v1 => }/m.room.message#m.text | 0 .../schema/{v1 => }/m.room.message#m.video | 0 .../schema/{v1 => }/m.room.message.feedback | 0 event-schemas/schema/{v1 => }/m.room.name | 0 .../schema/{v1 => }/m.room.power_levels | 0 .../schema/{v1 => }/m.room.redaction | 0 .../schema/{v1 => }/m.room.third_party_invite | 0 event-schemas/schema/{v1 => }/m.room.topic | 0 event-schemas/schema/{v1 => }/m.tag | 0 event-schemas/schema/{v1 => }/m.typing | 0 .../v1/core-event-schema/core-event-schema | 1 - event-schemas/schema/v1/v1-event-schema | 1 - jenkins.sh | 2 +- scripts/generate-http-docs.sh | 2 +- specification/client_server_api.rst | 6 +-- specification/modules/account_data.rst | 2 +- specification/modules/receipts.rst | 2 +- specification/modules/tags.rst | 2 +- templating/matrix_templates/units.py | 44 +++++++------------ 123 files changed, 52 insertions(+), 77 deletions(-) rename api/client-server/{v2_alpha => }/account-data.yaml (100%) rename api/client-server/{v1 => }/application_service.yaml (100%) rename api/client-server/{v1 => }/banning.yaml (100%) rename api/client-server/{v1 => }/content-repo.yaml (100%) rename api/client-server/{v1 => }/create_room.yaml (100%) rename api/client-server/{v1 => }/definitions/error.yaml (100%) rename api/client-server/{v2_alpha => }/definitions/event.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_batch.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_filter.json (100%) rename api/client-server/{v1 => }/definitions/push_condition.json (100%) rename api/client-server/{v1 => }/definitions/push_rule.json (100%) rename api/client-server/{v1 => }/definitions/push_ruleset.json (100%) rename api/client-server/{v2_alpha => }/definitions/room_event_filter.json (92%) rename api/client-server/{v2_alpha => }/definitions/sync_filter.json (84%) rename api/client-server/{v2_alpha => }/definitions/timeline_batch.json (87%) rename api/client-server/{v1 => }/directory.yaml (100%) rename api/client-server/{v2_alpha => }/filter.yaml (100%) rename api/client-server/{v1 => }/guest_events.yaml (97%) rename api/client-server/{v1 => }/inviting.yaml (100%) rename api/client-server/{v1 => }/joining.yaml (100%) rename api/client-server/{v1 => }/leaving.yaml (100%) rename api/client-server/{v1 => }/list_public_rooms.yaml (100%) rename api/client-server/{v1 => }/login.yaml (100%) rename api/client-server/{v1 => }/message_pagination.yaml (100%) rename api/client-server/{v1/sync.yaml => old_sync.yaml} (96%) rename api/client-server/{v1 => }/presence.yaml (98%) rename api/client-server/{v1 => }/profile.yaml (100%) rename api/client-server/{v1 => }/push_notifier.yaml (100%) rename api/client-server/{v1 => }/pusher.yaml (100%) rename api/client-server/{v1 => }/pushrules.yaml (100%) rename api/client-server/{v2_alpha => }/receipts.yaml (100%) rename api/client-server/{v2_alpha => }/registration.yaml (100%) rename api/client-server/{v1 => }/room_send.yaml (100%) rename api/client-server/{v1 => }/room_state.yaml (100%) rename api/client-server/{v1 => }/rooms.yaml (97%) rename api/client-server/{v1 => }/search.yaml (97%) rename api/client-server/{v2_alpha => }/sync.yaml (100%) rename api/client-server/{v2_alpha => }/tags.yaml (100%) rename api/client-server/{v1 => }/third_party_membership.yaml (100%) rename api/client-server/{v1 => }/typing.yaml (100%) delete mode 120000 api/client-server/v1/core-event-schema delete mode 120000 api/client-server/v1/v1-event-schema delete mode 120000 api/client-server/v2_alpha/definitions/definitions delete mode 100644 api/client-server/v2_alpha/definitions/error.yaml rename api/client-server/{v1 => }/voip.yaml (100%) rename event-schemas/examples/{v1 => }/m.call.answer (100%) rename event-schemas/examples/{v1 => }/m.call.candidates (100%) rename event-schemas/examples/{v1 => }/m.call.hangup (100%) rename event-schemas/examples/{v1 => }/m.call.invite (100%) rename event-schemas/examples/{v1 => }/m.presence (100%) rename event-schemas/examples/{v1 => }/m.receipt (100%) rename event-schemas/examples/{v1 => }/m.room.aliases (100%) rename event-schemas/examples/{v1 => }/m.room.avatar (100%) rename event-schemas/examples/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/examples/{v1 => }/m.room.create (100%) rename event-schemas/examples/{v1 => }/m.room.guest_access (100%) rename event-schemas/examples/{v1 => }/m.room.history_visibility (100%) rename event-schemas/examples/{v1 => }/m.room.join_rules (100%) rename event-schemas/examples/{v1 => }/m.room.member (100%) rename event-schemas/examples/{v1 => }/m.room.member#invite_room_state (100%) rename event-schemas/examples/{v1 => }/m.room.member#third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.file (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.image (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.location (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.text (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.video (100%) rename event-schemas/examples/{v1 => }/m.room.message.feedback (100%) rename event-schemas/examples/{v1 => }/m.room.name (100%) rename event-schemas/examples/{v1 => }/m.room.power_levels (100%) rename event-schemas/examples/{v1 => }/m.room.redaction (100%) rename event-schemas/examples/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.topic (100%) rename event-schemas/examples/{v1 => }/m.tag (100%) rename event-schemas/examples/{v1 => }/m.typing (100%) rename event-schemas/schema/{v1 => }/core-event-schema/event.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/msgtype_infos/image_info.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/room_event.json (93%) rename event-schemas/schema/{v1 => }/core-event-schema/state_event.json (93%) rename event-schemas/schema/{v1 => }/m.call.answer (100%) rename event-schemas/schema/{v1 => }/m.call.candidates (100%) rename event-schemas/schema/{v1 => }/m.call.hangup (100%) rename event-schemas/schema/{v1 => }/m.call.invite (100%) rename event-schemas/schema/{v1 => }/m.presence (100%) rename event-schemas/schema/{v1 => }/m.receipt (100%) rename event-schemas/schema/{v1 => }/m.room.aliases (100%) rename event-schemas/schema/{v1 => }/m.room.avatar (100%) rename event-schemas/schema/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/schema/{v1 => }/m.room.create (100%) rename event-schemas/schema/{v1 => }/m.room.guest_access (100%) rename event-schemas/schema/{v1 => }/m.room.history_visibility (100%) rename event-schemas/schema/{v1 => }/m.room.join_rules (100%) rename event-schemas/schema/{v1 => }/m.room.member (100%) rename event-schemas/schema/{v1 => }/m.room.message (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.file (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.image (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.location (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.text (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.video (100%) rename event-schemas/schema/{v1 => }/m.room.message.feedback (100%) rename event-schemas/schema/{v1 => }/m.room.name (100%) rename event-schemas/schema/{v1 => }/m.room.power_levels (100%) rename event-schemas/schema/{v1 => }/m.room.redaction (100%) rename event-schemas/schema/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/schema/{v1 => }/m.room.topic (100%) rename event-schemas/schema/{v1 => }/m.tag (100%) rename event-schemas/schema/{v1 => }/m.typing (100%) delete mode 120000 event-schemas/schema/v1/core-event-schema/core-event-schema delete mode 120000 event-schemas/schema/v1/v1-event-schema diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/account-data.yaml similarity index 100% rename from api/client-server/v2_alpha/account-data.yaml rename to api/client-server/account-data.yaml diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/application_service.yaml similarity index 100% rename from api/client-server/v1/application_service.yaml rename to api/client-server/application_service.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/banning.yaml similarity index 100% rename from api/client-server/v1/banning.yaml rename to api/client-server/banning.yaml diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/content-repo.yaml similarity index 100% rename from api/client-server/v1/content-repo.yaml rename to api/client-server/content-repo.yaml diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/create_room.yaml similarity index 100% rename from api/client-server/v1/create_room.yaml rename to api/client-server/create_room.yaml diff --git a/api/client-server/v1/definitions/error.yaml b/api/client-server/definitions/error.yaml similarity index 100% rename from api/client-server/v1/definitions/error.yaml rename to api/client-server/definitions/error.yaml diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/definitions/event.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event.json rename to api/client-server/definitions/event.json diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/definitions/event_batch.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_batch.json rename to api/client-server/definitions/event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/definitions/event_filter.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_filter.json rename to api/client-server/definitions/event_filter.json diff --git a/api/client-server/v1/definitions/push_condition.json b/api/client-server/definitions/push_condition.json similarity index 100% rename from api/client-server/v1/definitions/push_condition.json rename to api/client-server/definitions/push_condition.json diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/definitions/push_rule.json similarity index 100% rename from api/client-server/v1/definitions/push_rule.json rename to api/client-server/definitions/push_rule.json diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/definitions/push_ruleset.json similarity index 100% rename from api/client-server/v1/definitions/push_ruleset.json rename to api/client-server/definitions/push_ruleset.json diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/definitions/room_event_filter.json similarity index 92% rename from api/client-server/v2_alpha/definitions/room_event_filter.json rename to api/client-server/definitions/room_event_filter.json index 86375781..02e5d0e0 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/definitions/room_event_filter.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref": "definitions/event_filter.json"}], + "allOf": [{"$ref": "event_filter.json"}], "properties": { "rooms": { "type": "array", diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/definitions/sync_filter.json similarity index 84% rename from api/client-server/v2_alpha/definitions/sync_filter.json rename to api/client-server/definitions/sync_filter.json index 0cd6a798..defc318a 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/definitions/sync_filter.json @@ -7,24 +7,24 @@ "state": { "description": "The state events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "timeline": { "description": "The message and state update events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "ephemeral": { "description": "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] } } }, "presence": { "description": "The presence updates to include.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "event_filter.json"}] }, "event_format": { "description": diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/definitions/timeline_batch.json similarity index 87% rename from api/client-server/v2_alpha/definitions/timeline_batch.json rename to api/client-server/definitions/timeline_batch.json index f27a5746..6f7e714a 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/event_batch.json"}], + "allOf": [{"$ref":"event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v1/directory.yaml b/api/client-server/directory.yaml similarity index 100% rename from api/client-server/v1/directory.yaml rename to api/client-server/directory.yaml diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/filter.yaml similarity index 100% rename from api/client-server/v2_alpha/filter.yaml rename to api/client-server/filter.yaml diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/guest_events.yaml similarity index 97% rename from api/client-server/v1/guest_events.yaml rename to api/client-server/guest_events.yaml index bbb5799a..671b355a 100644 --- a/api/client-server/v1/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -98,6 +98,6 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." diff --git a/api/client-server/v1/inviting.yaml b/api/client-server/inviting.yaml similarity index 100% rename from api/client-server/v1/inviting.yaml rename to api/client-server/inviting.yaml diff --git a/api/client-server/v1/joining.yaml b/api/client-server/joining.yaml similarity index 100% rename from api/client-server/v1/joining.yaml rename to api/client-server/joining.yaml diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/leaving.yaml similarity index 100% rename from api/client-server/v1/leaving.yaml rename to api/client-server/leaving.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml similarity index 100% rename from api/client-server/v1/list_public_rooms.yaml rename to api/client-server/list_public_rooms.yaml diff --git a/api/client-server/v1/login.yaml b/api/client-server/login.yaml similarity index 100% rename from api/client-server/v1/login.yaml rename to api/client-server/login.yaml diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/message_pagination.yaml similarity index 100% rename from api/client-server/v1/message_pagination.yaml rename to api/client-server/message_pagination.yaml diff --git a/api/client-server/v1/sync.yaml b/api/client-server/old_sync.yaml similarity index 96% rename from api/client-server/v1/sync.yaml rename to api/client-server/old_sync.yaml index 58cd5544..6dbe7e5e 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/old_sync.yaml @@ -84,7 +84,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -285,7 +285,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" rooms: type: array items: @@ -304,7 +304,7 @@ paths: title: "InviteEvent" description: "The invite event if ``membership`` is ``invite``" allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" messages: type: object title: PaginationChunk @@ -332,7 +332,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -345,7 +345,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -361,7 +361,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: @@ -398,6 +398,6 @@ paths: } schema: allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" 404: description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/presence.yaml b/api/client-server/presence.yaml similarity index 98% rename from api/client-server/v1/presence.yaml rename to api/client-server/presence.yaml index 5684398b..33df17d2 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/presence.yaml @@ -205,4 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" diff --git a/api/client-server/v1/profile.yaml b/api/client-server/profile.yaml similarity index 100% rename from api/client-server/v1/profile.yaml rename to api/client-server/profile.yaml diff --git a/api/client-server/v1/push_notifier.yaml b/api/client-server/push_notifier.yaml similarity index 100% rename from api/client-server/v1/push_notifier.yaml rename to api/client-server/push_notifier.yaml diff --git a/api/client-server/v1/pusher.yaml b/api/client-server/pusher.yaml similarity index 100% rename from api/client-server/v1/pusher.yaml rename to api/client-server/pusher.yaml diff --git a/api/client-server/v1/pushrules.yaml b/api/client-server/pushrules.yaml similarity index 100% rename from api/client-server/v1/pushrules.yaml rename to api/client-server/pushrules.yaml diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/receipts.yaml similarity index 100% rename from api/client-server/v2_alpha/receipts.yaml rename to api/client-server/receipts.yaml diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/registration.yaml similarity index 100% rename from api/client-server/v2_alpha/registration.yaml rename to api/client-server/registration.yaml diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/room_send.yaml similarity index 100% rename from api/client-server/v1/room_send.yaml rename to api/client-server/room_send.yaml diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/room_state.yaml similarity index 100% rename from api/client-server/v1/room_state.yaml rename to api/client-server/room_state.yaml diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/rooms.yaml similarity index 97% rename from api/client-server/v1/rooms.yaml rename to api/client-server/rooms.yaml index ff5587a9..f7654f14 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -173,7 +173,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" 403: description: > You aren't a member of the room and weren't previously a @@ -355,7 +355,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -368,7 +368,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -383,7 +383,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id"] 403: description: > @@ -453,7 +453,7 @@ paths: title: MemberEvent type: object allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" 403: description: > You aren't a member of the room and weren't previously a diff --git a/api/client-server/v1/search.yaml b/api/client-server/search.yaml similarity index 97% rename from api/client-server/v1/search.yaml rename to api/client-server/search.yaml index cc299056..3da9486f 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/search.yaml @@ -111,7 +111,7 @@ paths: title: Event description: The event that matched. allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" examples: application/json: |- { diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/sync.yaml similarity index 100% rename from api/client-server/v2_alpha/sync.yaml rename to api/client-server/sync.yaml diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/tags.yaml similarity index 100% rename from api/client-server/v2_alpha/tags.yaml rename to api/client-server/tags.yaml diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/third_party_membership.yaml similarity index 100% rename from api/client-server/v1/third_party_membership.yaml rename to api/client-server/third_party_membership.yaml diff --git a/api/client-server/v1/typing.yaml b/api/client-server/typing.yaml similarity index 100% rename from api/client-server/v1/typing.yaml rename to api/client-server/typing.yaml diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema deleted file mode 120000 index 045aecb0..00000000 --- a/api/client-server/v1/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/v1-event-schema b/api/client-server/v1/v1-event-schema deleted file mode 120000 index 7a0d0326..00000000 --- a/api/client-server/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1 \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions deleted file mode 120000 index 945c9b46..00000000 --- a/api/client-server/v2_alpha/definitions/definitions +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/error.yaml b/api/client-server/v2_alpha/definitions/error.yaml deleted file mode 100644 index 20312ae4..00000000 --- a/api/client-server/v2_alpha/definitions/error.yaml +++ /dev/null @@ -1,10 +0,0 @@ -type: object -description: A Matrix-level Error -properties: - errcode: - type: string - description: An error code. - error: - type: string - description: A human-readable error message. -required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/v1/voip.yaml b/api/client-server/voip.yaml similarity index 100% rename from api/client-server/v1/voip.yaml rename to api/client-server/voip.yaml diff --git a/event-schemas/README.md b/event-schemas/README.md index 1030a04c..950c3018 100644 --- a/event-schemas/README.md +++ b/event-schemas/README.md @@ -7,7 +7,7 @@ resolved correctly. For basic CLI testing, we recommend and have verified they work with the Node.js package [z-schema](https://github.com/zaggino/z-schema): ``` $ npm install -g z-schema - $ z-schema schema/v1/m.room.message examples/v1/m.room.message_m.text + $ z-schema schema/m.room.message examples/m.room.message_m.text schema validation passed json #1 validation passed ``` diff --git a/event-schemas/check.sh b/event-schemas/check.sh index a6d03b5a..3d411768 100755 --- a/event-schemas/check.sh +++ b/event-schemas/check.sh @@ -6,17 +6,17 @@ if ! which z-schema; then exit 1 fi -find schema/v1/m.* | while read line +find schema/m.* | while read line do split_path=(${line///// }) event_type=(${split_path[2]}) echo "Checking $event_type" echo "--------------------" # match exact name or exact name with a # - find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline + find examples -name $event_type -o -name "$event_type#*" | while read exline do echo " against $exline" # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/v1/$event_type $exline + z-schema schema/$event_type $exline done done diff --git a/event-schemas/examples/v1/m.call.answer b/event-schemas/examples/m.call.answer similarity index 100% rename from event-schemas/examples/v1/m.call.answer rename to event-schemas/examples/m.call.answer diff --git a/event-schemas/examples/v1/m.call.candidates b/event-schemas/examples/m.call.candidates similarity index 100% rename from event-schemas/examples/v1/m.call.candidates rename to event-schemas/examples/m.call.candidates diff --git a/event-schemas/examples/v1/m.call.hangup b/event-schemas/examples/m.call.hangup similarity index 100% rename from event-schemas/examples/v1/m.call.hangup rename to event-schemas/examples/m.call.hangup diff --git a/event-schemas/examples/v1/m.call.invite b/event-schemas/examples/m.call.invite similarity index 100% rename from event-schemas/examples/v1/m.call.invite rename to event-schemas/examples/m.call.invite diff --git a/event-schemas/examples/v1/m.presence b/event-schemas/examples/m.presence similarity index 100% rename from event-schemas/examples/v1/m.presence rename to event-schemas/examples/m.presence diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/m.receipt similarity index 100% rename from event-schemas/examples/v1/m.receipt rename to event-schemas/examples/m.receipt diff --git a/event-schemas/examples/v1/m.room.aliases b/event-schemas/examples/m.room.aliases similarity index 100% rename from event-schemas/examples/v1/m.room.aliases rename to event-schemas/examples/m.room.aliases diff --git a/event-schemas/examples/v1/m.room.avatar b/event-schemas/examples/m.room.avatar similarity index 100% rename from event-schemas/examples/v1/m.room.avatar rename to event-schemas/examples/m.room.avatar diff --git a/event-schemas/examples/v1/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias similarity index 100% rename from event-schemas/examples/v1/m.room.canonical_alias rename to event-schemas/examples/m.room.canonical_alias diff --git a/event-schemas/examples/v1/m.room.create b/event-schemas/examples/m.room.create similarity index 100% rename from event-schemas/examples/v1/m.room.create rename to event-schemas/examples/m.room.create diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/m.room.guest_access similarity index 100% rename from event-schemas/examples/v1/m.room.guest_access rename to event-schemas/examples/m.room.guest_access diff --git a/event-schemas/examples/v1/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility similarity index 100% rename from event-schemas/examples/v1/m.room.history_visibility rename to event-schemas/examples/m.room.history_visibility diff --git a/event-schemas/examples/v1/m.room.join_rules b/event-schemas/examples/m.room.join_rules similarity index 100% rename from event-schemas/examples/v1/m.room.join_rules rename to event-schemas/examples/m.room.join_rules diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/m.room.member similarity index 100% rename from event-schemas/examples/v1/m.room.member rename to event-schemas/examples/m.room.member diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state similarity index 100% rename from event-schemas/examples/v1/m.room.member#invite_room_state rename to event-schemas/examples/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.member#third_party_invite rename to event-schemas/examples/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.audio rename to event-schemas/examples/m.room.message#m.audio diff --git a/event-schemas/examples/v1/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.emote rename to event-schemas/examples/m.room.message#m.emote diff --git a/event-schemas/examples/v1/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.file rename to event-schemas/examples/m.room.message#m.file diff --git a/event-schemas/examples/v1/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.image rename to event-schemas/examples/m.room.message#m.image diff --git a/event-schemas/examples/v1/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.location rename to event-schemas/examples/m.room.message#m.location diff --git a/event-schemas/examples/v1/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.notice rename to event-schemas/examples/m.room.message#m.notice diff --git a/event-schemas/examples/v1/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.text rename to event-schemas/examples/m.room.message#m.text diff --git a/event-schemas/examples/v1/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.video rename to event-schemas/examples/m.room.message#m.video diff --git a/event-schemas/examples/v1/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback similarity index 100% rename from event-schemas/examples/v1/m.room.message.feedback rename to event-schemas/examples/m.room.message.feedback diff --git a/event-schemas/examples/v1/m.room.name b/event-schemas/examples/m.room.name similarity index 100% rename from event-schemas/examples/v1/m.room.name rename to event-schemas/examples/m.room.name diff --git a/event-schemas/examples/v1/m.room.power_levels b/event-schemas/examples/m.room.power_levels similarity index 100% rename from event-schemas/examples/v1/m.room.power_levels rename to event-schemas/examples/m.room.power_levels diff --git a/event-schemas/examples/v1/m.room.redaction b/event-schemas/examples/m.room.redaction similarity index 100% rename from event-schemas/examples/v1/m.room.redaction rename to event-schemas/examples/m.room.redaction diff --git a/event-schemas/examples/v1/m.room.third_party_invite b/event-schemas/examples/m.room.third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.third_party_invite rename to event-schemas/examples/m.room.third_party_invite diff --git a/event-schemas/examples/v1/m.room.topic b/event-schemas/examples/m.room.topic similarity index 100% rename from event-schemas/examples/v1/m.room.topic rename to event-schemas/examples/m.room.topic diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/m.tag similarity index 100% rename from event-schemas/examples/v1/m.tag rename to event-schemas/examples/m.tag diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/m.typing similarity index 100% rename from event-schemas/examples/v1/m.typing rename to event-schemas/examples/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/core-event-schema/event.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/event.json rename to event-schemas/schema/core-event-schema/event.json diff --git a/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json b/event-schemas/schema/core-event-schema/msgtype_infos/image_info.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json rename to event-schemas/schema/core-event-schema/msgtype_infos/image_info.json diff --git a/event-schemas/schema/v1/core-event-schema/room_event.json b/event-schemas/schema/core-event-schema/room_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/room_event.json rename to event-schemas/schema/core-event-schema/room_event.json index d5413f8a..80f7d265 100644 --- a/event-schemas/schema/v1/core-event-schema/room_event.json +++ b/event-schemas/schema/core-event-schema/room_event.json @@ -3,7 +3,7 @@ "title": "Room Event", "description": "In addition to the Event fields, Room Events MUST have the following additional field.", "allOf":[{ - "$ref": "core-event-schema/event.json" + "$ref": "event.json" }], "properties": { "room_id": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/core-event-schema/state_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/state_event.json rename to event-schemas/schema/core-event-schema/state_event.json index 5809cf7f..cfcb212c 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/core-event-schema/state_event.json @@ -3,7 +3,7 @@ "title": "State Event", "description": "In addition to the Room Event fields, State Events have the following additional fields.", "allOf":[{ - "$ref": "core-event-schema/room_event.json" + "$ref": "room_event.json" }], "properties": { "state_key": { diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/m.call.answer similarity index 100% rename from event-schemas/schema/v1/m.call.answer rename to event-schemas/schema/m.call.answer diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/m.call.candidates similarity index 100% rename from event-schemas/schema/v1/m.call.candidates rename to event-schemas/schema/m.call.candidates diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/m.call.hangup similarity index 100% rename from event-schemas/schema/v1/m.call.hangup rename to event-schemas/schema/m.call.hangup diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/m.call.invite similarity index 100% rename from event-schemas/schema/v1/m.call.invite rename to event-schemas/schema/m.call.invite diff --git a/event-schemas/schema/v1/m.presence b/event-schemas/schema/m.presence similarity index 100% rename from event-schemas/schema/v1/m.presence rename to event-schemas/schema/m.presence diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/m.receipt similarity index 100% rename from event-schemas/schema/v1/m.receipt rename to event-schemas/schema/m.receipt diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/m.room.aliases similarity index 100% rename from event-schemas/schema/v1/m.room.aliases rename to event-schemas/schema/m.room.aliases diff --git a/event-schemas/schema/v1/m.room.avatar b/event-schemas/schema/m.room.avatar similarity index 100% rename from event-schemas/schema/v1/m.room.avatar rename to event-schemas/schema/m.room.avatar diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/m.room.canonical_alias similarity index 100% rename from event-schemas/schema/v1/m.room.canonical_alias rename to event-schemas/schema/m.room.canonical_alias diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/m.room.create similarity index 100% rename from event-schemas/schema/v1/m.room.create rename to event-schemas/schema/m.room.create diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/m.room.guest_access similarity index 100% rename from event-schemas/schema/v1/m.room.guest_access rename to event-schemas/schema/m.room.guest_access diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/m.room.history_visibility similarity index 100% rename from event-schemas/schema/v1/m.room.history_visibility rename to event-schemas/schema/m.room.history_visibility diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/m.room.join_rules similarity index 100% rename from event-schemas/schema/v1/m.room.join_rules rename to event-schemas/schema/m.room.join_rules diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/m.room.member similarity index 100% rename from event-schemas/schema/v1/m.room.member rename to event-schemas/schema/m.room.member diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/m.room.message similarity index 100% rename from event-schemas/schema/v1/m.room.message rename to event-schemas/schema/m.room.message diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/m.room.message#m.audio similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.audio rename to event-schemas/schema/m.room.message#m.audio diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/m.room.message#m.emote similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.emote rename to event-schemas/schema/m.room.message#m.emote diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/m.room.message#m.file similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.file rename to event-schemas/schema/m.room.message#m.file diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/m.room.message#m.image similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.image rename to event-schemas/schema/m.room.message#m.image diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/m.room.message#m.location similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.location rename to event-schemas/schema/m.room.message#m.location diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/m.room.message#m.notice similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.notice rename to event-schemas/schema/m.room.message#m.notice diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/m.room.message#m.text similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.text rename to event-schemas/schema/m.room.message#m.text diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/m.room.message#m.video similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.video rename to event-schemas/schema/m.room.message#m.video diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/m.room.message.feedback similarity index 100% rename from event-schemas/schema/v1/m.room.message.feedback rename to event-schemas/schema/m.room.message.feedback diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/m.room.name similarity index 100% rename from event-schemas/schema/v1/m.room.name rename to event-schemas/schema/m.room.name diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/m.room.power_levels similarity index 100% rename from event-schemas/schema/v1/m.room.power_levels rename to event-schemas/schema/m.room.power_levels diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/m.room.redaction similarity index 100% rename from event-schemas/schema/v1/m.room.redaction rename to event-schemas/schema/m.room.redaction diff --git a/event-schemas/schema/v1/m.room.third_party_invite b/event-schemas/schema/m.room.third_party_invite similarity index 100% rename from event-schemas/schema/v1/m.room.third_party_invite rename to event-schemas/schema/m.room.third_party_invite diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/m.room.topic similarity index 100% rename from event-schemas/schema/v1/m.room.topic rename to event-schemas/schema/m.room.topic diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/m.tag similarity index 100% rename from event-schemas/schema/v1/m.tag rename to event-schemas/schema/m.tag diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/m.typing similarity index 100% rename from event-schemas/schema/v1/m.typing rename to event-schemas/schema/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema deleted file mode 120000 index 945c9b46..00000000 --- a/event-schemas/schema/v1/core-event-schema/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema deleted file mode 120000 index 945c9b46..00000000 --- a/event-schemas/schema/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/jenkins.sh b/jenkins.sh index f5ed3b15..abafefa8 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,7 +5,7 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) -(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") +(cd api && npm install && node validator.js -s "client-server") (cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index 0a2bc5f6..b3894a2c 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -18,7 +18,7 @@ This contains the client-server API for the reference implementation of the home EOF -for f in ../api/client-server/*/*.yaml; do +for f in ../api/client-server/*.yaml; do f="$(basename "${f/.yaml/_http_api}")" echo "{{${f/-/_}}}" >> tmp/http_apis done diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 134c12c3..f43daa7b 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -468,7 +468,7 @@ This section refers to API Version 2. These API calls currently use the prefix .. _User-Interactive Authentication: `sect:auth-api`_ -{{v2_registration_http_api}} +{{registration_http_api}} Old V1 API docs: |register|_ @@ -769,9 +769,9 @@ When the client first logs in, they will need to initially synchronise with their home server. This is achieved via the initial sync API described below. This API also returns an ``end`` token which can be used with the event stream. -{{sync_http_api}} +{{old_sync_http_api}} -{{v2_sync_http_api}} +{{sync_http_api}} Getting events for a room diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 784bc874..f3fc72b6 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -24,4 +24,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_account_data_http_api}} +{{account_data_http_api}} diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a8ad3cd3..702a7275 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -52,7 +52,7 @@ dismissing a notification in order for the event to count as "read". A client can update the markers for its user by interacting with the following HTTP APIs. -{{v2_receipts_http_api}} +{{receipts_http_api}} Server behaviour ---------------- diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index bb467bc0..f8c28c55 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -45,4 +45,4 @@ Two special names are listed in the specification: Client Behaviour ---------------- -{{v2_tags_http_api}} +{{tags_http_api}} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 75a12a5b..06994082 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -17,11 +17,10 @@ import subprocess import urllib import yaml -V1_CLIENT_API = "../api/client-server/v1" -V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" -V1_EVENT_SCHEMA = "../event-schemas/schema/v1" -V2_CLIENT_API = "../api/client-server/v2_alpha" -CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +HTTP_APIS = "../api/client-server" +V1_EVENT_EXAMPLES = "../event-schemas/examples" +V1_EVENT_SCHEMA = "../event-schemas/schema" +CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -549,30 +548,21 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - paths = [ - V1_CLIENT_API, V2_CLIENT_API - ] apis = {} - for path in paths: - is_v2 = (path == V2_CLIENT_API) - if not os.path.exists(V2_CLIENT_API): - self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) + path = HTTP_APIS + for filename in os.listdir(path): + if not filename.endswith(".yaml"): continue - for filename in os.listdir(path): - if not filename.endswith(".yaml"): - continue - self.log("Reading swagger API: %s" % filename) - filepath = os.path.join(path, filename) - with open(filepath, "r") as f: - # strip .yaml - group_name = filename[:-5].replace("-", "_") - if is_v2: - group_name = "v2_" + group_name - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta( - filepath, api, group_name - ) - apis[group_name] = api + self.log("Reading swagger API: %s" % filename) + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) + apis[group_name] = api return apis def load_common_event_fields(self): From c9c433bc15ede21a806900503df5d9a935984faa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 14:31:24 +0000 Subject: [PATCH 141/177] Make section depths consistent --- specification/intro.rst | 26 +++++++++++++------------- specification/targets.yaml | 8 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/specification/intro.rst b/specification/intro.rst index 8c08bf24..211a3562 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -9,7 +9,7 @@ https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} APIs -~~~~ +---- The following APIs are documented in this specification: - `Client-Server API `_ for writing Matrix clients. @@ -19,7 +19,7 @@ The following APIs are documented in this specification: There are also some `appendices `_. Changelog -~~~~~~~~~ +--------- {{spec_changelog}} For a full changelog, see @@ -29,7 +29,7 @@ https://github.com/matrix-org/matrix-doc/blob/master/CHANGELOG.rst .. sectnum:: Introduction -============ +------------ .. WARNING:: The Matrix specification is still evolving: the APIs are not yet frozen and this document is in places a work in progress or stale. We have made every @@ -103,10 +103,10 @@ reliably and persistently pushed from A to B in an inter-operable and federated manner. Overview -======== +-------- Architecture ------------- +~~~~~~~~~~~~ Matrix defines APIs for synchronising extensible JSON objects known as "events" between compatible clients, servers and services. Clients are @@ -158,7 +158,7 @@ a long-lived GET request. Users -~~~~~ ++++++ Each client is associated with a user account, which is identified in Matrix using a unique "User ID". This ID is namespaced to the homeserver which @@ -173,7 +173,7 @@ this user. The ``domain`` of a user ID is the domain of the homeserver. - Need to specify precise grammar for Matrix IDs Events -~~~~~~ +++++++ All data exchanged over Matrix is expressed as an "event". Typically each client action (e.g. sending a message) correlates with exactly one event. Each event @@ -188,7 +188,7 @@ of a "Room". .. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions Event Graphs -~~~~~~~~~~~~ +++++++++++++ .. _sect:event-graph: @@ -212,7 +212,7 @@ of its parents. The root event should have a depth of 1. Thus if one event is before another, then it must have a strictly smaller depth. Room structure -~~~~~~~~~~~~~~ +++++++++++++++ A room is a conceptual place where users can send and receive events. Events are sent to a room, and all participants in that room with sufficient access will @@ -291,7 +291,7 @@ from the other servers participating in a room. Room Aliases -++++++++++++ +^^^^^^^^^^^^ Each room can also have multiple "Room Aliases", which look like:: @@ -327,7 +327,7 @@ that are in the room that can be used to join via. |________________________________| Identity -~~~~~~~~ +++++++++ Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix @@ -347,7 +347,7 @@ user IDs using 3PIDs. Profiles -~~~~~~~~ +++++++++ Users may publish arbitrary key/value data associated with their account - such as a human readable display name, a profile photo URL, contact information @@ -358,7 +358,7 @@ as a human readable display name, a profile photo URL, contact information names allowed to be? Private User Data -~~~~~~~~~~~~~~~~~ ++++++++++++++++++ Users may also store arbitrary private key/value data in their account - such as client preferences, or server configuration settings which lack any other diff --git a/specification/targets.yaml b/specification/targets.yaml index cdbfa62e..310870a0 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -7,9 +7,9 @@ targets: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - - modules.rst - - { 1: feature_profiles.rst } - - { 1: "group:modules" } # reference a group of files + - { 1: modules.rst } + - { 2: feature_profiles.rst } + - { 2: "group:modules" } # reference a group of files application_service: files: - application_service_api.rst @@ -37,7 +37,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/account_data.rst -title_styles: ["=", "-", "~", "+", "^", "`"] +title_styles: ["=", "-", "~", "+", "^", "`", "@"] # The templating system doesn't know the right title style to use when generating # RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title From 79244e8065e989e409177180a43e8ea312d32739 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 16:40:48 +0000 Subject: [PATCH 142/177] Remove redundant node event schema checker The python one does the same --- event-schemas/check.sh | 22 ---------------------- event-schemas/check_examples.py | 2 +- jenkins.sh | 1 - 3 files changed, 1 insertion(+), 24 deletions(-) delete mode 100755 event-schemas/check.sh diff --git a/event-schemas/check.sh b/event-schemas/check.sh deleted file mode 100755 index 3d411768..00000000 --- a/event-schemas/check.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# Runs z-schema over all of the schema files (looking for matching examples) - -if ! which z-schema; then - echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema" - exit 1 -fi - -find schema/m.* | while read line -do - split_path=(${line///// }) - event_type=(${split_path[2]}) - echo "Checking $event_type" - echo "--------------------" - # match exact name or exact name with a # - find examples -name $event_type -o -name "$event_type#*" | while read exline - do - echo " against $exline" - # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/$event_type $exline - done -done diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index b10b2d0e..5a409407 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/env python import sys import json diff --git a/jenkins.sh b/jenkins.sh index abafefa8..c58ad473 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -6,7 +6,6 @@ set -ex (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server") -(cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} mkdir -p "${GOPATH}" From c6e0322a9e557c8fafa7d6f1fc894f253e5a2cbc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 13:53:03 +0000 Subject: [PATCH 143/177] Swaggerify /account --- api/client-server/administrative_contact.yaml | 157 ++++++++++++++++++ specification/client_server_api.rst | 53 +----- 2 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 api/client-server/administrative_contact.yaml diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml new file mode 100644 index 00000000..68fd0fba --- /dev/null +++ b/api/client-server/administrative_contact.yaml @@ -0,0 +1,157 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Account Administrative Contact API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/account/password": + post: + summary: Changes a user's password. + description: |- + This API endpoint uses the User-Interactive Authentication API. + An access token should be submitted to this endpoint if the client has + an active session. + The Home Server may change the flows available depending on whether a + valid access token is provided. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "new_password": "ihatebananas" + } + properties: + new_password: + type: string + description: The new password for the account. + required: ["new_password"] + responses: + 200: + description: The password has been changed. + examples: + application/json: "{}" + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/account/3pid": + get: + summary: Gets a list of a user's third party identifiers. + description: |- + Gets a list of the third party identifiers that the homeserver has + associated with the user's account. + + This is *not* the same as the list of third party identifiers bound to + the user's Matrix ID in Identity Servers. + + Identifiers in this list may be used by the Home Server as, for example, + identifiers that it will accept to reset the user's account password. + security: + - accessToken: [] + responses: + 200: + description: The lookup was successful. + examples: + application/json: |- + { + "threepids": [ + { + "medium": "email", + "address": "monkey@banana.island" + } + ] + } + schema: + type: object + properties: + threepids: + type: array + items: + type: object + title: Third party identifier + properties: + medium: + type: string + description: The medium of the third party identifier. + enum: ["email"] + address: + type: string + description: The third party identifier address. + post: + summary: Adds contact information to the user's account. + description: Adds contact information to the user's account. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + properties: + threePidCreds: + title: "ThreePidCredentials" + type: object + description: The third party credentials to associate with the account. + properties: + client_secret: + type: string + description: The client secret used in the session with the Identity Server. + id_server: + type: string + description: The Identity Server to use. + sid: + type: string + description: The session identifier given by the Identity Server. + required: ["client_secret", "id_server", "sid"] + bind: + type: boolean + description: |- + Whether the home server should also bind this third party + identifier to the account's Matrix ID with the passed identity + server. Default: ``false``. + x-example: true + required: ["threePidCreds"] + example: |- + { + "threePidCreds": { + "id_server": "matrix.org", + "sid": "abc123987", + "client_secret": "d0n'tT3ll" + }, + "bind": false + } + responses: + 200: + description: The addition was successful. + examples: + application/json: "{}" + schema: + type: object + 403: + description: The credentials could not be verified with the identity server. + examples: + application/json: |- + { + "errcode": "M_THREEPID_AUTH_FAILED", + "error": "The third party credentials could not be verified by the identity server." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index f43daa7b..67973c6b 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -510,58 +510,11 @@ database. Adding Account Administrative Contact Information +++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - POST $V2PREFIX/account/3pid - -Used to add contact information to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing contact information. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The contact information object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifier. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Contact Information -+++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: +A homeserver may keep some contact information for administrative use. +This is independent of any information kept by any Identity Servers. -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address +{{administrative_contact_http_api}} Pagination ---------- From c4eaf7458ff8489697aae8683256c2d8f4bcee80 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:19:31 +0000 Subject: [PATCH 144/177] 302 to spec/head/index.html rather than serving it on /spec/head Otherwise relative links are broken --- scripts/speculator/main.go | 46 +++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3736672f..c32eec94 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -193,15 +193,31 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } -func extractPath(path, base string) string { - // Assume exactly one flat directory - max := strings.Count(base, "/") + 2 +// extractPath extracts the file path within the gen directory which should be served for the request. +// Returns one of (file to serve, path to redirect to). +// path is the actual path being requested, e.g. "/spec/head/client_server.html". +// base is the base path of the handler, including a trailing slash, before the PR number, e.g. "/spec/". +func extractPath(path, base string) (string, string) { + // Assumes exactly one flat directory + + // Count slashes in /spec/head/client_server.html + // base is /spec/ + // +1 for the PR number - /spec/head + // +1 for the path-part after the slash after the PR number + max := strings.Count(base, "/") + 1 parts := strings.SplitN(path, "/", max) - if len(parts) < max || parts[max-1] == "" { - return "index.html" + if len(parts) < max { + // Path is base/pr - redirect to base/pr/index.html + return "", path + "/index.html" } - return parts[max-1] + if parts[max-1] == "" { + // Path is base/pr/ - serve index.html + return "index.html", "" + } + + // Path is base/pr/file.html - serve file + return parts[max-1], "" } func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { @@ -283,7 +299,11 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache.Add(sha, pathToContent) } - requestedPath := extractPath(req.URL.Path, "/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } if b, ok := pathToContent[requestedPath]; ok { w.Write(b) return @@ -299,6 +319,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { w.Write([]byte("Not found")) } +func (s *server) redirectTo(w http.ResponseWriter, req *http.Request, path string) { + req.URL.Path = path + w.Header().Set("Location", req.URL.String()) + w.WriteHeader(302) +} + // lookupHeadSHA looks up what origin/master's HEAD SHA is. // It attempts to `git fetch` before doing so. // If this fails, it may still return a stale sha, but will also return an error. @@ -397,7 +423,11 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/diff/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b From a4668c1d8c69124e029b456bc6b7cf13dfac0385 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:21:20 +0000 Subject: [PATCH 145/177] Fix typo --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index c32eec94..602cdb57 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -204,7 +204,7 @@ func extractPath(path, base string) (string, string) { // base is /spec/ // +1 for the PR number - /spec/head // +1 for the path-part after the slash after the PR number - max := strings.Count(base, "/") + 1 + max := strings.Count(base, "/") + 2 parts := strings.SplitN(path, "/", max) if len(parts) < max { From ebed3b60bd68ef8a258ec9ebbd866477283297c9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 15:58:15 +0000 Subject: [PATCH 146/177] Add trailing slashes to links --- scripts/speculator/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 602cdb57..1c2780e3 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -469,12 +469,12 @@ func listPulls(w http.ResponseWriter, req *http.Request) { } s := "
    " for _, pull := range pulls { - s += fmt.Sprintf(`
  • %d: %s: %s: spec spec diff rst diff
    • `, + s += fmt.Sprintf(`
  • %d: %s: %s: spec spec diff rst diff
    • `, pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } - s += `
` + s += `` if *includesDir != "" { - s += `` + s += `` } io.WriteString(w, s) From 681c25820602b543264eb81f341284f6d4671ab8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 16:56:28 +0000 Subject: [PATCH 147/177] Fix turnServer endpoint --- api/client-server/voip.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/voip.yaml b/api/client-server/voip.yaml index 5fdf1ca7..629f2c17 100644 --- a/api/client-server/voip.yaml +++ b/api/client-server/voip.yaml @@ -18,7 +18,7 @@ securityDefinitions: name: access_token in: query paths: - "/turnServer": + "/voip/turnServer": get: summary: Obtain TURN server credentials. description: |- From f6229f649a88a2ec365f270896d9917ab6e86244 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 16:58:40 +0000 Subject: [PATCH 148/177] Remove confusing wording --- .../modules/typing_notifications.rst | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index da383e73..8be1ac28 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -34,28 +34,6 @@ to inform the server that the user has stopped typing. {{typing_http_api}} -Server behaviour ----------------- - -Servers MUST emit typing EDUs in a different form to ``m.typing`` events which -are shown to clients. This form looks like:: - - { - "type": "m.typing", - "content": { - "room_id": "!room-id-here:matrix.org", - "user_id": "@user-id-here:matrix.org", - "typing": true/false - } - } - -This does not contain timing information so it is up to originating homeservers -to ensure they eventually send "stop" notifications. - -.. TODO - ((This will eventually need addressing, as part of the wider typing/presence - timer addition work)) - Security considerations ----------------------- From 97fd1fdd627c45522cf888ec5d858c067933b02c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:02:23 +0000 Subject: [PATCH 149/177] Embed client and server release numbers Note that this also removes the changelog - I'm going to re-add the changelog differently soon. --- scripts/gendoc.py | 35 +++++++++++++++++++++---- scripts/generate-http-docs.sh | 13 ++++++++- specification/intro.rst | 20 ++++---------- templating/build.py | 22 ++++++++++++++-- templating/matrix_templates/sections.py | 4 --- templating/matrix_templates/units.py | 21 +++++---------- 6 files changed, 73 insertions(+), 42 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 9b87cb30..801c06bb 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -250,7 +250,7 @@ def addAnchors(path): f.write(line + "\n") -def run_through_template(input, set_verbose): +def run_through_template(input, set_verbose, substitutions): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: @@ -260,6 +260,9 @@ def run_through_template(input, set_verbose): "-o", "../scripts/tmp", "../scripts/"+input ] + for k, v in substitutions.items(): + args.append("--substitution=%s=%s" % (k, v)) + if set_verbose: args.insert(2, "-v") log("EXEC: %s" % " ".join(args)) @@ -392,7 +395,7 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(requested_target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates, substitutions): prepare_env() log("Building spec [target=%s]" % requested_target_name) @@ -407,7 +410,7 @@ def main(requested_target_name, keep_intermediates): target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename=templated_file) - run_through_template(templated_file, VERBOSE) + run_through_template(templated_file, VERBOSE, substitutions) fix_relative_titles( target=target, filename=templated_file, out_filename=rst_file, @@ -417,13 +420,21 @@ def main(requested_target_name, keep_intermediates): if requested_target_name == "all": shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() +def extract_major(s): + major_version = s + match = re.match("^(r\d)+(\.\d+)?$", s) + if match: + major_version = match.group(1) + return major_version + + if __name__ == '__main__': parser = ArgumentParser( "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." @@ -441,9 +452,23 @@ if __name__ == '__main__': "--verbose", "-v", action="store_true", help="Turn on verbose mode." ) + parser.add_argument( + "--client_release", "-c", action="store", default="unstable", + help="The client-server release tag to generate, e.g. r1.2" + ) + parser.add_argument( + "--server_release", "-s", action="store", default="unstable", + help="The server-server release tag to generate, e.g. r1.2" + ) args = parser.parse_args() if not args.target: parser.print_help() sys.exit(1) VERBOSE = args.verbose - main(args.target, args.nodelete) + substitutions = { + "%CLIENT_RELEASE_LABEL%": args.client_release, + "%CLIENT_MAJOR_VERSION%": extract_major(args.client_release), + "%SERVER_RELEASE_LABEL%": args.server_release, + "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + } + main(args.target, args.nodelete, substitutions) diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index b3894a2c..48c01bdb 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -4,6 +4,17 @@ # It takes all of the swagger YAML files for the client-server API, and turns # them into API docs, with none of the narrative found in the rst files which # normally wrap these API docs. +# +# Optionally takes one positional argument, the label of the release, e.g. r1.2. +# This falls back to "unstable" if unspecified. + +if [[ $# == 1 ]]; then + client_release=$1 +else + client_release="unstable" +fi + +client_major_version="$(echo "${client_release}" | perl -pe 's/^(r\d+)\..*$/$1/g')" cd "$(dirname $0)" @@ -23,5 +34,5 @@ for f in ../api/client-server/*.yaml; do echo "{{${f/-/_}}}" >> tmp/http_apis done -(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis) +(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis --substitution=%CLIENT_RELEASE_LABEL%="${client_release}" --substitution=%CLIENT_MAJOR_VERSION%="${client_major_version}") rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html diff --git a/specification/intro.rst b/specification/intro.rst index 211a3562..eb6f98fc 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -1,30 +1,20 @@ Matrix Specification ==================== -Version: {{spec_version}} ------------------------------ This specification has been generated from https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of -revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +revision ``{{git_version}}`` - +https://github.com/matrix-org/matrix-doc/tree/{{git_rev}}. -APIs ----- The following APIs are documented in this specification: -- `Client-Server API `_ for writing Matrix clients. -- `Server-Server API `_ for writing servers which can federate with Matrix. -- `Application Service API `_ for writing privileged plugins to servers. +- `Client-Server API `_ version %CLIENT_RELEASE_LABEL% for writing Matrix clients. +- `Server-Server API `_ version %SERVER_RELEASE_LABEL% for writing servers which can federate with Matrix. +- `Application Service API `_ version %CLIENT_RELEASE_LABEL% for writing privileged plugins to servers. There are also some `appendices `_. -Changelog ---------- -{{spec_changelog}} - -For a full changelog, see -https://github.com/matrix-org/matrix-doc/blob/master/CHANGELOG.rst - .. contents:: Table of Contents .. sectnum:: diff --git a/templating/build.py b/templating/build.py index 10fb5aea..f08c2201 100755 --- a/templating/build.py +++ b/templating/build.py @@ -44,6 +44,7 @@ import importlib import json import logging import os +import re import sys from textwrap import TextWrapper @@ -56,7 +57,7 @@ def check_unaccessed(name, store): log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log(unaccessed_keys) -def main(input_module, file_stream=None, out_dir=None, verbose=False): +def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): if out_dir and not os.path.exists(out_dir): os.makedirs(out_dir) @@ -167,6 +168,12 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): temp = Template(temp_str) log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) + + # Do these substitutions outside of the ordinary templating system because + # we want them to apply to things like the underlying swagger used to + # generate the templates, not just the top-level sections. + for old, new in substitutions.items(): + output = output.replace(old, new) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: @@ -209,6 +216,10 @@ if __name__ == '__main__': "--verbose", "-v", action="store_true", help="Turn on verbose mode." ) + parser.add_argument( + "--substitution", action="append", + help="Substitutions to apply to the generated output, of form NEEDLE=REPLACEMENT." + ) args = parser.parse_args() if args.verbose: @@ -226,7 +237,14 @@ if __name__ == '__main__': parser.print_help() sys.exit(1) + substitutions = {} + for substitution in args.substitution: + parts = substitution.split("=", 1) + if len(parts) != 2: + raise Exception("Invalid substitution") + substitutions[parts[0]] = parts[1] + main( args.input, file_stream=args.file, out_dir=args.out_directory, - verbose=args.verbose + substitutions=substitutions, verbose=args.verbose ) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 78aabca7..d285bb61 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -15,10 +15,6 @@ class MatrixSections(Sections): def render_git_rev(self): return self.units.get("git_version")["revision"] - def render_spec_version(self): - spec_meta = self.units.get("spec_meta") - return spec_meta["version"] - def render_spec_changelog(self): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 06994082..feae0aae 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -18,8 +18,8 @@ import urllib import yaml HTTP_APIS = "../api/client-server" -V1_EVENT_EXAMPLES = "../event-schemas/examples" -V1_EVENT_SCHEMA = "../event-schemas/schema" +EVENT_EXAMPLES = "../event-schemas/examples" +EVENT_SCHEMA = "../event-schemas/schema" CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -605,7 +605,7 @@ class MatrixUnits(Units): return event_types def load_event_examples(self): - path = V1_EVENT_EXAMPLES + path = EVENT_EXAMPLES examples = {} for filename in os.listdir(path): if not filename.startswith("m."): @@ -622,7 +622,7 @@ class MatrixUnits(Units): return examples def load_event_schemas(self): - path = V1_EVENT_SCHEMA + path = EVENT_SCHEMA schemata = {} for filename in os.listdir(path): @@ -714,7 +714,6 @@ class MatrixUnits(Units): def load_spec_meta(self): path = CHANGELOG title_part = None - version = None changelog_lines = [] with open(path, "r") as f: prev_line = None @@ -738,19 +737,11 @@ class MatrixUnits(Units): break changelog_lines.append(line) - # parse out version from title - for word in title_part.split(): - if re.match("^v[0-9\.]+$", word): - version = word[1:] # strip the 'v' - - self.log("Version: %s Title part: %s Changelog line count: %s" % ( - version, title_part, len(changelog_lines) + self.log("Title part: %s Changelog line count: %s" % ( + title_part, len(changelog_lines) )) - if not version or len(changelog_lines) == 0: - raise Exception("Failed to parse CHANGELOG.rst") return { - "version": version, "changelog": "".join(changelog_lines) } From aa4ed10821220c0f1b43541ea61d611f1993908d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:23:58 +0000 Subject: [PATCH 150/177] Unify v1 and v2 event schemas --- api/client-server/guest_events.yaml | 2 +- api/client-server/old_sync.yaml | 19 ++++---- api/client-server/presence.yaml | 3 +- api/client-server/rooms.yaml | 31 ++++++------- api/client-server/search.yaml | 2 +- event-schemas/examples/m.call.answer | 2 +- event-schemas/examples/m.call.candidates | 2 +- event-schemas/examples/m.call.hangup | 2 +- event-schemas/examples/m.call.invite | 2 +- event-schemas/examples/m.room.aliases | 2 +- event-schemas/examples/m.room.avatar | 2 +- event-schemas/examples/m.room.canonical_alias | 2 +- event-schemas/examples/m.room.create | 2 +- event-schemas/examples/m.room.guest_access | 2 +- .../examples/m.room.history_visibility | 2 +- event-schemas/examples/m.room.join_rules | 2 +- event-schemas/examples/m.room.member | 2 +- .../examples/m.room.member#invite_room_state | 2 +- .../examples/m.room.member#third_party_invite | 2 +- event-schemas/examples/m.room.message#m.audio | 4 +- event-schemas/examples/m.room.message#m.emote | 2 +- event-schemas/examples/m.room.message#m.file | 4 +- event-schemas/examples/m.room.message#m.image | 2 +- .../examples/m.room.message#m.location | 2 +- .../examples/m.room.message#m.notice | 2 +- event-schemas/examples/m.room.message#m.text | 2 +- event-schemas/examples/m.room.message#m.video | 4 +- .../examples/m.room.message.feedback | 2 +- event-schemas/examples/m.room.name | 2 +- event-schemas/examples/m.room.power_levels | 2 +- event-schemas/examples/m.room.redaction | 2 +- event-schemas/examples/m.room.topic | 2 +- event-schemas/examples/m.typing | 2 +- .../schema/core-event-schema/room_event.json | 34 +++++++++++---- specification/events.rst | 43 ++----------------- 35 files changed, 87 insertions(+), 109 deletions(-) diff --git a/api/client-server/guest_events.yaml b/api/client-server/guest_events.yaml index 671b355a..4d7a957d 100644 --- a/api/client-server/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -74,7 +74,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ] } diff --git a/api/client-server/old_sync.yaml b/api/client-server/old_sync.yaml index 6dbe7e5e..fe76e3bc 100644 --- a/api/client-server/old_sync.yaml +++ b/api/client-server/old_sync.yaml @@ -60,7 +60,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ] } @@ -154,7 +154,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 343511809, @@ -166,7 +166,7 @@ paths: "origin_server_ts": 1432804487480, "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "type": "m.room.message", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" } ], "end": "s3456_9_0", @@ -184,13 +184,12 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 6547561012, "content": { "avatar_url": "mxc://localhost/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:localhost", @@ -199,7 +198,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "@alice:localhost", "type": "m.room.member", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 7148267200, @@ -211,7 +210,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.create", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" }, { "age": 1622568720, @@ -226,7 +225,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "@bob:localhost", "type": "m.room.member", - "user_id": "@bob:localhost" + "sender": "@bob:localhost" }, { "age": 7148267004, @@ -250,7 +249,7 @@ paths: "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:localhost" + "sender": "@alice:localhost" } ], "visibility": "private", @@ -392,7 +391,7 @@ paths: "msgtype": "m.text" }, "room_id:": "!wfgy43Sg4a:matrix.org", - "user_id": "@bob:matrix.org", + "sender": "@bob:matrix.org", "event_id": "$asfDuShaf7Gafaw:matrix.org", "type": "m.room.message" } diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index 33df17d2..6ac204dd 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -85,8 +85,7 @@ paths: application/json: |- { "presence": "unavailable", - "last_active_ago": 420845, - "status_msg": null + "last_active_ago": 420845 } schema: type: object diff --git a/api/client-server/rooms.yaml b/api/client-server/rooms.yaml index f7654f14..cc3aecc4 100644 --- a/api/client-server/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -92,13 +92,12 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -107,7 +106,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 7148267200, @@ -119,7 +118,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -134,7 +133,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" }, { "age": 7148267004, @@ -158,7 +157,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" } ] schema: @@ -212,7 +211,7 @@ paths: "origin_server_ts": 1432804485886, "room_id": "!636q39766251:example.com", "type": "m.room.message", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 343511809, @@ -224,7 +223,7 @@ paths: "origin_server_ts": 1432804487480, "room_id": "!636q39766251:example.com", "type": "m.room.message", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" } ], "end": "s3456_9_0", @@ -242,13 +241,12 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -257,7 +255,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 7148267200, @@ -269,7 +267,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -284,7 +282,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" }, { "age": 7148267004, @@ -308,7 +306,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" } ], "visibility": "private", @@ -416,7 +414,6 @@ paths: "age": 6547561012, "content": { "avatar_url": "mxc://example.com/fzysBrHpPEeTGANCVLXWXNMI#auto", - "displayname": null, "membership": "join" }, "event_id": "$1426600438280zExKY:example.com", @@ -425,7 +422,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", - "user_id": "@alice:example.com" + "sender": "@alice:example.com" }, { "age": 1622568720, @@ -440,7 +437,7 @@ paths: "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", - "user_id": "@bob:example.com" + "sender": "@bob:example.com" } ] } diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 3da9486f..8a8c3f5a 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -131,7 +131,7 @@ paths: "origin_server_ts": 1444298308034, "room_id": "!qPewotXpIctQySfjSy:localhost", "type": "m.room.message", - "user_id": "@test:localhost" + "sender": "@test:localhost" } } } diff --git a/event-schemas/examples/m.call.answer b/event-schemas/examples/m.call.answer index 0301fb3b..f7d14439 100644 --- a/event-schemas/examples/m.call.answer +++ b/event-schemas/examples/m.call.answer @@ -13,5 +13,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.answer", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.candidates b/event-schemas/examples/m.call.candidates index 389d2271..8e6849bb 100644 --- a/event-schemas/examples/m.call.candidates +++ b/event-schemas/examples/m.call.candidates @@ -15,5 +15,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.candidates", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.hangup b/event-schemas/examples/m.call.hangup index 52ddad16..42e1f346 100644 --- a/event-schemas/examples/m.call.hangup +++ b/event-schemas/examples/m.call.hangup @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.hangup", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.call.invite b/event-schemas/examples/m.call.invite index 2df25929..974a5b4c 100644 --- a/event-schemas/examples/m.call.invite +++ b/event-schemas/examples/m.call.invite @@ -13,5 +13,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.call.invite", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.aliases b/event-schemas/examples/m.room.aliases index 07b4b330..ca87510e 100644 --- a/event-schemas/examples/m.room.aliases +++ b/event-schemas/examples/m.room.aliases @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.aliases", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.avatar b/event-schemas/examples/m.room.avatar index 9fb1189c..2080d96e 100644 --- a/event-schemas/examples/m.room.avatar +++ b/event-schemas/examples/m.room.avatar @@ -14,5 +14,5 @@ "type": "m.room.avatar", "state_key": "", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias index 0203a851..59df586d 100644 --- a/event-schemas/examples/m.room.canonical_alias +++ b/event-schemas/examples/m.room.canonical_alias @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.canonical_alias", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.create b/event-schemas/examples/m.room.create index a3598853..34dabb53 100644 --- a/event-schemas/examples/m.room.create +++ b/event-schemas/examples/m.room.create @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.create", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.guest_access b/event-schemas/examples/m.room.guest_access index 8ad4d294..c636ff39 100644 --- a/event-schemas/examples/m.room.guest_access +++ b/event-schemas/examples/m.room.guest_access @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEG:localhost", "type": "m.room.guest_access", "room_id": "!Cuyf34gef24u:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility index fcc3f881..6fedc5dc 100644 --- a/event-schemas/examples/m.room.history_visibility +++ b/event-schemas/examples/m.room.history_visibility @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.history_visibility", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.join_rules b/event-schemas/examples/m.room.join_rules index f22ad97e..39e14fc5 100644 --- a/event-schemas/examples/m.room.join_rules +++ b/event-schemas/examples/m.room.join_rules @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.join_rules", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member b/event-schemas/examples/m.room.member index e2ca5668..133cad96 100644 --- a/event-schemas/examples/m.room.member +++ b/event-schemas/examples/m.room.member @@ -26,5 +26,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state index e2ca5668..133cad96 100644 --- a/event-schemas/examples/m.room.member#invite_room_state +++ b/event-schemas/examples/m.room.member#invite_room_state @@ -26,5 +26,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite index 2457302a..1bae2e9f 100644 --- a/event-schemas/examples/m.room.member#third_party_invite +++ b/event-schemas/examples/m.room.member#third_party_invite @@ -21,5 +21,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.member", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio index b57adc9a..367eb954 100644 --- a/event-schemas/examples/m.room.message#m.audio +++ b/event-schemas/examples/m.room.message#m.audio @@ -14,5 +14,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote index bd984831..4280928e 100644 --- a/event-schemas/examples/m.room.message#m.emote +++ b/event-schemas/examples/m.room.message#m.emote @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file index 7ac36c90..e52c3a94 100644 --- a/event-schemas/examples/m.room.message#m.file +++ b/event-schemas/examples/m.room.message#m.file @@ -14,5 +14,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image index ba405a1a..91e72be2 100644 --- a/event-schemas/examples/m.room.message#m.image +++ b/event-schemas/examples/m.room.message#m.image @@ -15,5 +15,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location index 54198acb..fcbeb97e 100644 --- a/event-schemas/examples/m.room.message#m.location +++ b/event-schemas/examples/m.room.message#m.location @@ -16,5 +16,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice index 2e5fff97..978c67e6 100644 --- a/event-schemas/examples/m.room.message#m.notice +++ b/event-schemas/examples/m.room.message#m.notice @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text index d877ee64..e00c7aa5 100644 --- a/event-schemas/examples/m.room.message#m.text +++ b/event-schemas/examples/m.room.message#m.text @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video index e3da9077..576d80de 100644 --- a/event-schemas/examples/m.room.message#m.video +++ b/event-schemas/examples/m.room.message#m.video @@ -23,5 +23,5 @@ "origin_server_ts": 1432735824653, "room_id": "!jEsUZKDJdhlrceRyVU:localhost", "type": "m.room.message", - "user_id": "@example:localhost" -} \ No newline at end of file + "sender": "@example:localhost" +} diff --git a/event-schemas/examples/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback index 6282305b..16fe0ee0 100644 --- a/event-schemas/examples/m.room.message.feedback +++ b/event-schemas/examples/m.room.message.feedback @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.message.feedback", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.name b/event-schemas/examples/m.room.name index 636119b8..87db2008 100644 --- a/event-schemas/examples/m.room.name +++ b/event-schemas/examples/m.room.name @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.name", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 8278597a..42ef5a5d 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -20,5 +20,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.power_levels", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.redaction b/event-schemas/examples/m.room.redaction index 2e8260ea..5f4aae1e 100644 --- a/event-schemas/examples/m.room.redaction +++ b/event-schemas/examples/m.room.redaction @@ -8,5 +8,5 @@ "type": "m.room.redaction", "room_id": "!Cuyf34gef24t:localhost", "redacts": "!fukweghifu23:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.room.topic b/event-schemas/examples/m.room.topic index 8a469122..65daa987 100644 --- a/event-schemas/examples/m.room.topic +++ b/event-schemas/examples/m.room.topic @@ -8,5 +8,5 @@ "event_id": "$WLGTSEFSEF:localhost", "type": "m.room.topic", "room_id": "!Cuyf34gef24t:localhost", - "user_id": "@example:localhost" + "sender": "@example:localhost" } diff --git a/event-schemas/examples/m.typing b/event-schemas/examples/m.typing index bd53f6fb..1d2c517b 100644 --- a/event-schemas/examples/m.typing +++ b/event-schemas/examples/m.typing @@ -4,4 +4,4 @@ "content": { "user_ids": ["@alice:matrix.org", "@bob:example.com"] } -} \ No newline at end of file +} diff --git a/event-schemas/schema/core-event-schema/room_event.json b/event-schemas/schema/core-event-schema/room_event.json index 80f7d265..09cb2478 100644 --- a/event-schemas/schema/core-event-schema/room_event.json +++ b/event-schemas/schema/core-event-schema/room_event.json @@ -1,23 +1,41 @@ { "type": "object", "title": "Room Event", - "description": "In addition to the Event fields, Room Events MUST have the following additional field.", + "description": "In addition to the Event fields, Room Events may have the following additional fields.", "allOf":[{ "$ref": "event.json" }], "properties": { - "room_id": { + "event_id": { "type": "string", - "description": "The ID of the room associated with this event." + "description": "Required. The globally unique event identifier." }, - "event_id": { + "room_id": { "type": "string", - "description": "The globally unique event identifier." + "description": "Required. The ID of the room associated with this event." }, - "user_id": { + "sender": { "type": "string", - "description": "Contains the fully-qualified ID of the user who *sent* this event." + "description": "Required. Contains the fully-qualified ID of the user who *sent* this event." + }, + "unsigned": { + "type": "object", + "description": "Contains optional extra information about the event.", + "properties": { + "age": { + "type": "integer", + "description": "The time in milliseconds that has elapsed since the event was sent" + }, + "redacted_because": { + "type": "string", + "description": "The reason this event was redacted, if it was redacted" + }, + "transaction_id": { + "type": "string", + "description": "The client-supplied transaction ID, if the client being given the event is the same one which sent it." + } + } } }, - "required": ["room_id"] + "required": ["event_id", "room_id", "sender"] } diff --git a/specification/events.rst b/specification/events.rst index 5a003115..92d52d22 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -5,6 +5,9 @@ All communication in Matrix is expressed in the form of data objects called Events. These are the fundamental building blocks common to the client-server, server-server and application-service APIs, and are described below. +Note that the structure of these events may be different than those in the +server-server API. + {{common_event_fields}} {{common_room_event_fields}} @@ -12,51 +15,13 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} -Differences between /v1 and /v2 events --------------------------------------- - -There are a few differences between how events are formatted for sending -between servers over federation and how they are formatted for sending between -a server and its clients. - -Additionally there are a few differences between the format of events in the -responses to client APIs with a /v1 prefix and responses APIs with a /v2 -prefix. - -Events in responses for APIs with the /v2 prefix are generated from an event -formatted for federation by: - -* Removing the following keys: - ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``. -* Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has elapsed since the event was sent. -* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the - event is a ``state event``, which give the previous content and previous - sender of that state key -* Adding a ``redacted_because`` to the ``unsigned`` object if the event was - redacted which gives the event that redacted it. -* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent - by the client requesting it. - -Events in responses for APIs with the /v1 prefix are generated from an event -formatted for the /v2 prefix by: - -* Moving the folling keys from the ``unsigned`` object to the top level event - object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. -* Removing the ``unsigned`` object. -* Rename the ``sender`` key to ``user_id``. -* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` - then adding a ``invite_room_state`` key to the top level event object. - - Size limits ----------- The total size of any event MUST NOT exceed 65 KB. There are additional restrictions on sizes per key: -- ``user_id`` MUST NOT exceed 255 bytes (including domain). +- ``sender`` MUST NOT exceed 255 bytes (including domain). - ``room_id`` MUST NOT exceed 255 bytes. - ``state_key`` MUST NOT exceed 255 bytes. - ``type`` MUST NOT exceed 255 bytes. From 7f07da4b507f6d0c20a8b17cab23276d9d050d7f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 17:29:09 +0000 Subject: [PATCH 151/177] Remove obsolete documentation --- api/client-server/definitions/event.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/client-server/definitions/event.json b/api/client-server/definitions/event.json index 5a8f52f6..98aac21e 100644 --- a/api/client-server/definitions/event.json +++ b/api/client-server/definitions/event.json @@ -39,10 +39,6 @@ "type": "object", "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." }, - "replaces_state": { - "type": "string", - "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, "transaction_id": { "type": "string", "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." From b946d398864343180ca96837f56916d726bbb8cb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 2 Dec 2015 01:25:07 +0000 Subject: [PATCH 152/177] typo --- specification/modules/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/search.rst b/specification/modules/search.rst index 089953d2..655b1ed3 100644 --- a/specification/modules/search.rst +++ b/specification/modules/search.rst @@ -34,7 +34,7 @@ The supported keys to search over are: The search will *not* include rooms that are end to end encrypted. The results include a ``rank`` key that can be used to sort the results by -revelancy. The higher the ``rank`` the more relevant the result is. +relevancy. The higher the ``rank`` the more relevant the result is. The value of ``count`` may not match the number of results. For example due to the search query matching 1000s of results and the server truncating the From 1ce33579270f86ef2c85601a5eb6423fdd009d9f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 14:53:52 +0000 Subject: [PATCH 153/177] continuserv: serve multiple paths --- scripts/continuserv/main.go | 44 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 14d0c5bf..a1295077 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -26,7 +26,7 @@ var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") mu sync.Mutex // Prevent multiple updates in parallel. - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero. wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. wg sync.WaitGroup // Indicates how many updates are pending. @@ -116,14 +116,29 @@ func serve(w http.ResponseWriter, req *http.Request) { wgMu.Lock() wg.Wait() wgMu.Unlock() - b := toServe.Load().(bytesOrErr) - if b.err != nil { + + file := req.URL.Path + if file[0] == '/' { + file = file[1:] + } + if file == "" { + file = "index.html" + } + m := toServe.Load().(bytesOrErr) + if m.err != nil { w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(b.err.Error())) - } else { + w.Write([]byte(m.err.Error())) + return + } + b, ok := m.bytes[file] + if ok { w.Header().Set("Content-Type", "text/html") - w.Write([]byte(b.bytes)) + w.Write([]byte(b)) + return } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(404) + w.Write([]byte("Not found")) } func populateOnce(dir string) { @@ -139,12 +154,21 @@ func populateOnce(dir string) { toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } - specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + fis, err := ioutil.ReadDir(path.Join(dir, "scripts", "gen")) if err != nil { - toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) + toServe.Store(bytesOrErr{nil, err}) return } - toServe.Store(bytesOrErr{specBytes, nil}) + files := make(map[string][]byte) + for _, fi := range fis { + bytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", fi.Name())) + if err != nil { + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) + return + } + files[fi.Name()] = bytes + } + toServe.Store(bytesOrErr{files, nil}) } func doPopulate(ch chan struct{}, dir string) { @@ -173,6 +197,6 @@ func exists(path string) bool { } type bytesOrErr struct { - bytes []byte + bytes map[string][]byte // filename -> contents err error } From e072d215cf466e76552a6eb53dc9e1cea8a4838c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 15:27:25 +0000 Subject: [PATCH 154/177] Specify redaction --- api/client-server/redaction.yaml | 82 +++++++++++++++++++++++++++++ specification/client_server_api.rst | 17 +++--- 2 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 api/client-server/redaction.yaml diff --git a/api/client-server/redaction.yaml b/api/client-server/redaction.yaml new file mode 100644 index 00000000..d30ef8c1 --- /dev/null +++ b/api/client-server/redaction.yaml @@ -0,0 +1,82 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server message redaction API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/redact/{eventId}/{txnId}": + put: + summary: Strips all non-integrity-critical information out of an event. + description: |- + Strips all information out of an event which isn't critical to the + integrity of the server-side representation of the room. + + This cannot be undone. + + Users may redact their own events, and any user with a power level + greater than or equal to the `redact` power level of the room may + redact events there. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room from which to redact the event. + required: true + x-example: "!637q39766251:example.com" + - in: path + type: string + name: eventId + description: The ID of the event to redact + required: true + x-example: "bai2b1i9:matrix.org" + - in: path + name: txnId + type: string + description: |- + The transaction ID for this event. Clients should generate a + unique ID; it will be used by the server to ensure idempotency of requests. + required: true + x-example: "37" + - in: body + name: body + schema: + type: object + example: |- + { + "reason": "Indecent material" + } + properties: + reason: + type: string + description: The reason for the event being redacted. + responses: + 200: + description: "An ID for the redaction event." + examples: + application/json: |- + { + "event_id": "YUwQidLecu" + } + schema: + type: object + properties: + event_id: + type: string + description: |- + A unique identifier for the event. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 67973c6b..4c24c350 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -793,10 +793,6 @@ owners to delete the offending content from the databases. Events that have been redacted include a ``redacted_because`` key whose value is the event that caused it to be redacted, which may include a reason. -.. TODO - Currently, only room admins can redact events by sending a ``m.room.redaction`` - event, but server admins also need to be able to redact events by a similar - mechanism. Upon receipt of a redaction event, the server should strip off any keys not in the following list: @@ -819,13 +815,20 @@ one of the following event types: ``kick``, ``redact``, ``state_default``, ``users``, ``users_default``. - ``m.room.aliases`` allows key ``aliases`` -.. TODO - Need to update m.room.power_levels to reflect new power levels formatting - The redaction event should be added under the key ``redacted_because``. When a client receives a redaction event it should change the redacted event in the same way a server does. +Events +++++++ + +{{m_room_redaction_event}} + +Client behaviour +++++++++++++++++ + +{{redaction_http_api}} + Rooms ----- From 2f3a00fe34b2414484ab2c2265c571d84db9328b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 2 Dec 2015 19:23:33 +0000 Subject: [PATCH 155/177] Consistently spell homeserver as homeserver --- api/client-server/administrative_contact.yaml | 6 +- api/client-server/create_room.yaml | 2 +- api/client-server/inviting.yaml | 2 +- api/client-server/login.yaml | 2 +- api/client-server/pusher.yaml | 2 +- api/client-server/registration.yaml | 4 +- api/client-server/third_party_membership.yaml | 4 +- attic/v1_registration_login.rst | 48 ++++++++-------- ...ient_federated_versioning_design_notes.rst | 8 +-- drafts/as-http-api.rst | 42 +++++++------- drafts/erikj_federation.rst | 6 +- drafts/general_api.rst | 54 +++++++++--------- drafts/human-id-rules.rst | 14 ++--- drafts/model/presence.rst | 12 ++-- drafts/model/profiles.rst | 16 +++--- drafts/model/room-join-workflow.rst | 12 ++-- drafts/model/rooms.rst | 20 +++---- drafts/model/third-party-id.rst | 2 +- scripts/generate-http-docs.sh | 2 +- specification/application_service_api.rst | 14 ++--- specification/client_server_api.rst | 56 +++++++++---------- specification/intro.rst | 4 +- specification/modules/presence.rst | 4 +- specification/modules/push.rst | 2 +- specification/server_server_api.rst | 50 ++++++++--------- .../examples/application-services.rst | 8 +-- .../guides/2015-08-10-client-server.rst | 16 +++--- .../guides/2015-08-21-application_services.md | 18 +++--- supporting-docs/howtos/client-server.rst | 16 +++--- .../jsfiddles/create_room_send_msg/demo.html | 2 +- .../jsfiddles/create_room_send_msg/demo.js | 2 +- .../howtos/jsfiddles/event_stream/demo.html | 2 +- .../howtos/jsfiddles/event_stream/demo.js | 2 +- .../howtos/jsfiddles/example_app/demo.html | 2 +- .../howtos/jsfiddles/example_app/demo.js | 4 +- .../howtos/jsfiddles/register_login/demo.html | 2 +- .../howtos/jsfiddles/register_login/demo.js | 4 +- .../jsfiddles/room_memberships/demo.html | 2 +- .../howtos/jsfiddles/room_memberships/demo.js | 2 +- 39 files changed, 235 insertions(+), 235 deletions(-) diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index 68fd0fba..7e3f8bce 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -25,7 +25,7 @@ paths: This API endpoint uses the User-Interactive Authentication API. An access token should be submitted to this endpoint if the client has an active session. - The Home Server may change the flows available depending on whether a + The homeserver may change the flows available depending on whether a valid access token is provided. security: - accessToken: [] @@ -64,7 +64,7 @@ paths: This is *not* the same as the list of third party identifiers bound to the user's Matrix ID in Identity Servers. - Identifiers in this list may be used by the Home Server as, for example, + Identifiers in this list may be used by the homeserver as, for example, identifiers that it will accept to reset the user's account password. security: - accessToken: [] @@ -126,7 +126,7 @@ paths: bind: type: boolean description: |- - Whether the home server should also bind this third party + Whether the homeserver should also bind this third party identifier to the account's Matrix ID with the passed identity server. Default: ``false``. x-example: true diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 051c4b84..20c00f6c 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -57,7 +57,7 @@ paths: description: |- The desired room alias **local part**. If this is included, a room alias will be created and mapped to the newly created - room. The alias will belong on the *same* home server which + room. The alias will belong on the *same* homeserver which created the room. For example, if this was set to "foo" and sent to the homeserver "example.com" the complete room alias would be ``#foo:example.com``. diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 0a028710..d9b0ad10 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -38,7 +38,7 @@ paths: Only users currently in a particular room can invite other users to join that room. - If the user was invited to the room, the home server will append a + If the user was invited to the room, the homeserver will append a ``m.room.member`` event to the room. .. _third party invites section: `invite-by-third-party-id-endpoint`_ diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index bbab46df..f2e3d044 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -81,7 +81,7 @@ paths: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. home_server: type: string - description: The hostname of the Home Server on which the account has been registered. + description: The hostname of the homeserver on which the account has been registered. 400: description: |- Part of the request was invalid. For example, the login type may not be recognised. diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 8c243f2b..ed6c4f91 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -114,7 +114,7 @@ paths: description: |- If true, the homeserver should add another pusher with the given pushkey and App ID in addition to any others with - different user IDs. Otherwise, the Home Server must remove any + different user IDs. Otherwise, the homeserver must remove any other pushers with the same App ID and pushkey for different users. The default is ``false``. required: ['profile_tag', 'kind', 'app_id', 'app_display_name', diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 681654ae..7bb49f2c 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -94,7 +94,7 @@ paths: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint. home_server: type: string - description: The hostname of the Home Server on which the account has been registered. + description: The hostname of the homeserver on which the account has been registered. 400: description: |- Part of the request was invalid. This may include one of the following error codes: @@ -107,7 +107,7 @@ paths: including after authentication if the requested user ID was registered whilst the client was performing authentication. - Home Servers MUST perform the relevant checks and return these codes before + Homeservers MUST perform the relevant checks and return these codes before performing `User-Interactive Authentication`_, although they may also return them after authentication is completed if, for example, the requested user ID was registered whilst the client was performing authentication. diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index a10d6167..e9c4d2f2 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -39,7 +39,7 @@ paths: join that room. If the identity server did know the Matrix user identifier for the - third party identifier, the home server will append a ``m.room.member`` + third party identifier, the homeserver will append a ``m.room.member`` event to the room. If the identity server does not know a Matrix user identifier for the @@ -62,7 +62,7 @@ paths: - The matrix user ID who invited them to the room - If a token is requested from the identity server, the home server will + If a token is requested from the identity server, the homeserver will append a ``m.room.third_party_invite`` event to the room. .. _joining rooms section: `invite-by-user-id-endpoint`_ diff --git a/attic/v1_registration_login.rst b/attic/v1_registration_login.rst index 4b80600b..1364319b 100644 --- a/attic/v1_registration_login.rst +++ b/attic/v1_registration_login.rst @@ -1,17 +1,17 @@ Registration and Login ---------------------- -Clients must register with a home server in order to use Matrix. After +Clients must register with a homeserver in order to use Matrix. After registering, the client will be given an access token which must be used in ALL -requests to that home server as a query parameter 'access_token'. +requests to that homeserver as a query parameter 'access_token'. If the client has already registered, they need to be able to login to their -account. The home server may provide many different ways of logging in, such as +account. The homeserver may provide many different ways of logging in, such as user/password auth, login via a social network (OAuth2), login by confirming a token sent to their email address, etc. This specification does not define how -home servers should authorise their users who want to login to their existing +homeservers should authorise their users who want to login to their existing accounts, but instead defines the standard interface which implementations -should follow so that ANY client can login to ANY home server. Clients login +should follow so that ANY client can login to ANY homeserver. Clients login using the |login|_ API. Clients register using the |register|_ API. Registration follows the same general procedure as login, but the path requests are sent to and the details contained in them are different. @@ -26,7 +26,7 @@ In order to determine up-front what the server's requirements are, the client can request from the server a complete description of all of its acceptable flows of the registration or login process. It can then inspect the list of returned flows looking for one for which it believes it can complete all of the -required stages, and perform it. As each home server may have different ways of +required stages, and perform it. As each homeserver may have different ways of logging in, the client needs to know how they should login. All distinct login stages MUST have a corresponding ``type``. A ``type`` is a namespaced string which details the mechanism for logging in. @@ -64,12 +64,12 @@ ID and a new access token MUST be returned:: "access_token": "abcdef0123456789" } -The ``user_id`` key is particularly useful if the home server wishes to support +The ``user_id`` key is particularly useful if the homeserver wishes to support localpart entry of usernames (e.g. "user" rather than "@user:matrix.org"), as the client may not be able to determine its ``user_id`` in this case. -If the flow has multiple stages to it, the home server may wish to create a -session to store context between requests. If a home server responds with a +If the flow has multiple stages to it, the homeserver may wish to create a +session to store context between requests. If a homeserver responds with a ``session`` key to a request, clients MUST submit it in subsequent requests until the flow is completed:: @@ -99,7 +99,7 @@ To respond to this type, reply with:: "password": "" } -The home server MUST respond with either new credentials, the next stage of the +The homeserver MUST respond with either new credentials, the next stage of the login process, or a standard error response. Captcha-based @@ -123,7 +123,7 @@ To respond to this type, reply with:: Recaptcha.get_challenge(); Recaptcha.get_response(); -The home server MUST respond with either new credentials, the next stage of the +The homeserver MUST respond with either new credentials, the next stage of the login process, or a standard error response. OAuth2-based @@ -146,24 +146,24 @@ The server MUST respond with:: "uri": } -The home server acts as a 'confidential' client for the purposes of OAuth2. If +The homeserver acts as a 'confidential' client for the purposes of OAuth2. If the uri is a ``sevice selection URI``, it MUST point to a webpage which prompts the user to choose which service to authorize with. On selection of a service, this MUST link through to an ``Authorization Request URI``. If there is only 1 -service which the home server accepts when logging in, this indirection can be +service which the homeserver accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the OAuth2 Allow/Deny prompt. Hitting 'Allow' returns the ``redirect URI`` with the -auth code. Home servers can choose any path for the ``redirect URI``. The +auth code. Homeservers can choose any path for the ``redirect URI``. The client should visit the ``redirect URI``, which will then finish the OAuth2 -login process, granting the home server an access token for the chosen service. -When the home server gets this access token, it verifies that the cilent has +login process, granting the homeserver an access token for the chosen service. +When the homeserver gets this access token, it verifies that the cilent has authorised with the 3rd party, and can now complete the login. The OAuth2 ``redirect URI`` (with auth code) MUST respond with either new credentials, the next stage of the login process, or a standard error response. -For example, if a home server accepts OAuth2 from Google, it would return the +For example, if a homeserver accepts OAuth2 from Google, it would return the Authorization Request URI for Google:: { @@ -171,7 +171,7 @@ Authorization Request URI for Google:: client_id=CLIENT_ID&redirect_uri=REDIRECT_URI&scope=photos" } -The client then visits this URI and authorizes the home server. The client then +The client then visits this URI and authorizes the homeserver. The client then visits the REDIRECT_URI with the auth code= query parameter which returns:: { @@ -195,7 +195,7 @@ To respond to this type, reply with:: "email": "" } -After validating the email address, the home server MUST send an email +After validating the email address, the homeserver MUST send an email containing an authentication code and return:: { @@ -212,7 +212,7 @@ code:: "code": "" } -The home server MUST respond to this with either new credentials, the next +The homeserver MUST respond to this with either new credentials, the next stage of the login process, or a standard error response. Email-based (url) @@ -231,7 +231,7 @@ To respond to this type, reply with:: "email": "" } -After validating the email address, the home server MUST send an email +After validating the email address, the homeserver MUST send an email containing an authentication URL and return:: { @@ -247,7 +247,7 @@ client should perform another request:: "session": "" } -The home server MUST respond to this with either new credentials, the next +The homeserver MUST respond to this with either new credentials, the next stage of the login process, or a standard error response. A common client implementation will be to periodically poll until the link is @@ -264,7 +264,7 @@ Email-based (identity server) Prior to submitting this, the client should authenticate with an identity server. After authenticating, the session information should be submitted to -the home server. +the homeserver. To respond to this type, reply with:: @@ -293,7 +293,7 @@ of a previous login stage:: "next": "" } -If a home server implements N-factor authentication, it MUST respond with all +If a homeserver implements N-factor authentication, it MUST respond with all ``stages`` when initially queried for their login requirements:: { diff --git a/drafts/ancient_federated_versioning_design_notes.rst b/drafts/ancient_federated_versioning_design_notes.rst index ffda6063..036989eb 100644 --- a/drafts/ancient_federated_versioning_design_notes.rst +++ b/drafts/ancient_federated_versioning_design_notes.rst @@ -1,11 +1,11 @@ -Versioning is, like, hard for backfilling backwards because of the number of Home Servers involved. +Versioning is, like, hard for backfilling backwards because of the number of homeservers involved. -The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For backfilling purposes, this is done on a per context basis. +The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For backfilling purposes, this is done on a per context basis. When we send a PDU we include all PDUs that have been received for that context that hasn't been subsequently listed in a later PDU. The trivial case is a simple list of PDUs, e.g. A <- B <- C. However, if two servers send out a PDU at the same to, both B and C would point at A - a later PDU would then list both B and C. Problems with opaque version strings: - - How do you do clustering without mandating that a cluster can only have one transaction in flight to a given remote home server at a time. + - How do you do clustering without mandating that a cluster can only have one transaction in flight to a given remote homeserver at a time. If you have multiple transactions sent at once, then you might drop one transaction, receive another with a version that is later than the dropped transaction and which point ARGH WE LOST A TRANSACTION. - - How do you do backfilling? A version string defines a point in a stream w.r.t. a single home server, not a point in the context. + - How do you do backfilling? A version string defines a point in a stream w.r.t. a single homeserver, not a point in the context. We only need to store the ends of the directed graph, we DO NOT need to do the whole one table of nodes and one of edges. diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 2c8f8e37..4f8d6bd4 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -13,9 +13,9 @@ Application Services HTTP API .. sectnum:: -Application Service -> Home Server +Application Service -> Homeserver ---------------------------------- -This contains home server APIs which are used by the application service. +This contains homeserver APIs which are used by the application service. Registration API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -34,7 +34,7 @@ Output: Side effects: - The HS will start delivering events to the URL base specified if this 200s. API called when: - - The application service wants to register with a brand new home server. + - The application service wants to register with a brand new homeserver. Notes: - An application service can state whether they should be the only ones who can manage a specified namespace. This is referred to as an "exclusive" @@ -100,7 +100,7 @@ Notes: Unregister API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API unregisters a previously registered AS from the home server. +This API unregisters a previously registered AS from the homeserver. Inputs: - AS token @@ -122,9 +122,9 @@ API called when: } -Home Server -> Application Service +Homeserver -> Application Service ---------------------------------- -This contains application service APIs which are used by the home server. +This contains application service APIs which are used by the homeserver. User Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~ @@ -152,9 +152,9 @@ Notes: - This is deemed more flexible than alternative methods (e.g. returning a JSON blob with the user's display name and get the HS to provision the user). Retry notes: - - The home server cannot respond to the client's request until the response to + - The homeserver cannot respond to the client's request until the response to this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a + - Recommended that homeservers try a few times then time out, returning a 408 Request Timeout to the client. :: @@ -199,9 +199,9 @@ Notes: style JSON blob and get the HS to provision the room). It also means that the AS knows the room ID -> alias mapping. Retry notes: - - The home server cannot respond to the client's request until the response to + - The homeserver cannot respond to the client's request until the response to this API is obtained from the AS. - - Recommended that home servers try a few times then time out, returning a + - Recommended that homeservers try a few times then time out, returning a 408 Request Timeout to the client. :: @@ -236,13 +236,13 @@ Data flows: :: Typical - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <--- : AS sends back 200 OK. AS ACK Lost - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. - HS ---> AS : Home server retries with the same transaction ID of T. + HS ---> AS : Homeserver retries with the same transaction ID of T. <--- : AS sends back 200 OK. If the AS had processed these events already, it can NO-OP this request (and it knows if it is the same events based on the transacton ID). @@ -253,15 +253,15 @@ Retry notes: - Since ASes by definition cannot alter the traffic being passed to it (unlike say, a Policy Server), these requests can be done in parallel to general HS processing; the HS doesn't need to block whilst doing this. - - Home servers should use exponential backoff as their retry algorithm. - - Home servers MUST NOT alter (e.g. add more) events they were going to + - Homeservers should use exponential backoff as their retry algorithm. + - Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID on retries, as the AS may have already processed the events. Ordering notes: - The events sent to the AS should be linearised, as they are from the event stream. - - The home server will need to maintain a queue of transactions to send to + - The homeserver will need to maintain a queue of transactions to send to the AS. :: @@ -336,7 +336,7 @@ Notes: Server admin style permissions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The home server needs to give the application service *full control* over its +The homeserver needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as create/delete any user in its namespace. No additional API changes need to be @@ -451,15 +451,15 @@ Examples IRC ~~~ Pre-conditions: - - Server admin stores the AS token "T_a" on the home server. - - Home server has a token "T_h". - - Home server has the domain "hsdomain.com" + - Server admin stores the AS token "T_a" on the homeserver. + - Homeserver has a token "T_h". + - Homeserver has the domain "hsdomain.com" 1. Application service registration :: - AS -> HS: Registers itself with the home server + AS -> HS: Registers itself with the homeserver POST /register { url: "https://someapp.com/matrix", diff --git a/drafts/erikj_federation.rst b/drafts/erikj_federation.rst index 0644f271..ce089def 100644 --- a/drafts/erikj_federation.rst +++ b/drafts/erikj_federation.rst @@ -237,7 +237,7 @@ Domain specific string ``room_id`` A domain specific string with prefix ``!`` that is static across all events in a graph and uniquely identifies it. The ``domain`` should be that of the - home server that created the room (i.e., the server that generated the + homeserver that created the room (i.e., the server that generated the first ``m.room.create`` event). ``sender`` @@ -246,7 +246,7 @@ Domain specific string User Id A domain specific string with prefix ``@`` representing a user account. The - ``domain`` is the home server of the user and is the server used to contact + ``domain`` is the homeserver of the user and is the server used to contact the user. Joining a room @@ -280,7 +280,7 @@ can then process the join event itself. Inviting a user --------------- -To invite a remote user to a room we need their home server to sign the invite +To invite a remote user to a room we need their homeserver to sign the invite event. This is done by sending the event to the remote server, which then signs the event, before distributing the invite to other servers. diff --git a/drafts/general_api.rst b/drafts/general_api.rst index ae2446d6..baad3901 100644 --- a/drafts/general_api.rst +++ b/drafts/general_api.rst @@ -61,7 +61,7 @@ This version will change the path prefix for HTTP: - Version 2: ``/_matrix/client/v2`` Note the lack of the ``api`` segment. This is for consistency between other -home server path prefixes. +homeserver path prefixes. Terminology: - ``Chunk token`` : An opaque string which can be used to return another chunk @@ -169,16 +169,16 @@ Outputs: ``content`` key. Deleted message events are ``m.room.redaction`` events. - New position in the stream. (chunk token) State Events Ordering Notes: - - Home servers may receive state events over federation that are superceded by - state events previously sent to the client. The home server *cannot* send + - Homeservers may receive state events over federation that are superceded by + state events previously sent to the client. The homeserver *cannot* send these events to the client else they would end up erroneously clobbering the superceding state event. - - As a result, the home server reserves the right to omit sending state events + - As a result, the homeserver reserves the right to omit sending state events which are known to be superceded already. - This may result in missed *state* events. However, the state of the room will always be eventually consistent. Message Events Ordering Notes: - - Home servers may receive message events over federation that happened a long + - Homeservers may receive message events over federation that happened a long time ago. The client may or may not be interested in these message events. - For clients which do not store scrollback for a room (they discard events after processing them), this is not a problem as they only care about the @@ -191,11 +191,11 @@ Message Events Ordering Notes: - The event, when it comes down the stream, will indicate which event it comes after. Rejected events: - - A home server may find out via federation that it should not have accepted + - A homeserver may find out via federation that it should not have accepted an event (e.g. to send a message/state event in a room). For example, it may - send an event to another home server and receive an auth event stating + send an event to another homeserver and receive an auth event stating that the event should not have been sent. - - If this happens, the home server will send a ``m.room.redaction`` for the + - If this happens, the homeserver will send a ``m.room.redaction`` for the event in question. This will be a local server event (not shared with other servers). - If the event was a state event, it will synthesise a new state event to @@ -206,7 +206,7 @@ Unknown rooms: - You could receive events for rooms you are unaware of (e.g. you didn't do an initial sync, or your HS lost its database and is told from another HS that they are in this room). How do you handle this? - - The simplest option would be to redo the initial sync with a filter on the + - The simplest option would be to redo the initial sync with a filter on the room ID you're unaware of. This would retrieve the room state so you can display the room. What data flows does it address: @@ -291,7 +291,7 @@ Scrollback API ``[Draft]`` but as a purely informational display thing it would be nice. Additional Inputs: - - flag to say if the home server should do a backfill over federation + - flag to say if the homeserver should do a backfill over federation Additional Outputs: - whether there are more events on the local HS / over federation. What data flows does it address: @@ -313,7 +313,7 @@ Additional Outputs: Room Alias API ``[Draft]`` -------------------------- This provides mechanisms for creating and removing room aliases for a room on a -home server. Typically, any user in a room can make an alias for that room. The +homeserver. Typically, any user in a room can make an alias for that room. The alias creator (or anyone in the room?) can delete that alias. Server admins can also delete any alias on their server. @@ -323,7 +323,7 @@ Inputs: - Room Alias Output: - Room ID - - List of home servers to join via. + - List of homeservers to join via. Mapping a room to an alias: @@ -334,7 +334,7 @@ Inputs: Output: - Room alias Notes: - - The home server may add restrictions e.g. the user must be in the room. + - The homeserver may add restrictions e.g. the user must be in the room. Deleting a mapping: @@ -347,11 +347,11 @@ Output: Published room list API ``[Draft]`` ----------------------------------- -This provides mechanisms for searching for published rooms on a home server. +This provides mechanisms for searching for published rooms on a homeserver. Inputs: - Search text (e.g. room alias/name/topic to search on) - - Home server to search on (this may just be the URL hit for HTTP) + - Homeserver to search on (this may just be the URL hit for HTTP) - Any existing pagination token, can be missing if this is the first hit. - Limit for pagination Output: @@ -378,7 +378,7 @@ Notes: User Profile API ``[Draft]`` ---------------------------- -Every user on a home server has a profile. This profile is effectively a +Every user on a homeserver has a profile. This profile is effectively a key-value store scoped to a user ID. It can include an ``avatar_url``, ``displayname`` and other metadata. Updates to a profile should propagate to other interested users. @@ -435,7 +435,7 @@ This had a number of problems associated with it: flicker. - Name/avatar changes created more ``m.room.member`` events which meant they needed to be included in the auth chains for federation. This - created long auth chains which is suboptimal since home servers need + created long auth chains which is suboptimal since homeservers need to store the auth chains forever. These problems can be resolved by creating an ``m.room.member.profile`` @@ -448,7 +448,7 @@ However, this introduces its own set of problems, namely flicker. The client would receive the ``m.room.member`` event first, followed by the ``m.room.member.profile`` event, which could cause a flicker. In addition, federation may not send both events in a single transaction, -resulting in missing information on the receiving home server. +resulting in missing information on the receiving homeserver. For federation, these problems can be resolved by sending the ``m.room.member`` event as they are in v1 (with ``displayname`` and @@ -457,7 +457,7 @@ they cannot be in the ``unsigned`` part of the event. The receiving home server will then extract these keys and create a server-generated ``m.room.member.profile`` event. To avoid confusion with duplicate information, the ``avatar_url`` and ``displayname`` keys should be -removed from the ``m.room.member`` event by the receiving home server. +removed from the ``m.room.member`` event by the receiving homeserver. When a client requests these events (either from the event stream or from an initial sync), the server will send the generated ``m.room.member.profile`` event under the ``unsigned.profile`` key of the @@ -840,17 +840,17 @@ information per device to all other users just redirects the union problem to the client, which will commonly be presenting this information as an icon alongside the user. -When a client hits the event stream, the home server can treat the user as +When a client hits the event stream, the homeserver can treat the user as "online". This behaviour should be able to be overridden to avoid flicker during connection losses when the client is appear offline (e.g. device is appear offline > goes into a tunnel > server times out > device regains connection and hits the event stream forcing the device online before the "appear offline" state can be set). When the client has not hit the event -stream for a certain period of time, the home server can treat the user as +stream for a certain period of time, the homeserver can treat the user as "offline". The user can also set a global *per-user* appear offline flag. The user should also be able to set their presence state via a direct API, -without having to hit the event stream. The home server will set a timer when +without having to hit the event stream. The homeserver will set a timer when the connection ends, after which it will set that device to offline. As the idle flag and online state is determined per device, there needs to be a @@ -970,12 +970,12 @@ the hashes the same is the best as that means clients do not need to request the capabilities for the given hash. On first signup, the client will attempt to send the hash and be most likely -refused by the home server as it does not know the full capability set for that +refused by the homeserver as it does not know the full capability set for that hash. The client will then have to upload the full capability set to the home server. The client will then be able to send the hash as normal. When a client receives a hash, the client will either recognise the hash or -will have to request the capability set from their home server: +will have to request the capability set from their homeserver: Inputs: - Hash @@ -1070,7 +1070,7 @@ Main use cases for ``updates``: - Call signalling (child events are ICE candidates, answer to the offer, and termination) - *Local* Delivery/Read receipts : "Local" means they are not shared with other - users on the same home server or via federation but *are* shared between + users on the same homeserver or via federation but *are* shared between clients for the same user; useful for push notifications, read count markers, etc. This is done to avoid the ``n^2`` problem for sending receipts, where the vast majority of traffic tends towards sending more receipts. @@ -1168,11 +1168,11 @@ Events (breaking changes; event version 2) ``[Draft]`` when dealing with custom event types. E.g. ``_custom.event`` would allow anything in the state key, ``_@custom.event`` would only allow user IDs in the state key, etc. -- s/user_id/sender/g given that home servers can send events, not just users. +- s/user_id/sender/g given that homeservers can send events, not just users. Server-generated events ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Home servers may want to send events to their local clients or to other home +Homeservers may want to send events to their local clients or to other home servers e.g. for server status notifications. These events look like regular events but have a server domain name as the diff --git a/drafts/human-id-rules.rst b/drafts/human-id-rules.rst index 914a9a42..d0c2ee2c 100644 --- a/drafts/human-id-rules.rst +++ b/drafts/human-id-rules.rst @@ -13,9 +13,9 @@ phishing/spoofing of IDs, commonly known as a homograph attack. Web browers encountered this problem when International Domain Names were introduced. A variety of checks were put in place in order to protect users. If an address failed the check, the raw punycode would be displayed to -disambiguate the address. Similar checks are performed by home servers in +disambiguate the address. Similar checks are performed by homeservers in Matrix. However, Matrix does not use punycode representations, and so does not -show raw punycode on a failed check. Instead, home servers must outright reject +show raw punycode on a failed check. Instead, homeservers must outright reject these misleading IDs. Types of human-readable IDs @@ -48,12 +48,12 @@ Checks Rejecting --------- -- Home servers MUST reject room aliases which do not pass the check, both on +- Homeservers MUST reject room aliases which do not pass the check, both on GETs and PUTs. -- Home servers MUST reject user ID localparts which do not pass the check, both +- Homeservers MUST reject user ID localparts which do not pass the check, both on creation and on events. -- Any home server whose domain does not pass this check, MUST use their punycode - domain name instead of the IDN, to prevent other home servers rejecting you. +- Any homeserver whose domain does not pass this check, MUST use their punycode + domain name instead of the IDN, to prevent other homeservers rejecting you. - Error code is ``M_FAILED_HUMAN_ID_CHECK``. (generic enough for both failing due to homograph attacks, and failing due to including ``:`` s, etc) - Error message MAY go into further information about which characters were @@ -74,7 +74,7 @@ Other considerations - High security: Outright rejection of the ID at the point of creation / receiving event. Point of creation rejection is preferable to avoid the ID entering the system in the first place. However, malicious HSes can just - allow the ID. Hence, other home servers must reject them if they see them in + allow the ID. Hence, other homeservers must reject them if they see them in events. Client never sees the problem ID, provided the HS is correctly implemented. - High security decided; client doesn't need to worry about it, no additional diff --git a/drafts/model/presence.rst b/drafts/model/presence.rst index bd60cba0..c2c88737 100644 --- a/drafts/model/presence.rst +++ b/drafts/model/presence.rst @@ -5,14 +5,14 @@ A simple implementation of presence messaging has the ability to cause a large amount of Internet traffic relating to presence updates. In order to minimise the impact of such a feature, the following observations can be made: - * There is no point in a Home Server polling status for peers in a user's + * There is no point in a homeserver polling status for peers in a user's presence list if the user has no clients connected that care about it. * It is highly likely that most presence subscriptions will be symmetric - a given user watching another is likely to in turn be watched by that user. * It is likely that most subscription pairings will be between users who share - at least one Room in common, and so their Home Servers are actively + at least one Room in common, and so their homeservers are actively exchanging message PDUs or transactions relating to that Room. * Presence update messages do not need realtime guarantees. It is acceptable to @@ -25,7 +25,7 @@ promise to send them when required. Rather than actively polling for the current state all the time, HSes can rely on their relative stability to only push updates when required. -A Home Server should not rely on the longterm validity of this presence +A homeserver should not rely on the longterm validity of this presence information, however, as this would not cover such cases as a user's server crashing and thus failing to inform their peers that users it used to host are no longer available online. Therefore, each promise of future updates should @@ -33,7 +33,7 @@ carry with a timeout value (whether explicit in the message, or implicit as some defined default in the protocol), after which the receiving HS should consider the information potentially stale and request it again. -However, because of the likelihood that two home servers are exchanging messages +However, because of the likelihood that two homeservers are exchanging messages relating to chat traffic in a room common to both of them, the ongoing receipt of these messages can be taken by each server as an implicit notification that the sending server is still up and running, and therefore that no status changes @@ -98,7 +98,7 @@ The data model presented here puts the following requirements on the APIs: Client-Server ------------- -Requests that a client can make to its Home Server +Requests that a client can make to its homeserver * get/set current presence state Basic enumeration + ability to set a custom piece of text @@ -128,7 +128,7 @@ Requests that a client can make to its Home Server Server-Server ------------- -Requests that Home Servers make to others +Requests that homeservers make to others * request permission to add a user to presence list diff --git a/drafts/model/profiles.rst b/drafts/model/profiles.rst index 15f65feb..f81e115e 100644 --- a/drafts/model/profiles.rst +++ b/drafts/model/profiles.rst @@ -9,7 +9,7 @@ Overview ======== Internally within Synapse users are referred to by an opaque ID, which consists -of some opaque localpart combined with the domain name of their home server. +of some opaque localpart combined with the domain name of their homeserver. Obviously this does not yield a very nice user experience; users would like to see readable names for other users that are in some way meaningful to them. Additionally, users like to be able to publish "profile" details to inform other @@ -59,7 +59,7 @@ servers should be accounted for here.]] Visibility Permissions ====================== -A home server implementation could offer the ability to set permissions on +A homeserver implementation could offer the ability to set permissions on limited visibility of those fields. When another user requests access to the target user's profile, their own identity should form part of that request. The HS implementation can then decide which fields to make available to the @@ -130,12 +130,12 @@ namespace to allocate names into. It would also be nice from a user experience perspective if the profile that a given name links to can also declare that name as part of its metadata. Furthermore as a security and consistency perspective it would be nice if each -end (the directory server and the user's home server) check the validity of the +end (the directory server and the user's homeserver) check the validity of the mapping in some way. This needs investigation from a security perspective to ensure against spoofing. One such model may be that the user starts by declaring their intent to use a -given user name link to their home server, which then contacts the directory +given user name link to their homeserver, which then contacts the directory service. At some point later (maybe immediately for "public open FCFS servers", maybe after some kind of human intervention for verification) the DS decides to honour this link, and includes it in its served output. It should also tell the @@ -170,7 +170,7 @@ balancing choice on behalf of the user who would choose, or not, to make use of such a feature to publish their information. Additionally, unless some form of strong end-to-end user-based encryption is -used, a user of ACLs for information privacy has to trust other home servers not +used, a user of ACLs for information privacy has to trust other homeservers not to lie about the identify of the user requesting access to the Profile. @@ -182,7 +182,7 @@ The data model presented here puts the following requirements on the APIs: Client-Server ------------- -Requests that a client can make to its Home Server +Requests that a client can make to its homeserver * get/set my Display Name This should return/take a simple "text/plain" field @@ -207,7 +207,7 @@ TODO(paul): At some later stage we should consider the API for: Server-Server ------------- -Requests that Home Servers make to others +Requests that homeservers make to others * get a user's Display Name / Avatar @@ -221,7 +221,7 @@ Requests that Home Servers make to others Room Event PDU Types -------------------- -Events that are pushed from Home Servers to other Home Servers or clients. +Events that are pushed from homeservers to other homeservers or clients. * user Display Name change diff --git a/drafts/model/room-join-workflow.rst b/drafts/model/room-join-workflow.rst index c321a64f..ff469740 100644 --- a/drafts/model/room-join-workflow.rst +++ b/drafts/model/room-join-workflow.rst @@ -8,7 +8,7 @@ Discovery ========= To join a room, a user has to discover the room by some mechanism in order to -obtain the (opaque) Room ID and a candidate list of likely home servers that +obtain the (opaque) Room ID and a candidate list of likely homeservers that contain it. Sending an Invitation @@ -21,7 +21,7 @@ The inviter's HS sets the membership status of the invitee to "invited" in the "m.members" state key by sending a state update PDU. The HS then broadcasts this PDU among the existing members in the usual way. An invitation message is also sent to the invited user, containing the Room ID and the PDU ID of this -invitation state change and potentially a list of some other home servers to use +invitation state change and potentially a list of some other homeservers to use to accept the invite. The user's client can then choose to display it in some way to alert the user. @@ -34,7 +34,7 @@ Directory Service Alternatively, the user may discover the channel via a directory service; either by performing a name lookup, or some kind of browse or search acitivty. However -this is performed, the end result is that the user's home server requests the +this is performed, the end result is that the user's homeserver requests the Room ID and candidate list from the directory service. [[TODO(paul): At present, no API has been designed or described for this @@ -44,14 +44,14 @@ directory service]] Joining ======= -Once the ID and home servers are obtained, the user can then actually join the +Once the ID and homeservers are obtained, the user can then actually join the room. Accepting an Invite ------------------- If a user has received and accepted an invitation to join a room, the invitee's -home server can now send an invite acceptance message to a chosen candidate +homeserver can now send an invite acceptance message to a chosen candidate server from the list given in the invitation, citing also the PDU ID of the invitation as "proof" of their invite. (This is required as due to late message propagation it could be the case that the acceptance is received before the @@ -85,7 +85,7 @@ can instead post a "knock" message, which informs other members of the room that the would-be joiner wishes to become a member and sets their membership value to "knocked". If any of them wish to accept this, they can then send an invitation in the usual way described above. Knowing that the user has already knocked and -expressed an interest in joining, the invited user's home server should +expressed an interest in joining, the invited user's homeserver should immediately accept that invitation on the user's behalf, and go on to join the room in the usual way. diff --git a/drafts/model/rooms.rst b/drafts/model/rooms.rst index 07c5e71f..2f57f2bc 100644 --- a/drafts/model/rooms.rst +++ b/drafts/model/rooms.rst @@ -18,19 +18,19 @@ users, and other management and miscellaneous metadata), and a message history. Room Identity and Naming ======================== -Rooms can be arbitrarily created by any user on any home server; at which point -the home server will sign the message that creates the channel, and the +Rooms can be arbitrarily created by any user on any homeserver; at which point +the homeserver will sign the message that creates the channel, and the fingerprint of this signature becomes the strong persistent identify of the -room. This now identifies the room to any home server in the network regardless +room. This now identifies the room to any homeserver in the network regardless of its original origin. This allows the identify of the room to outlive any particular server. Subject to appropriate permissions [to be discussed later], any current member of a room can invite others to join it, can post messages that become part of its history, and can change the persistent state of the room (including its current set of permissions). -Home servers can provide a directory service, allowing a lookup from a +Homeservers can provide a directory service, allowing a lookup from a convenient human-readable form of room label to a room ID. This mapping is -scoped to the particular home server domain and so simply represents that server +scoped to the particular homeserver domain and so simply represents that server administrator's opinion of what room should take that label; it does not have to be globally replicated and does not form part of the stored state of that room. @@ -156,7 +156,7 @@ m.public_history to be a member of the room. m.archive_servers - For "public" rooms with public history, gives a list of home servers that + For "public" rooms with public history, gives a list of homeservers that should be included in message distribution to the room, even if no users on that server are present. These ensure that a public room can still persist even if no users are currently members of it. This list should be consulted by @@ -179,7 +179,7 @@ m.topic Room Creation Templates ======================= -A client (or maybe home server?) could offer a few templates for the creation of +A client (or maybe homeserver?) could offer a few templates for the creation of new rooms. For example, for a simple private one-to-one chat the channel could assign the creator a power-level of 1, requiring a level of 1 to invite, and needing an invite before members can join. An invite is then sent to the other @@ -215,7 +215,7 @@ permissions to allow this direct messaging. Between any given pair of user IDs that wish to exchange private messages, there will exist a single shared Room, created lazily by either side. These rooms will -need a certain amount of special handling in both home servers and display on +need a certain amount of special handling in both homeservers and display on clients, but as much as possible should be treated by the lower layers of code the same as other rooms. @@ -226,7 +226,7 @@ clients should display these in a special way too as the room name is not important; instead it should distinguish them on the Display Name of the other party. -Home Servers will need a client-API option to request setting up a new user-user +Homeservers will need a client-API option to request setting up a new user-user chat room, which will then need special handling within the server. It will create a new room with the following @@ -260,7 +260,7 @@ history with each other simultaneously create a room and invite the other to it. This is called a "glare" situation. There are two possible ideas for how to resolve this: - * Each Home Server should persist the mapping of (user ID pair) to room ID, so + * Each homeserver should persist the mapping of (user ID pair) to room ID, so that duplicate requests can be suppressed. On receipt of a room creation request that the HS thinks there already exists a room for, the invitation to join can be rejected if: diff --git a/drafts/model/third-party-id.rst b/drafts/model/third-party-id.rst index 1f8138dd..838a6799 100644 --- a/drafts/model/third-party-id.rst +++ b/drafts/model/third-party-id.rst @@ -66,7 +66,7 @@ Privacy A User may publish the association between their phone number and Matrix User ID on the Identity Server without publishing the number in their Profile hosted on -their Home Server. +their homeserver. Identity Servers should refrain from publishing reverse mappings and should take steps, such as rate limiting, to prevent attackers enumerating the space of diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index 48c01bdb..41a65296 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -24,7 +24,7 @@ cat >tmp/http_apis < Application Service API +Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pushing events @@ -101,13 +101,13 @@ events. Each list of events includes a transaction ID, which works as follows: :: Typical - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <--- : AS sends back 200 OK. AS ACK Lost - HS ---> AS : Home server sends events with transaction ID T. + HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. - HS ---> AS : Home server retries with the same transaction ID of T. + HS ---> AS : Homeserver retries with the same transaction ID of T. <--- : AS sends back 200 OK. If the AS had processed these events already, it can NO-OP this request (and it knows if it is the same events based on the transaction ID). @@ -145,7 +145,7 @@ this request (e.g. to join a room alias). HTTP APIs +++++++++ -This contains application service APIs which are used by the home server. All +This contains application service APIs which are used by the homeserver. All application services MUST implement these APIs. These APIs are defined below. {{application_service_http_api}} @@ -158,7 +158,7 @@ Client-Server v2 API Extensions Application services can utilise a more powerful version of the client-server API by identifying itself as an application service to the -home server. +homeserver. Identity assertion ++++++++++++++++++ @@ -212,7 +212,7 @@ Server admin style permissions .. _sect:asapi-permissions: -The home server needs to give the application service *full control* over its +The homeserver needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as create/delete any user in its namespace. No additional API changes need to be diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 67973c6b..9edf7e55 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -147,12 +147,12 @@ User-Interactive Authentication API This section refers to API Version 2. Some API endpoints such as ``login`` or ``register`` require authentication that -interacts with the user. The home server may provide many different ways of +interacts with the user. The homeserver may provide many different ways of authenticating, such as user/password auth, login via a social network (OAuth2), login by confirming a token sent to their email address, etc. This specification -does not define how home servers should authorise their users but instead +does not define how homeservers should authorise their users but instead defines the standard interface which implementations should follow so that ANY -client can login to ANY home server. +client can login to ANY homeserver. The process takes the form of one or more stages, where at each stage the client submits a set of data for a given stage type and awaits a response from the @@ -168,9 +168,9 @@ more than one stage to implement n-factor auth. When all stages are complete, authentication is complete and the API call succeeds. To establish what flows a server supports for an endpoint, a client sends the request with no authentication. A request to an endpoint that uses User-Interactive -Authentication never succeeds without auth. Home Servers may allow requests that +Authentication never succeeds without auth. Homeservers may allow requests that don't require auth by offering a stage with only the ``m.login.dummy`` auth -type. The home server returns a response with HTTP status 401 and a JSON object +type. The homeserver returns a response with HTTP status 401 and a JSON object as follows:: { @@ -208,7 +208,7 @@ does this by resubmitting the same request with the the addition of an 'auth' key in the object that it submits. This dictionary contains a ``type`` key whose value is the name of the stage type that the client is attempting to complete. It must also contains a ``session`` key with the value of the session key given -by the home server, if one was given. It also contains other keys dependent on +by the homeserver, if one was given. It also contains other keys dependent on the stage type being attempted. For example, if the client is attempting to complete login type ``example.type.foo``, it might submit something like this:: @@ -222,7 +222,7 @@ complete login type ``example.type.foo``, it might submit something like this:: } } -If the home server deems the authentication attempt to be successful but still +If the homeserver deems the authentication attempt to be successful but still requires more stages to be completed, it returns HTTP status 401 along with the same object as when no authentication was attempted, with the addition of the ``completed`` key which is an array of stage type the client has completed @@ -246,7 +246,7 @@ successfully:: "session": "xxxxxx" } -If the home server decides the attempt was unsuccessful, it returns an error +If the homeserver decides the attempt was unsuccessful, it returns an error message in the standard format:: { @@ -258,7 +258,7 @@ Individual stages may require more than one request to complete, in which case the response will be as if the request was unauthenticated with the addition of any other keys as defined by the login type. -If the client has completed all stages of a flow, the home server performs the +If the client has completed all stages of a flow, the homeserver performs the API call and returns the result as normal. Some authentication types may be completed by means other than through the @@ -389,16 +389,16 @@ OAuth2-based ``uri``: Authorization Request URI OR service selection URI. Both contain an encoded ``redirect URI``. -The home server acts as a 'confidential' client for the purposes of OAuth2. If +The homeserver acts as a 'confidential' client for the purposes of OAuth2. If the uri is a ``service selection URI``, it MUST point to a webpage which prompts the user to choose which service to authorize with. On selection of a service, this MUST link through to an ``Authorization Request URI``. If there is only one -service which the home server accepts when logging in, this indirection can be +service which the homeserver accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the OAuth2 Allow/Deny prompt. Hitting 'Allow' redirects to the ``redirect URI`` with -the auth code. Home servers can choose any path for the ``redirect URI``. Once +the auth code. Homeservers can choose any path for the ``redirect URI``. Once the OAuth flow has completed, the client retries the request with the session only, as above. @@ -412,7 +412,7 @@ Email-based (identity server) Prior to submitting this, the client should authenticate with an identity server. After authenticating, the session information should be submitted to -the home server. +the homeserver. To respond to this type, reply with an auth dict as follows:: @@ -450,12 +450,12 @@ Clients cannot be expected to be able to know how to process every single login type. If a client does not know how to handle a given login type, it can direct the user to a web browser with the URL of a fallback page which will allow the user to complete that login step out-of-band in their web browser. The URL it -should open is the Home Server base URL plus prefix, plus:: +should open is the homeserver base URL plus prefix, plus:: /auth//fallback/web?session= Where ``stage type`` is the type name of the stage it is attempting and -``session id`` is the ID of the session given by the home server. +``session id`` is the ID of the session given by the homeserver. This MUST return an HTML page which can perform this authentication stage. This page must attempt to call the JavaScript function ``window.onAuthDone`` when @@ -494,7 +494,7 @@ Request:: This API endpoint uses the User-Interactive Authentication API. An access token should be submitted to this endpoint if the client has an active session. The -Home Server may change the flows available depending on whether a valid access +homeserver may change the flows available depending on whether a valid access token is provided. The body of the POST request is a JSON object containing: @@ -505,7 +505,7 @@ new_password On success, an empty JSON object is returned. The error code M_NOT_FOUND is returned if the user authenticated with a third -party identifier but the Home Server could not find a matching account in its +party identifier but the homeserver could not find a matching account in its database. Adding Account Administrative Contact Information @@ -699,7 +699,7 @@ namespaced for each application and reduces the risk of clashes. Syncing ~~~~~~~ -Clients receive new events by "long-polling" the home server via the events API. +Clients receive new events by "long-polling" the homeserver via the events API. This involves specifying a timeout in the request which will hold open the HTTP connection for a short period of time waiting for new events, returning early if an event occurs. Only the events API supports long-polling. @@ -719,7 +719,7 @@ last request left off. Multiple events can be returned per long-poll. Do we ever support streaming requests? Why not websockets? When the client first logs in, they will need to initially synchronise with -their home server. This is achieved via the initial sync API described below. +their homeserver. This is achieved via the initial sync API described below. This API also returns an ``end`` token which can be used with the event stream. {{old_sync_http_api}} @@ -831,7 +831,7 @@ Rooms Creation ~~~~~~~~ -The home server will create an ``m.room.create`` event when a room is created, +The homeserver will create an ``m.room.create`` event when a room is created, which serves as the root of the event graph for this room. This event also has a ``creator`` key which contains the user ID of the room creator. It will also generate several other events in order to manage permissions in this room. This @@ -852,9 +852,9 @@ Room aliases Servers may host aliases for rooms with human-friendly names. Aliases take the form ``#friendlyname:server.name``. -As room aliases are scoped to a particular home server domain name, it is -likely that a home server will reject attempts to maintain aliases on other -domain names. This specification does not provide a way for home servers to +As room aliases are scoped to a particular homeserver domain name, it is +likely that a homeserver will reject attempts to maintain aliases on other +domain names. This specification does not provide a way for homeservers to send update requests to other servers. Rooms store a *partial* list of room aliases via the ``m.room.aliases`` state @@ -867,8 +867,8 @@ appears to have a room alias of ``#alias:example.com``, this SHOULD be checked to make sure that the room's ID matches the ``room_id`` returned from the request. -Home servers can respond to resolve requests for aliases on other domains than -their own by using the federation API to ask other domain name home servers. +Homeservers can respond to resolve requests for aliases on other domains than +their own by using the federation API to ask other domain name homeservers. {{directory_http_api}} @@ -985,10 +985,10 @@ values. This change is conveyed using two separate mechanisms: values of the ``displayname`` and ``avatar_url`` keys, in addition to the required ``presence`` key containing the current presence state of the user. -Both of these should be done automatically by the home server when a user +Both of these should be done automatically by the homeserver when a user successfully changes their display name or avatar URL fields. -Additionally, when home servers emit room membership events for their own +Additionally, when homeservers emit room membership events for their own users, they should include the display name and avatar URL fields in these events so that clients already have these details to hand, and do not have to perform extra round trips to query it. @@ -998,7 +998,7 @@ Security Rate limiting ~~~~~~~~~~~~~ -Home servers SHOULD implement rate limiting to reduce the risk of being +Homeservers SHOULD implement rate limiting to reduce the risk of being overloaded. If a request is refused due to rate limiting, it should return a standard error response of the form:: diff --git a/specification/intro.rst b/specification/intro.rst index eb6f98fc..ced721fa 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -140,7 +140,7 @@ a long-lived GET request. | V | V +------------------+ +------------------+ | |---------( HTTPS )--------->| | - | Home Server | | Home Server | + | homeserver | | homeserver | | |<--------( HTTPS )----------| | +------------------+ Server-Server API +------------------+ History Synchronisation @@ -227,7 +227,7 @@ They are case-sensitive. The following conceptual diagram shows an | | V | +------------------+ +------------------+ - | Home Server | | Home Server | + | homeserver | | homeserver | | matrix.org | | domain.com | +------------------+ +------------------+ | ^ diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 3602d105..4eddaeee 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -61,7 +61,7 @@ recommended. Server behaviour ---------------- -Each user's home server stores a "presence list" per user. Once a user accepts +Each user's homeserver stores a "presence list" per user. Once a user accepts a presence list, both user's HSes must track the subscription. Propagating profile information @@ -73,7 +73,7 @@ automatic propagation event to occur, informing likely-interested parties of the new values. One of these change mechanisms SHOULD be via ``m.presence`` events. These events should set ``displayname`` and ``avatar_url`` to the new values along with the presence-specific keys. This SHOULD be done automatically by the -home server when a user successfully changes their display name or avatar URL. +homeserver when a user successfully changes their display name or avatar URL. .. admonition:: Rationale diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 893e9481..de45bfe6 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -11,7 +11,7 @@ Push Notifications | | | | +-------------------+ | +----------------+ | | +---------------+ | | | | | | | | | | | - | Matrix Home Server+-----> Push Gateway +------> Push Provider | | + | Matrix homeserver+-----> Push Gateway +------> Push Provider | | | | | | | | | | | | +-^-----------------+ | +----------------+ | | +----+----------+ | | | | | | | diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 012c2ac9..ba6b7d35 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1,8 +1,8 @@ Federation API ============== -Matrix home servers use the Federation APIs (also known as server-server APIs) -to communicate with each other. Home servers use these APIs to push messages to +Matrix homeservers use the Federation APIs (also known as server-server APIs) +to communicate with each other. Homeservers use these APIs to push messages to each other in real-time, to request historic messages from each other, and to query profile and presence information about users on each other's servers. @@ -11,10 +11,10 @@ servers. These HTTPS requests are strongly authenticated using public key signatures at the TLS transport layer and using public key signatures in HTTP Authorization headers at the HTTP layer. -There are three main kinds of communication that occur between home servers: +There are three main kinds of communication that occur between homeservers: Persisted Data Units (PDUs): - These events are broadcast from one home server to any others that have + These events are broadcast from one homeserver to any others that have joined the same room (identified by Room ID). They are persisted in long-term storage and record the history of messages and state for a room. @@ -25,9 +25,9 @@ Persisted Data Units (PDUs): deliver them through third-party servers. Ephemeral Data Units (EDUs): - These events are pushed between pairs of home servers. They are not + These events are pushed between pairs of homeservers. They are not persisted and are not part of the history of a room, nor does the - receiving home server have to reply to them. + receiving homeserver have to reply to them. Queries: These are single request/response interactions between a given pair of @@ -38,7 +38,7 @@ Queries: EDUs and PDUs are further wrapped in an envelope called a Transaction, which is -transferred from the origin to the destination home server using an HTTPS PUT +transferred from the origin to the destination homeserver using an HTTPS PUT request. .. contents:: Table of Contents @@ -50,7 +50,7 @@ Server Discovery Resolving Server Names ~~~~~~~~~~~~~~~~~~~~~~ -Each matrix home server is identified by a server name consisting of a DNS name +Each matrix homeserver is identified by a server name consisting of a DNS name and an optional TLS port. .. code:: @@ -67,7 +67,7 @@ is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV record for the DNS name. If this record does not exist then the server is discovered by looking up an AAAA or A record on the DNS name and taking the default fallback port number of 8448. -Home servers may use SRV records to load balance requests between multiple TLS +Homeservers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. Retrieving Server Keys @@ -76,8 +76,8 @@ Retrieving Server Keys Version 2 +++++++++ -Each home server publishes its public keys under ``/_matrix/key/v2/server/``. -Home servers query for keys by either getting ``/_matrix/key/v2/server/`` +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/``. +Homeservers query for keys by either getting ``/_matrix/key/v2/server/`` directly or by querying an intermediate notary server using a ``/_matrix/key/v2/query`` API. Intermediate notary servers query the ``/_matrix/key/v2/server/`` API on behalf of another server and sign the @@ -95,7 +95,7 @@ server by querying other servers. Publishing Keys ^^^^^^^^^^^^^^^ -Home servers publish the allowed TLS fingerprints and signing keys in a JSON +Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the server and for signing events. It contains a list of ``old_verify_keys`` @@ -114,7 +114,7 @@ certificate currently in use by the server. These fingerprints are valid until the millisecond POSIX timestamp in ``valid_until_ts``. The ``verify_keys`` can be used to sign requests and events made by the server -until the millisecond POSIX timestamp in ``valid_until_ts``. If a Home Server +until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver receives an event with a ``origin_server_ts`` after the ``valid_until_ts`` then it should request that ``key_id`` for the originating server to check whether the key has expired. @@ -136,8 +136,8 @@ events sent by that server can still be checked. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the home server. -``verify_keys`` Object Public keys of the home server for +``server_name`` String DNS name of the homeserver. +``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``old_verify_keys`` Object The public keys that the server used to use and when it stopped using them. @@ -238,14 +238,14 @@ Version 1 Version 1 of key distribution is obsolete -Home servers publish their TLS certificates and signing keys in a JSON object +Homeservers publish their TLS certificates and signing keys in a JSON object at ``/_matrix/key/v1``. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the home server. -``verify_keys`` Object Public keys of the home server for +``server_name`` String DNS name of the homeserver. +``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``signatures`` Object Digital signatures for this object signed using the ``verify_keys``. @@ -278,9 +278,9 @@ Transactions .. WARNING:: This section may be misleading or inaccurate. -The transfer of EDUs and PDUs between home servers is performed by an exchange +The transfer of EDUs and PDUs between homeservers is performed by an exchange of Transaction messages, which are encoded as JSON objects, passed over an HTTP -PUT request. A Transaction is meaningful only to the pair of home servers that +PUT request. A Transaction is meaningful only to the pair of homeservers that exchanged it; they are not globally-meaningful. Each transaction has: @@ -445,7 +445,7 @@ EDUs EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and -destination home server names, and the actual nested content. +destination homeserver names, and the actual nested content. ======================== ============ ========================================= Key Type Description @@ -531,7 +531,7 @@ To make a query:: Query args: as specified by the individual query types Response: JSON encoding of a response object -Performs a single query request on the receiving home server. The Query Type +Performs a single query request on the receiving homeserver. The Query Type part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. @@ -791,9 +791,9 @@ because HTTP services like Matrix are often deployed behind load balancers that handle the TLS and these load balancers make it difficult to check TLS client certificates. -A home server may provide a TLS client certificate and the receiving home server +A homeserver may provide a TLS client certificate and the receiving homeserver may check that the client certificate matches the certificate of the origin -home server. +homeserver. Server-Server Authorization --------------------------- @@ -905,7 +905,7 @@ Querying profile information:: If the query contains the optional ``field`` key, it should give the name of a result field. If such is present, then the result should contain only a field of that name, with no others present. If not, the result should contain as much -of the user's profile as the home server has available and can make public. +of the user's profile as the homeserver has available and can make public. Directory --------- diff --git a/supporting-docs/examples/application-services.rst b/supporting-docs/examples/application-services.rst index bde4531b..fae3613d 100644 --- a/supporting-docs/examples/application-services.rst +++ b/supporting-docs/examples/application-services.rst @@ -6,15 +6,15 @@ This file contains examples of some application service IRC Bridge ---------- Pre-conditions: - - Server admin stores the AS token "T_a" on the home server. - - Home server has a token "T_h". - - Home server has the domain "hsdomain.com" + - Server admin stores the AS token "T_a" on the homeserver. + - Homeserver has a token "T_h". + - Homeserver has the domain "hsdomain.com" 1. Application service registration :: - AS -> HS: Registers itself with the home server + AS -> HS: Registers itself with the homeserver POST /register { url: "https://someapp.com/matrix", diff --git a/supporting-docs/guides/2015-08-10-client-server.rst b/supporting-docs/guides/2015-08-10-client-server.rst index 2800584c..fe6b464a 100644 --- a/supporting-docs/guides/2015-08-10-client-server.rst +++ b/supporting-docs/guides/2015-08-10-client-server.rst @@ -19,11 +19,11 @@ How to use the client-server API The git version of this document is {% project_version %} This guide focuses on how the client-server APIs *provided by the reference -home server* can be used. Since this is specific to a home server +homeserver* can be used. Since this is specific to a homeserver implementation, there may be variations in relation to registering/logging in which are not covered in extensive detail in this guide. -If you haven't already, get a home server up and running on +If you haven't already, get a homeserver up and running on ``http://localhost:8008``. @@ -56,8 +56,8 @@ if you forget the ``access_token``. Implementation note: The matrix specification does not enforce how users register with a server. It just specifies the URL path and absolute minimum -keys. The reference home server uses a username/password to authenticate user, -but other home servers may use different methods. This is why you need to +keys. The reference homeserver uses a username/password to authenticate user, +but other homeservers may use different methods. This is why you need to specify the ``type`` of method. Login @@ -82,13 +82,13 @@ The aim when logging in is to get an access token for your existing user ID:: "user_id": "@example:localhost" } -Implementation note: Different home servers may implement different methods for +Implementation note: Different homeservers may implement different methods for logging in to an existing account. In order to check that you know how to login -to this home server, you must perform a ``GET`` first and make sure you +to this homeserver, you must perform a ``GET`` first and make sure you recognise the login type. If you do not know how to login, you can ``GET /login/fallback`` which will return a basic webpage which you can use to -login. The reference home server implementation support username/password login, -but other home servers may support different login methods (e.g. OAuth2). +login. The reference homeserver implementation support username/password login, +but other homeservers may support different login methods (e.g. OAuth2). Communicating diff --git a/supporting-docs/guides/2015-08-21-application_services.md b/supporting-docs/guides/2015-08-21-application_services.md index dd532d42..e034e154 100644 --- a/supporting-docs/guides/2015-08-21-application_services.md +++ b/supporting-docs/guides/2015-08-21-application_services.md @@ -6,7 +6,7 @@ categories: guides # Application services -Application services are distinct modules which which sit alongside a home server providing arbitrary extensible functionality decoupled from the home server implementation. Just like the rest of Matrix, they communicate via HTTP using JSON. Application services function in a very similar way to traditional clients, but they are given much more power than a normal client. They can reserve entire namespaces of room aliases and user IDs for their own purposes. They can silently monitor events in rooms, or any events directed at any user ID. This power allows application services to have extremely useful abilities which overall enhance the end user experience. +Application services are distinct modules which which sit alongside a homeserver providing arbitrary extensible functionality decoupled from the homeserver implementation. Just like the rest of Matrix, they communicate via HTTP using JSON. Application services function in a very similar way to traditional clients, but they are given much more power than a normal client. They can reserve entire namespaces of room aliases and user IDs for their own purposes. They can silently monitor events in rooms, or any events directed at any user ID. This power allows application services to have extremely useful abilities which overall enhance the end user experience. | @@ -47,17 +47,17 @@ At present, the IRC application service is in beta, and is being run on #matrix | # What Application services can do for you -Application services have enormous potential for creating new and exciting ways to transform and enhance the core Matrix protocol. For example, you could aggregate information from multiple rooms into a summary room, or create throwaway virtual user accounts to proxy messages for a fixed user ID on-the-fly. As you may expect, all of this power assumes a high degree of trust between application services and home servers. Only home server admins can allow an application service to link up with their home server, and the application service is in no way federated to other home servers. You can think of application services as additional logic on the home server itself, without messing around with the book-keeping that home servers have to do. This makes adding useful functionality very easy. +Application services have enormous potential for creating new and exciting ways to transform and enhance the core Matrix protocol. For example, you could aggregate information from multiple rooms into a summary room, or create throwaway virtual user accounts to proxy messages for a fixed user ID on-the-fly. As you may expect, all of this power assumes a high degree of trust between application services and homeservers. Only homeserver admins can allow an application service to link up with their homeserver, and the application service is in no way federated to other homeservers. You can think of application services as additional logic on the homeserver itself, without messing around with the book-keeping that homeservers have to do. This makes adding useful functionality very easy. | ### Example -The application service (AS) API itself uses webhooks to communicate from the home server to the AS: +The application service (AS) API itself uses webhooks to communicate from the homeserver to the AS: -- Room Alias Query API : The home server hits a URL on your application server to see if a room alias exists. -- User Query API : The home server hits a URL on your application server to see if a user ID exists. -- Push API : The home server hits a URL on your application server to notify you of new events for your users and rooms. +- Room Alias Query API : The homeserver hits a URL on your application server to see if a room alias exists. +- User Query API : The homeserver hits a URL on your application server to see if a user ID exists. +- Push API : The homeserver hits a URL on your application server to notify you of new events for your users and rooms. A very basic application service may want to log all messages in rooms which have an alias starting with "#logged_" (side note: logging won't work if these rooms are using end-to-end encryption). @@ -85,7 +85,7 @@ Set your new application service running on port 5000 with: python app_service.py -The home server needs to know that the application service exists before it will send requests to it. This is done via a registration YAML file which is specified in Synapse's main config file e.g. homeserver.yaml. The server admin needs to add the application service registration configuration file as an entry to this file. +The homeserver needs to know that the application service exists before it will send requests to it. This is done via a registration YAML file which is specified in Synapse's main config file e.g. homeserver.yaml. The server admin needs to add the application service registration configuration file as an entry to this file. # homeserver.yaml app_service_config_files: @@ -115,7 +115,7 @@ NB: Note the "-" at the start; this indicates a list element. The registration f - exclusive: false regex: "#logged_.*" -**You will need to restart the home server after editing the config file before it will take effect.** +**You will need to restart the homeserver after editing the config file before it will take effect.** | @@ -138,6 +138,6 @@ This makes the application service lazily create a room with the requested alias | -Application services are powerful components which extend the functionality of home servers, but they are limited. They can only ever function in a "passive" way. For example, you cannot implement an application service which censors swear words in rooms, because there is no way to prevent the event from being sent. Aside from the fact that censoring will not work when using end-to-end encryption, all federated home servers would also need to reject the event in order to stop developing an inconsistent event graph. To "actively" monitor events, another component called a "Policy Server" is required, which is beyond the scope of this post.  Also, Application Services can result in a performance bottleneck, as all events on the homeserver must be ordered and sent to the registered application services.  If you are bridging huge amounts of traffic, you may be better off having your bridge directly talk the Server-Server federation API rather than the simpler Application Service API. +Application services are powerful components which extend the functionality of homeservers, but they are limited. They can only ever function in a "passive" way. For example, you cannot implement an application service which censors swear words in rooms, because there is no way to prevent the event from being sent. Aside from the fact that censoring will not work when using end-to-end encryption, all federated homeservers would also need to reject the event in order to stop developing an inconsistent event graph. To "actively" monitor events, another component called a "Policy Server" is required, which is beyond the scope of this post.  Also, Application Services can result in a performance bottleneck, as all events on the homeserver must be ordered and sent to the registered application services.  If you are bridging huge amounts of traffic, you may be better off having your bridge directly talk the Server-Server federation API rather than the simpler Application Service API. I hope this demonstrates how easy it is to create an application service, along with a few ideas of the kinds of things you can do with them. Obvious uses include build protocol bridges, search engines, invisible bots, etc. For more information on the AS HTTP API, check out the new Application Service API section in the spec, or the raw drafts and spec in https://github.com/matrix-org/matrix-doc/. diff --git a/supporting-docs/howtos/client-server.rst b/supporting-docs/howtos/client-server.rst index 3bed5a9f..e5a85d41 100644 --- a/supporting-docs/howtos/client-server.rst +++ b/supporting-docs/howtos/client-server.rst @@ -11,11 +11,11 @@ How to use the client-server API The git version of this document is ``{{git_version}}`` This guide focuses on how the client-server APIs *provided by the reference -home server* can be used. Since this is specific to a home server +homeserver* can be used. Since this is specific to a homeserver implementation, there may be variations in relation to registering/logging in which are not covered in extensive detail in this guide. -If you haven't already, get a home server up and running on +If you haven't already, get a homeserver up and running on ``http://localhost:8008``. @@ -48,8 +48,8 @@ if you forget the ``access_token``. Implementation note: The matrix specification does not enforce how users register with a server. It just specifies the URL path and absolute minimum -keys. The reference home server uses a username/password to authenticate user, -but other home servers may use different methods. This is why you need to +keys. The reference homeserver uses a username/password to authenticate user, +but other homeservers may use different methods. This is why you need to specify the ``type`` of method. Login @@ -74,13 +74,13 @@ The aim when logging in is to get an access token for your existing user ID:: "user_id": "@example:localhost" } -Implementation note: Different home servers may implement different methods for +Implementation note: Different homeservers may implement different methods for logging in to an existing account. In order to check that you know how to login -to this home server, you must perform a ``GET`` first and make sure you +to this homeserver, you must perform a ``GET`` first and make sure you recognise the login type. If you do not know how to login, you can ``GET /login/fallback`` which will return a basic webpage which you can use to -login. The reference home server implementation support username/password login, -but other home servers may support different login methods (e.g. OAuth2). +login. The reference homeserver implementation support username/password login, +but other homeservers may support different login methods (e.g. OAuth2). Communicating diff --git a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html index 088ff7ac..b7e874c2 100644 --- a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html +++ b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.html @@ -1,5 +1,5 @@
-

This room creation / message sending demo requires a home server to be running on http://localhost:8008

+

This room creation / message sending demo requires a homeserver to be running on http://localhost:8008

diff --git a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js index 9c346e2f..c16395ac 100644 --- a/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js +++ b/supporting-docs/howtos/jsfiddles/create_room_send_msg/demo.js @@ -19,7 +19,7 @@ $('.login').live('click', function() { showLoggedIn(data); }, error: function(err) { - var errMsg = "To try this, you need a home server running!"; + var errMsg = "To try this, you need a homeserver running!"; var errJson = $.parseJSON(err.responseText); if (errJson) { errMsg = JSON.stringify(errJson); diff --git a/supporting-docs/howtos/jsfiddles/event_stream/demo.html b/supporting-docs/howtos/jsfiddles/event_stream/demo.html index 7657780d..3de7b081 100644 --- a/supporting-docs/howtos/jsfiddles/event_stream/demo.html +++ b/supporting-docs/howtos/jsfiddles/event_stream/demo.html @@ -1,5 +1,5 @@
-

This event stream demo requires a home server to be running on http://localhost:8008

+

This event stream demo requires a homeserver to be running on http://localhost:8008

diff --git a/supporting-docs/howtos/jsfiddles/event_stream/demo.js b/supporting-docs/howtos/jsfiddles/event_stream/demo.js index acba8391..65b118d2 100644 --- a/supporting-docs/howtos/jsfiddles/event_stream/demo.js +++ b/supporting-docs/howtos/jsfiddles/event_stream/demo.js @@ -58,7 +58,7 @@ $('.login').live('click', function() { showLoggedIn(data); }, error: function(err) { - var errMsg = "To try this, you need a home server running!"; + var errMsg = "To try this, you need a homeserver running!"; var errJson = $.parseJSON(err.responseText); if (errJson) { errMsg = JSON.stringify(errJson); diff --git a/supporting-docs/howtos/jsfiddles/example_app/demo.html b/supporting-docs/howtos/jsfiddles/example_app/demo.html index 7a9dffdd..a183f61d 100644 --- a/supporting-docs/howtos/jsfiddles/example_app/demo.html +++ b/supporting-docs/howtos/jsfiddles/example_app/demo.html @@ -1,5 +1,5 @@