From 36e035c79e5efccb76a429194efdc19fcc450d0b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Jul 2015 19:31:11 +0100 Subject: [PATCH 001/111] Add some specification for end-to-end --- specification/41_end_to_end_encryption.rst | 215 +++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 023881527..a09c8fc60 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -13,3 +13,218 @@ participating homeservers. End-to-end crypto is still being designed and prototyped - notes on the design may be found at https://lwn.net/Articles/634144/ + +Overview +======== + +.. code:: + + 1) Bob publishes the public keys and supported algorithms for his device. + + +----------+ +--------------+ + | Bob's HS | | Bob's Device | + +----------+ +--------------+ + | | + |<=============| + /keys/upload + + 2) Alice requests Bob's public key and supported algorithms. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/query + + 3) Alice selects an algorithm takes any one time keys needed. + + +----------------+ +------------+ +----------+ + | Alice's Device | | Alice's HS | | Bob's HS | + +----------------+ +------------+ +----------+ + | | | + |=================>|==============>| + /keys/take + + 4) Alice sends an encrypted message to Bob. + + +----------------+ +------------+ +----------+ +--------------+ + | Alice's Device | | Alice's HS | | Bob's HS | | Bob's Device | + +----------------+ +------------+ +----------+ +--------------+ + | | | | + |----------------->|-------------->|------------->| + /send/ + + +Client Behaviour +---------------- + +Uploading Keys +~~~~~~~~~~~~~~ + +Keys are uploaded as a signed JSON object. The JSON object must include an +ed25519 key and must be signed by that key. A device may only have one ed25519 +signing key. This key is used as the fingerprint for a device by other clients. + + +.. code:: http + + POST /_matrix/client/v2_alpha/keys/upload/ HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "" { + ":": "" + } } }, + "one_time_keys": { + ":": "" + }, + } + + +Downloading Keys +~~~~~~~~~~~~~~~~ + +Keys are downloaded a collection of signed JSON objects. There +will be JSON object per device per user. If one of the user's +devices doesn't support end-to-end encryption then their +homeserver will synthesise a JSON object without any device keys +for that device. + +The JSON must be signed by both the homeserver of +the user querying the keys and by the homeserver of the device +being queried. This provides an audit trail if either homeserver +lies about the keys a user owns. + +.. code:: http + + POST /keys/query HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "": [""] + } } + + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_keys": { + "": { + "": { + "user_id": "", + "device_id": "", + "valid_after_ts": 1234567890123, + "valid_until_ts": 2345678901234, + "algorithms": [ + "", + ], + "keys": { + ":": "", + }, + "signatures:" { + "": { + ":": "" + }, + "": { + ":": "" + }, + "": { + ":": "" + } } } } } } + + +Taking One Time Keys +~~~~~~~~~~~~~~~~~~~~ + +Some algorithms require one time keys to improve their secrecy and deniability. +Theses keys are used once during session establishment, and are then thrown +away. In order for these keys to be useful for improving deniability they +must not be signed using the ed25519 key for a device. + +A device will generate a number of these keys and publish them onto their +homeserver. A device will periodically check how many one time keys their +homeserver still has. If the number has become too small then the device will +generate new one time keys and upload them to the homeserver. + +Devices will store the private part of each one time key they upload. They can +discard the private part of the one time key when they receive a message using +that key. However one-keys given out by a homeserver may never end up being +used. Therefore a device may end up trying to store too many private keys. A +device that is trying to store too many private keys may discard keys starting +with the oldest. + +A homeserver should ratelimit the number of one time keys that a given user or +remote server can take. A homeserver should discard the public part of a one +time key once it has given that key to another user. + + +.. code:: http + + POST /keys/take HTTP/1.1 + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": "" + } } } + +.. code:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "one_time_keys": { + "": { + "": { + ":": "" + } } } } + + +Sending a Message +~~~~~~~~~~~~~~~~~ + +Encrypted messages are sent in the form. + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "" + } + } + + +.. code:: json + + { + "type": "m.room.message" + "content": {} + "encrypted": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "ciphertexts": { + "" { + ": { + "type": 0, + "body": "" + } } } } } From 01927cee9b6d8152579eaa24d4ae6572933e9811 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:21:25 +0100 Subject: [PATCH 002/111] Rename "take" to "claim". Hyphenate "one-time". --- specification/41_end_to_end_encryption.rst | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a09c8fc60..f30fc09d9 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -37,14 +37,14 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm takes any one time keys needed. + 3) Alice selects an algorithm claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | +----------------+ +------------+ +----------+ | | | |=================>|==============>| - /keys/take + /keys/claim 4) Alice sends an encrypted message to Bob. @@ -97,7 +97,7 @@ signing key. This key is used as the fingerprint for a device by other clients. Downloading Keys ~~~~~~~~~~~~~~~~ -Keys are downloaded a collection of signed JSON objects. There +Keys are downloaded as a collection of signed JSON objects. There will be JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver will synthesise a JSON object without any device keys @@ -150,34 +150,35 @@ lies about the keys a user owns. } } } } } } -Taking One Time Keys +Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~ -Some algorithms require one time keys to improve their secrecy and deniability. -Theses keys are used once during session establishment, and are then thrown +Some algorithms require one-time keys to improve their secrecy and deniability. +These keys are used once during session establishment, and are then thrown away. In order for these keys to be useful for improving deniability they must not be signed using the ed25519 key for a device. -A device will generate a number of these keys and publish them onto their -homeserver. A device will periodically check how many one time keys their -homeserver still has. If the number has become too small then the device will -generate new one time keys and upload them to the homeserver. - -Devices will store the private part of each one time key they upload. They can -discard the private part of the one time key when they receive a message using -that key. However one-keys given out by a homeserver may never end up being -used. Therefore a device may end up trying to store too many private keys. A -device that is trying to store too many private keys may discard keys starting -with the oldest. - -A homeserver should ratelimit the number of one time keys that a given user or -remote server can take. A homeserver should discard the public part of a one +A device must generate a number of these keys and publish them onto their +homeserver. A device must periodically check how many one-time keys their +homeserver still has. If the number has become too small then the device must +generate new one-time keys and upload them to the homeserver. + +Devices must store the private part of each one-time key they upload. They can +discard the private part of the one-time key when they receive a message using +that key. However it's possible that a one-time key given out by a homeserver +will never be used, so the device that generates the key will never know that +it can discard the key. Therefore a device could end up trying to store too +many private keys. A device that is trying to store too many private keys may +discard keys starting with the oldest. + +A homeserver should ratelimit the number of one-time keys that a given user or +remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. .. code:: http - POST /keys/take HTTP/1.1 + POST /keys/claim HTTP/1.1 Content-Type: application/json { @@ -211,8 +212,7 @@ Encrypted messages are sent in the form. "content": {} "encrypted": { "algorithm": "" - } - } + } } .. code:: json From 42ad1f861240998658fb643c568db70cc2d86c2e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 09:38:08 +0100 Subject: [PATCH 003/111] Add a link to signing JSON section of the spec. Fixup the markup a bit --- specification/41_end_to_end_encryption.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f30fc09d9..19ee0cf1d 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -66,6 +66,8 @@ Keys are uploaded as a signed JSON object. The JSON object must include an ed25519 key and must be signed by that key. A device may only have one ed25519 signing key. This key is used as the fingerprint for a device by other clients. +The JSON object is signed using the process given by `Signing JSON`_. + .. code:: http @@ -90,17 +92,16 @@ signing key. This key is used as the fingerprint for a device by other clients. } } }, "one_time_keys": { ":": "" - }, - } + } } Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be JSON object per device per user. If one of the user's +will be a JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their -homeserver will synthesise a JSON object without any device keys +homeserver must synthesise a JSON object without any device keys for that device. The JSON must be signed by both the homeserver of @@ -151,7 +152,7 @@ lies about the keys a user owns. Claiming One Time Keys -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Some algorithms require one-time keys to improve their secrecy and deniability. These keys are used once during session establishment, and are then thrown @@ -228,3 +229,4 @@ Encrypted messages are sent in the form. "type": 0, "body": "" } } } } } + From 6f69707c71c50f783d1327c55179f2007129cbdb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Jul 2015 19:30:36 +0100 Subject: [PATCH 004/111] Update e2e spec: Group ciphertext by device key rather than device id, add return to docs for /keys/upload, Use "m.room.encrypted" for now, rather than trying to add an encrypted content to arbitrary event types --- specification/41_end_to_end_encryption.rst | 36 ++++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 19ee0cf1d..bf762a3d2 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -94,6 +94,17 @@ The JSON object is signed using the process given by `Signing JSON`_. ":": "" } } +.. code:: http + + 200 OK + Content-Type: application/json + + { + "one_time_key_counts": { + "": 50 + } + } + Downloading Keys ~~~~~~~~~~~~~~~~ @@ -209,10 +220,10 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.message" - "content": {} - "encrypted": { + "type": "m.room.encrypted" + "content": { "algorithm": "" + } } } @@ -220,13 +231,18 @@ Encrypted messages are sent in the form. { "type": "m.room.message" - "content": {} - "encrypted": { + "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": , "ciphertexts": { - "" { - ": { - "type": 0, - "body": "" - } } } } } + ": { + "type": 0, + "body": "" + } } } } + + +The plaintext payload is of the form: + +.. code:: json + TODO: SPEC the JSON plaintext format From 41d204e72c22b768ae4ac0b5dcda59cdcaa09acd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 09:43:46 +0100 Subject: [PATCH 005/111] Name the key 'ciphertext' rather than 'ciphertexts' --- specification/41_end_to_end_encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index bf762a3d2..553664f2e 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -223,8 +223,7 @@ Encrypted messages are sent in the form. "type": "m.room.encrypted" "content": { "algorithm": "" - } - } } + } } } .. code:: json @@ -234,7 +233,7 @@ Encrypted messages are sent in the form. "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": , - "ciphertexts": { + "ciphertext": { ": { "type": 0, "body": "" @@ -245,4 +244,5 @@ The plaintext payload is of the form: .. code:: json - TODO: SPEC the JSON plaintext format + { + } From 6597aaa448023b31fa62e23183e4a4eeeae0e22d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 11:21:14 +0100 Subject: [PATCH 006/111] Start describing the plaintext payload format for encrypted messages, add the exact URLs used for key queries from clients and for key queries for federation --- specification/41_end_to_end_encryption.rst | 57 ++++++++++++++++------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 553664f2e..4c64e92fe 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -96,7 +96,7 @@ The JSON object is signed using the process given by `Signing JSON`_. .. code:: http - 200 OK + HTTP/1.1 200 OK Content-Type: application/json { @@ -162,6 +162,12 @@ lies about the keys a user owns. } } } } } } +Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/query``. + + Claiming One Time Keys ~~~~~~~~~~~~~~~~~~~~~~ @@ -212,6 +218,11 @@ time key once it has given that key to another user. } } } } +Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to +claim keys for any user they wish to contact. Homeservers will respond with the +keys for their local users and forward requests for remote users to +``/_matrix/federation/v1/user/keys/claim``. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -220,24 +231,27 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" - "content": { - "algorithm": "" - } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "" + } } +Using Olm +######### + .. code:: json { - "type": "m.room.message" - "content": { - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": , - "ciphertext": { - ": { - "type": 0, - "body": "" - } } } } + "type": "m.room.encrypted" + "content": { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": "", + "ciphertext": { + "": { + "type": 0, + "body": "" + } } } } The plaintext payload is of the form: @@ -245,4 +259,19 @@ The plaintext payload is of the form: .. code:: json { + "type": "", + "content": "", + "room_id": "", + "fingerprint": "" } + +The type and content of the plaintext message event are given in the payload. +Encyrpting state events is not supported. + +We include the room ID in the payload, because otherwise the homeserver would +be able to change the room a message was sent in. We include a hash of the +participating keys so that clients can detect if another device is unexpectedly +included in the conversation. + +Clients must confirm that the ``sender_key`` actually belongs to the device +that sent the message. From c83e8480e83db63e76ffcbe62734de773794fb8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 15:55:21 +0100 Subject: [PATCH 007/111] Fix JSON syntax --- specification/41_end_to_end_encryption.rst | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 4c64e92fe..5f486f66a 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -86,8 +86,8 @@ The JSON object is signed using the process given by `Signing JSON`_. "keys": { ":": "", }, - "signatures:" { - "" { + "signatures": { + "": { ":": "" } } }, "one_time_keys": { @@ -150,7 +150,7 @@ lies about the keys a user owns. "keys": { ":": "", }, - "signatures:" { + "signatures": { "": { ":": "" }, @@ -163,9 +163,10 @@ lies about the keys a user owns. Clients use ``/_matrix/client/v2_alpha/keys/query`` on their own homeservers to -claim keys for any user they wish to contact. Homeservers will respond with the +query keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/query``. +``/_matrix/federation/v1/user/keys/query`` over federation to the remote +server. Claiming One Time Keys @@ -221,7 +222,9 @@ time key once it has given that key to another user. Clients use ``/_matrix/client/v2_alpha/keys/claim`` on their own homeservers to claim keys for any user they wish to contact. Homeservers will respond with the keys for their local users and forward requests for remote users to -``/_matrix/federation/v1/user/keys/claim``. +``/_matrix/federation/v1/user/keys/claim`` over federation to the remote +server. + Sending a Message ~~~~~~~~~~~~~~~~~ @@ -231,7 +234,7 @@ Encrypted messages are sent in the form. .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "" } } @@ -243,7 +246,7 @@ Using Olm .. code:: json { - "type": "m.room.encrypted" + "type": "m.room.encrypted", "content": { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": "", @@ -273,5 +276,5 @@ be able to change the room a message was sent in. We include a hash of the participating keys so that clients can detect if another device is unexpectedly included in the conversation. -Clients must confirm that the ``sender_key`` actually belongs to the device -that sent the message. +Clients must confirm that the ``sender_key`` belongs to the user that sent the +message. From e1f201f9e636fad8cd3ec2b031a58a0fd7018729 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:07:03 +0100 Subject: [PATCH 008/111] Add description of the olm type and body JSON keys --- specification/41_end_to_end_encryption.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 5f486f66a..1e60fad96 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -256,6 +256,21 @@ Using Olm "body": "" } } } } +The ciphertext is a mapping from device curve25519 key to an encypted payload +for that device. The ``body`` is a base64 encoded message body. The type is an +integer indicating the type of the message body: 0 for the intial pre-key +message, 1 for ordinary messages. + +Olm sessions will generate messages with a type of 0 until they receive a +message. Once a session has decrypted a message it will produce messages with +a type of 1. + +When a client receives a message with a type of 0 it must first check if it +already has a matching session. If it does then it will use that session to +try to decrypt the message. If there is no existing session then the client +must create a new session and use the new session to decrypt the message. A +client must not persist a session or remove one-time keys used by a session +until it has sucessfully decrypted a message using that session. The plaintext payload is of the form: From d06580a481f5396c53574ff86e4830b0cb47f7b4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Jul 2015 16:09:14 +0100 Subject: [PATCH 009/111] Spelling --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 1e60fad96..f0d8a5000 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -258,7 +258,7 @@ Using Olm The ciphertext is a mapping from device curve25519 key to an encypted payload for that device. The ``body`` is a base64 encoded message body. The type is an -integer indicating the type of the message body: 0 for the intial pre-key +integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. Olm sessions will generate messages with a type of 0 until they receive a @@ -270,7 +270,7 @@ already has a matching session. If it does then it will use that session to try to decrypt the message. If there is no existing session then the client must create a new session and use the new session to decrypt the message. A client must not persist a session or remove one-time keys used by a session -until it has sucessfully decrypted a message using that session. +until it has successfully decrypted a message using that session. The plaintext payload is of the form: @@ -284,7 +284,7 @@ The plaintext payload is of the form: } The type and content of the plaintext message event are given in the payload. -Encyrpting state events is not supported. +Encrypting state events is not supported. We include the room ID in the payload, because otherwise the homeserver would be able to change the room a message was sent in. We include a hash of the From 88176ef14845af72cc968d04ad093d8927bdbc1b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:05:51 +0100 Subject: [PATCH 010/111] Add notes on algorithm naming. Fix some typos --- specification/41_end_to_end_encryption.rst | 77 +++++++++++++++++----- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index f0d8a5000..a2b4ff39a 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -15,7 +15,7 @@ may be found at https://lwn.net/Articles/634144/ Overview -======== +-------- .. code:: @@ -37,7 +37,7 @@ Overview |=================>|==============>| /keys/query - 3) Alice selects an algorithm claims any one-time keys needed. + 3) Alice selects an algorithm and claims any one-time keys needed. +----------------+ +------------+ +----------+ | Alice's Device | | Alice's HS | | Bob's HS | @@ -56,6 +56,45 @@ Overview /send/ +Algorithms +---------- + +There are two kinds of algorithms: messaging algorithms and key algorithms. +Messaging algorithms are used to securely send messages between devices. +Key algorithms are used for key agreement and digital signatures. + +Messaging Algorithm Names +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messaging algorithm names use the extensible naming scheme used throughout this +specification. Algorithm names that start with "m." are reserved for algorithms +defined by this specification. Implementations wanting to experiment with new +algorithms are encouraged to pick algorithm names that start with their +domain to reduce the risk of collisions. + +The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for +ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the +hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 +for authenticated encryption. + +Algorithm names should be short and meaningful. A name of "m.olm.v1" is too +short. However a name of +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +is too long despite giving a more precise description of the algorithm. + +Algorithm names should list the primitives used by the algorithm so that it +easier to see if the algorithm is using a broken primitive. + +Key Algorithms +~~~~~~~~~~~~~~ + +The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +a Base64 encoded 32-byte Ed25519 public key. + +The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +a Base64 encoded 32-byte Curve25519 public key. + Client Behaviour ---------------- @@ -81,17 +120,17 @@ The JSON object is signed using the process given by `Signing JSON`_. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { - ":": "", + ":": "", }, "signatures": { "": { - ":": "" + ":": "" } } }, "one_time_keys": { - ":": "" + ":": "" } } .. code:: http @@ -101,7 +140,7 @@ The JSON object is signed using the process given by `Signing JSON`_. { "one_time_key_counts": { - "": 50 + "": 50 } } @@ -145,20 +184,20 @@ lies about the keys a user owns. "valid_after_ts": 1234567890123, "valid_until_ts": 2345678901234, "algorithms": [ - "", + "", ], "keys": { ":": "", }, "signatures": { "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" }, "": { - ":": "" + ":": "" } } } } } } @@ -190,7 +229,7 @@ it can discard the key. Therefore a device could end up trying to store too many private keys. A device that is trying to store too many private keys may discard keys starting with the oldest. -A homeserver should ratelimit the number of one-time keys that a given user or +A homeserver should rate-limit the number of one-time keys that a given user or remote server can claim. A homeserver should discard the public part of a one time key once it has given that key to another user. @@ -203,7 +242,7 @@ time key once it has given that key to another user. { "one_time_keys": { "": { - "": "" + "": "" } } } .. code:: http @@ -215,7 +254,7 @@ time key once it has given that key to another user. "one_time_keys": { "": { "": { - ":": "" + ":": "" } } } } @@ -225,7 +264,6 @@ keys for their local users and forward requests for remote users to ``/_matrix/federation/v1/user/keys/claim`` over federation to the remote server. - Sending a Message ~~~~~~~~~~~~~~~~~ @@ -236,13 +274,18 @@ Encrypted messages are sent in the form. { "type": "m.room.encrypted", "content": { - "algorithm": "" + "algorithm": "", + "": "" } } Using Olm ######### +Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their +list of supported chat algorithms, must list a Curve25519 device key, and +must publish Curve25519 one-time keys. + .. code:: json { @@ -256,7 +299,7 @@ Using Olm "body": "" } } } } -The ciphertext is a mapping from device curve25519 key to an encypted payload +The ciphertext is a mapping from device curve25519 key to an encrypted payload for that device. The ``body`` is a base64 encoded message body. The type is an integer indicating the type of the message body: 0 for the initial pre-key message, 1 for ordinary messages. From 7d805f105e1123aa9fbfb3fe3ba2b8f7fd80950a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Aug 2015 15:08:52 +0100 Subject: [PATCH 011/111] Mention that Olm uses AES in CBC mode --- specification/41_end_to_end_encryption.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index a2b4ff39a..023a56848 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -75,12 +75,12 @@ domain to reduce the risk of collisions. The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the -hash ratchet, and HKDF-SHA-256, AES-256, and 8 byte truncated HMAC-SHA-256 -for authenticated encryption. +hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated +HMAC-SHA-256 for authenticated encryption. Algorithm names should be short and meaningful. A name of "m.olm.v1" is too short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-hmac64sha256" +"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" is too long despite giving a more precise description of the algorithm. Algorithm names should list the primitives used by the algorithm so that it From 12e33a3b09ac85af785d077f80543984e1f71dc5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:14:24 +0000 Subject: [PATCH 012/111] Document a v2 api for setting tags on rooms --- api/client-server/v2_alpha/tags.yaml | 148 +++++++++++++++++++++++++ event-schemas/examples/v1/m.tag | 8 ++ event-schemas/schema/v1/m.tag | 23 ++++ specification/modules/room_tagging.rst | 21 ++++ 4 files changed, 200 insertions(+) create mode 100644 api/client-server/v2_alpha/tags.yaml create mode 100644 event-schemas/examples/v1/m.tag create mode 100644 event-schemas/schema/v1/m.tag create mode 100644 specification/modules/room_tagging.rst diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml new file mode 100644 index 000000000..91b812124 --- /dev/null +++ b/api/client-server/v2_alpha/tags.yaml @@ -0,0 +1,148 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server tag API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/rooms/{roomId}/tags": + get: + summary: List the tags for a room. + description: |- + List the tags set by a user on a room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to get tags for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to get tags for. + x-example: "!726s6s6q:example.com" + responses: + 200: + description: + The list of tags for the user for the room. + schema: + type: object + properties: + tags: + type: array + items: + type: string + examples: + application/json: |- + { + "tags": [ + "work", + "pinned" + ] + } + "/user/{userId}/rooms/{roomId}/tags/{tag}": + put: + summary: Add a tag to a room. + description: |- + Add a tag to the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to add a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to add a tag to. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to add. + x-example: "work" + - in: body + name: body + required: true + description: |- + An empty JSON object. + schema: + type: object + example: |- + {} + responses: + 200: + description: + The tag was successfully added. + schema: + type: object + examples: + application/json: |- + {} + delete: + summary: Remove a tag from the room. + description: |- + Remove a tag from the room. + security: + - access_token: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to remove a tag for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to remove a tag from. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: tag + required: true + description: |- + The tag to remove. + x-example: "work" + responses: + 200: + description: + The tag was successfully removed + schema: + type: object + examples: + application/json: |- + {} diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag new file mode 100644 index 000000000..28431b73a --- /dev/null +++ b/event-schemas/examples/v1/m.tag @@ -0,0 +1,8 @@ +{ + "type": "m.tag", + "content": { + "tags": [ + "work" + ] + } +} diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag new file mode 100644 index 000000000..235d442d2 --- /dev/null +++ b/event-schemas/schema/v1/m.tag @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "Tag Event", + "description": "Informs the client of tags on a room.", + "properties": { + "type": { + "type": "string", + "enum": ["m.tag"] + }, + "content": { + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "required": ["type", "content"] +} diff --git a/specification/modules/room_tagging.rst b/specification/modules/room_tagging.rst new file mode 100644 index 000000000..76fff1251 --- /dev/null +++ b/specification/modules/room_tagging.rst @@ -0,0 +1,21 @@ +Room Tagging +============ + +.. _module:tagging: + +Users can add tags to rooms. Tags are short strings used to label rooms, e.g. +"work", "familly". A room may have multiple tags. Tags are only visible to the +user that set them but are shared across all their devices. + +Events +------ + +The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` +section of a v2 room sync. + +{{m_tag_event}} + +Client Behaviour +---------------- + +{{v2_tags_http_api}} From 9b0d20315ab6cc5e13cb6a72d02c0cb328b2eb9d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 16:22:04 +0000 Subject: [PATCH 013/111] Add the tags module to the specification targets --- specification/modules/{room_tagging.rst => tags.rst} | 0 specification/targets.yaml | 1 + 2 files changed, 1 insertion(+) rename specification/modules/{room_tagging.rst => tags.rst} (100%) diff --git a/specification/modules/room_tagging.rst b/specification/modules/tags.rst similarity index 100% rename from specification/modules/room_tagging.rst rename to specification/modules/tags.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfda..2c6a8f205 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/tags.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 65066a76b36de8595bf5b67e075c631c8e77d5b1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 18:30:04 +0000 Subject: [PATCH 014/111] Add the m.tags event to a ``private_user_data`` key rather than including it under the ``ephemeral`` key --- api/client-server/v2_alpha/sync.yaml | 17 ++++++++++++++++- specification/modules/tags.rst | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 266f27bc4..3a1d79db6 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -121,6 +121,14 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" + private_user_data: + title: Private User Data + type: object + description: |- + The private data that this user has attached to + this room. + allOf: + - $ref: "definitions/event_batch.json" invited: title: Invited type: object @@ -253,11 +261,18 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } ] + }, + "private_user_data": { + "events": [ + { + "type": "m.tags", + "content": {"tags": ["work"]} + } + ] } } }, diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 76fff1251..1a364a434 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,8 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the ``ephemeral`` -section of a v2 room sync. +The tags on a room are passed as single ``m.tag`` event in the +``private_user_data`` section of a room in v2 sync. {{m_tag_event}} From 81a60a25ccd78b413d5fc0a25e6267dcf05db23d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:46:10 +0000 Subject: [PATCH 015/111] Update 3pid spec based on new implementation --- event-schemas/examples/v1/m.room.member | 15 ---------- event-schemas/schema/v1/m.room.member | 20 ++----------- specification/modules/third_party_invites.rst | 29 ++++++++++--------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a6..d7605dcd1 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -4,21 +4,6 @@ "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad705..1d09a0dd8 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -26,18 +26,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -57,13 +45,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c314..140bab86d 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -15,13 +15,15 @@ The homeserver asks the identity server whether a Matrix user ID is known for that identifier. If it is, an invite is simply issued for that user. If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets +the invitation, and to notify the invitee's homeserver of this pending invitation if it gets a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property whichi contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -39,9 +41,10 @@ Server behaviour All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -91,16 +94,16 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. + When the third party user validates their identity, their homeserver, H3, + is notified, and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. + H3 validates the signature given to it by the identity server. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 + Having validated these things, H1 writes the invite event to the room, and H3 begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking From b92a0f2b4de47093b229e9f0521aec2bb008435c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 14:49:22 +0000 Subject: [PATCH 016/111] Remove extra trailing comma --- event-schemas/examples/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index d7605dcd1..e2ca56685 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,7 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", + "displayname": "Alice Margatroid" }, "invite_room_state": [ { From 176f919fc89f5d01669aea17003aef8da903ce4e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:00:53 +0000 Subject: [PATCH 017/111] Show multiple examples where present --- .../v1/m.room.member#third_party_invite | 41 +++++++++++++++++++ templating/matrix_templates/sections.py | 2 +- .../matrix_templates/templates/events.tmpl | 4 +- templating/matrix_templates/units.py | 11 +++-- 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite new file mode 100644 index 000000000..4ef571ec5 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -0,0 +1,41 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid", + "third_party_invite": { + "signed": { + "mxid": "@alice:localhost", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + }, + "token": "abc123" + } + } + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index e75a75af8..81b2bb3cc 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -35,7 +35,7 @@ class MatrixSections(Sections): if not filterFn(event_name): continue sections.append(template.render( - example=examples[event_name], + examples=examples[event_name], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440c..fbefe17f3 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -21,8 +21,10 @@ ======================= ================= =========================================== {% endfor %} -Example: +Example{% if examples | length > 1 %}s{% endif %}: +{% for example in examples %} .. code:: json {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c43421411..d33ba81df 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -495,9 +495,14 @@ class MatrixUnits(Units): if not filename.startswith("m."): continue with open(os.path.join(path, filename), "r") as f: - examples[filename] = json.loads(f.read()) - if filename == "m.room.message#m.text": - examples["m.room.message"] = examples[filename] + event_name = filename.split("#")[0] + example = json.loads(f.read()) + + examples[filename] = examples.get(event_name, []) + examples[filename].append(example) + if filename != event_name: + examples[event_name] = examples.get(event_name, []) + examples[event_name].append(example) return examples def load_event_schemas(self): From 810922bb381a0f7162941605ec631f00927071ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 28 Oct 2015 15:06:44 +0000 Subject: [PATCH 018/111] Fix schema validator for multiple examples --- event-schemas/check_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index e54d3a1ca..b10b2d0ef 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir): continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) + if schemapath.find("#") >= 0: + schemapath = schemapath[:schemapath.find("#")] try: check_example_file(examplepath, schemapath) except Exception as e: From 5b6f858802332e8fcee368e9db6cdfac7533c995 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 28 Oct 2015 20:28:49 +0000 Subject: [PATCH 019/111] Some initial notes by way of the remote join handshake; with several TODOs and unanswered questions --- specification/server_server_api.rst | 149 +++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 26e040ca2..f02391286 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -533,6 +533,150 @@ part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. + +To join a room:: + + GET .../make_join// + Response: JSON encoding of a join proto-event + + PUT .../send_join// + Response: JSON encoding of the state of the room at the time of the event + +Performs the room join handshake. For more information, see "Joining Rooms" +below. + +Joining Rooms +------------- + +When a new user wishes to join room that the user's homeserver already knows +about, the homeserver can immediately determine if this is allowable by +inspecting the state of the room, and if it is acceptable, it can generate, +sign, and emit a new ``m.room.member`` state event adding the user into that +room. When the homeserver does not yet know about the room it cannot do this +directly. Instead, it must take a longer multi-stage handshaking process by +which it first selects a remote homeserver which is already participating in +that room, and uses it to assist in the joining procss. This is the remote join +handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting the +room alias the user is requesting to join with, and a homeserver where existing +room members are already present (referred to as the "resident" server). + +In summary, the remote join handshake consists of the joining server querying +the directory server for information about the room alias; receiving a room ID +and a list of join candidates. The joining server then requests information +about the current state of the room from one of the residents. It uses this +information to construct a ``m.room.member`` event which it finally sends to +a resident server. + +Conceptually these are three different roles of homeserver. In practice the +directory server is likely to be resident in the room, and so may be selected +by the joining server to be the assisting resident. Likewise, it is likely that +the joining server picks the same candidate resident for both phases of event +construction, though in principle any valid candidate may be used at each time. +Thus, any join handshake can potentially involve anywhere from two to four +homeservers, though most in practice will use just two. + +.. TODO-doc + - Consider drawing a request/response diagram here + +The first part of the handshake involves using the directory server to request +the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. + +Once the joining server has the room ID and the join candidates, it then needs +to obtain enough of the current state of the room to fill in the required +fields of the ``m.room.member`` event. It obtains this by selecting a resident +from the candidate list, and requesting the ``make_join`` endpoint using a +``GET`` request, specifying the room ID and the user ID of the new member who +is attempting to join. + +The resident server replies to this request with a JSON-encoded object having a +single key called ``event``; within this is an object whose fields contain some +of the information that the joining server will need. Despite its name, this +object is not a full object; notably it does not need to be hashed or signed by +the assisting resident. The required fields are: + +==================== ======== ============ + Key Type Description +==================== ======== ============ +``type`` String The value ``m.room.member`` +``auth_events`` String An event-reference list containing the + authorization events that would allow this member + to join +``content`` Object The event content +``depth`` Integer (this field must be present but is ignored; it + may be 0) +``event_id`` String A new event ID specified by the assisting + resident +``origin`` String The name of the assisting resident homeserver +``origin_server_ts`` Integer A timestamp added by the resident homeserver +``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_state`` List (TODO(paul): ? - I notice these can be blank) +``room_id`` String The room ID of the room +``sender`` String The user ID of the joining member +``state_key`` String The user ID of the joining member +==================== ======== ============ + +The ``content`` field itself must be an object, containing: + +============== ====== ============ + Key Type Description +============== ====== ============ +``membership`` String The value ``join`` +============== ====== ============ + +The joining server now has sufficient information to construct the real join +event from these protoevent fields. It copies the values of most of them, +adding (or replacing) the following fields: + +==================== ======= ============ + Key Type Description +==================== ======= ============ +``event_id`` String A new event ID specified by the joining homeserver +``origin`` String The name of the joining homeserver +``origin_server_ts`` Integer A timestamp added by the joining homeserver +==================== ======= ============ + +.. TODO-spec + - Why does the protoevent have an event_id, only for the real event to ignore + it and specify a different one? We should definitely pick one or the other. + +This will be a true event, so the joining server should apply the event-signing +algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` +fields. + +To complete the join handshake, the joining server must now submit this new +event to an assisting resident, by using the ``send_join`` endpoint. This is +invoked using the room ID and the event ID of the new member event. + +The assisting resident then accepts this event into the room's event graph, and +responds to the joining server with the full set of state for the newly-joined +room. This is returned as a two-element list, whose first element is the +integer 200, and whose second element contains the following keys: + +.. TODO-spec + - This is likely an implementation bug; see SYN-490. This should probably + actually just return the object directly + +============== ===== ============ + Key Type Description +============== ===== ============ +``auth_chain`` List A list of events giving the authorization chain for this + join event +``state`` List A complete list of the prevailing state events at the + instant just before accepting the new ``m.room.member`` + event +============== ===== ============ + +.. TODO-spec + - (paul) I don't really understand why the full auth_chain events are given + here. What purpose does it serve expanding them out in full, when surely + they'll appear in the state anyway? + - (paul) the state seems to be entirely ignored by synapse, so I'm not really + sure what ought to be there. + Backfilling ----------- .. NOTE:: @@ -763,6 +907,7 @@ Querying directory information:: servers: list of strings giving the join candidates The list of join candidates is a list of server names that are likely to hold -the given room; these are servers that the requesting server may wish to try -joining with. This list may or may not include the server answering the query. +the given room; these are servers that the requesting server may wish to use as +assisting resident servers as part of the remote join handshake. This list may +or may not include the server answering the query. From cddfc110edc9018e44ce1a3690f3f6431a0ea9b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 29 Oct 2015 12:48:04 +0000 Subject: [PATCH 020/111] Review comments --- specification/modules/third_party_invites.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 140bab86d..6d3f8f6a5 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -22,7 +22,7 @@ and public key to the inviting homeserver. When the invitee's homeserver receives the notification of the binding, it should insert an ``m.room.member`` event into the room's graph for that user, with ``content.membership`` = ``invite``, as well as a -``content.third_party_invite`` property whichi contains proof that the invitee +``content.third_party_invite`` property which contains proof that the invitee does indeed own that third party identifier. Events @@ -94,8 +94,8 @@ For example: H1 asks the identity server for a binding to a Matrix user ID, and has none, so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, their homeserver, H3, - is notified, and attempts to issue an ``m.room.member`` event to participate + When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate in the room. H3 validates the signature given to it by the identity server. From 9f4d81308d3eca3870f54c196335d2bb5176bd44 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 30 Oct 2015 16:21:50 +0000 Subject: [PATCH 021/111] Pull out separate invite_room_state example --- .../v1/m.room.member#invite_room_state | 30 +++++++++++++++++++ .../v1/m.room.member#third_party_invite | 16 ---------- 2 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 event-schemas/examples/v1/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/v1/m.room.member#invite_room_state new file mode 100644 index 000000000..e2ca56685 --- /dev/null +++ b/event-schemas/examples/v1/m.room.member#invite_room_state @@ -0,0 +1,30 @@ +{ + "age": 242352, + "content": { + "membership": "join", + "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", + "displayname": "Alice Margatroid" + }, + "invite_room_state": [ + { + "type": "m.room.name", + "state_key": "", + "content": { + "name": "Forest of Magic" + } + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": { + "join_rules": "invite" + } + } + ], + "state_key": "@alice:localhost", + "origin_server_ts": 1431961217939, + "event_id": "$WLGTSEFSEF:localhost", + "type": "m.room.member", + "room_id": "!Cuyf34gef24t:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/v1/m.room.member#third_party_invite index 4ef571ec5..2457302ac 100644 --- a/event-schemas/examples/v1/m.room.member#third_party_invite +++ b/event-schemas/examples/v1/m.room.member#third_party_invite @@ -16,22 +16,6 @@ } } }, - "invite_room_state": [ - { - "type": "m.room.name", - "state_key": "", - "content": { - "name": "Forest of Magic" - } - }, - { - "type": "m.room.join_rules", - "state_key": "", - "content": { - "join_rules": "invite" - } - } - ], "state_key": "@alice:localhost", "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", From b49472e3b09c385d6a3071c31bdfe425c16174f4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:52:34 +0000 Subject: [PATCH 022/111] Add private_user_data to v1 /initialSync --- api/client-server/v1/sync.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index d07e9399c..8517ae4d3 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -245,7 +245,12 @@ paths: "user_id": "@alice:localhost" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + }] } ] } @@ -332,6 +337,16 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to + this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: From ad86426e9506a9d0807267acf50e617497ebbd96 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 16:55:18 +0000 Subject: [PATCH 023/111] Add private_user_data to v1 room /initialSync --- api/client-server/v1/rooms.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d10..462736031 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -311,7 +311,12 @@ paths: "user_id": "@alice:example.com" } ], - "visibility": "private" + "visibility": "private", + "private_user_data": [{ + "type": "m.tag", + "content": {"tags": ["work"]}, + "room_id": "!636q39766251:example.com" + }] } schema: title: RoomInfo @@ -371,6 +376,15 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." + private_user_data: + type: array + description: |- + The private data that this user has attached to this room. + items: + title: Event + type: object + allOf: + - "$ref": "core-event-schema/event.json" required: ["room_id", "membership"] 403: description: > From f557e698606a2d33f61100cc63bd15a43d7494ba Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 10:13:47 +0000 Subject: [PATCH 024/111] Note that m.tag events can appear in v1 initialSync and /events as well as v2 sync. Only add the room_id for v1 /events since it is redundant in v1 /initialSync --- api/client-server/v1/sync.yaml | 3 +-- specification/modules/tags.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8517ae4d3..0f11a98f2 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,8 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost" + "content": {"tags": ["work"]} }] } ] diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 1a364a434..567e50854 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -10,8 +10,13 @@ user that set them but are shared across all their devices. Events ------ -The tags on a room are passed as single ``m.tag`` event in the -``private_user_data`` section of a room in v2 sync. +The tags on a room are receieved as single ``m.tag`` event in the +``private_user_data`` section of a room in a v2 /sync. + +The ``m.tag`` can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. {{m_tag_event}} From 52f55e05422024e6701b8d211258e1b6a4011a00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:31:25 +0000 Subject: [PATCH 025/111] Allow room tags to have asssociated content, and return that content in the m.tag events --- api/client-server/v2_alpha/tags.yaml | 17 ++++++++--------- event-schemas/examples/v1/m.tag | 6 +++--- event-schemas/schema/v1/m.tag | 7 ++++--- specification/modules/tags.rst | 3 +++ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 91b812124..009e6000c 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -49,16 +49,15 @@ paths: type: object properties: tags: - type: array - items: - type: string + title: Tags + type: object examples: application/json: |- { - "tags": [ - "work", - "pinned" - ] + "tags": { + "work": {"order": 1}, + "pinned": {} + } } "/user/{userId}/rooms/{roomId}/tags/{tag}": put: @@ -94,11 +93,11 @@ paths: name: body required: true description: |- - An empty JSON object. + Extra data for the tag, e.g. ordering. schema: type: object example: |- - {} + {"order": 1} responses: 200: description: diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 28431b73a..1fd3158ed 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -1,8 +1,8 @@ { "type": "m.tag", "content": { - "tags": [ - "work" - ] + "tags": { + "work": {} + } } } diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 235d442d2..2ec1a55f9 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -11,9 +11,10 @@ "type": "object", "properties": { "tags": { - "type": "array", - "items": { - "type": "string" + "type": "object", + "additionalProperties": { + "title": "Tag", + "type": "object" } } } diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 567e50854..c2b65b3e1 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -18,6 +18,9 @@ The ``m.tag`` can also be received in a v1 /events response or in the events appearing in v1 /events will have a ``room_id`` with the room the tags are for. +Each tag has an associated JSON object with information about the tag, e.g how +to order the tags. + {{m_tag_event}} Client Behaviour From 149890227aa62cfd0c01b94b0c532b7a5f2a1b36 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Nov 2015 13:35:11 +0000 Subject: [PATCH 026/111] Fix wording --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index c2b65b3e1..e9b4435e0 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -19,7 +19,7 @@ events appearing in v1 /events will have a ``room_id`` with the room the tags are for. Each tag has an associated JSON object with information about the tag, e.g how -to order the tags. +to order the rooms with a given tag. {{m_tag_event}} From 40fa339cf7790837f550cf1ac7fa580a8442ad80 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:00:18 +0000 Subject: [PATCH 027/111] Draw a pretty (well at least I think it's pretty) ASCII diagram of the remote join handshake --- specification/server_server_api.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f02391286..666333762 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -578,8 +578,23 @@ construction, though in principle any valid candidate may be used at each time. Thus, any join handshake can potentially involve anywhere from two to four homeservers, though most in practice will use just two. -.. TODO-doc - - Consider drawing a request/response diagram here +:: + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response The first part of the handshake involves using the directory server to request the room ID and join candidates. This is covered in more detail on the From f6c55979e0cdf6881ca4211745b9fc5cd6f6fbb7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 2 Nov 2015 15:17:18 +0000 Subject: [PATCH 028/111] Remove TODO comment about SYN-490 as it's unlikely to matter for v1; we'll fix it in v2 --- specification/server_server_api.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 666333762..d26cc26ec 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -671,10 +671,6 @@ responds to the joining server with the full set of state for the newly-joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: -.. TODO-spec - - This is likely an implementation bug; see SYN-490. This should probably - actually just return the object directly - ============== ===== ============ Key Type Description ============== ===== ============ From e9d361841b8be0f3f4edc2e042439cc77c57225f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:42:58 +0000 Subject: [PATCH 029/111] Fix tag examples --- api/client-server/v1/rooms.yaml | 3 +-- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 462736031..e7b93ad95 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,8 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]}, - "room_id": "!636q39766251:example.com" + "content": {"tags": {"work": {}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 0f11a98f2..89b9400e9 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 3a1d79db6..b5465601c 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": ["work"]} + "content": {"tags": {"work": {}}} } ] } From bcb8fac53c3b6beb47777148a387604d11c71a7d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:46:01 +0000 Subject: [PATCH 030/111] Add a description for the tag event --- event-schemas/schema/v1/m.tag | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index 2ec1a55f9..a36c6a450 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -12,13 +12,14 @@ "properties": { "tags": { "type": "object", + "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag", + "title": "Tag Contents", "type": "object" } } } } }, - "required": ["type", "content"] + "required": ["type", "content"] } From fba3c04e42113e4d3a079b9945abe737f68dc154 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:48:30 +0000 Subject: [PATCH 031/111] Apparently the spec generator breaks if the title in a schema is too long --- event-schemas/schema/v1/m.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/v1/m.tag index a36c6a450..4c5b4fa53 100644 --- a/event-schemas/schema/v1/m.tag +++ b/event-schemas/schema/v1/m.tag @@ -14,7 +14,7 @@ "type": "object", "description": "The tags on the room and their contents.", "additionalProperties": { - "title": "Tag Contents", + "title": "Tag", "type": "object" } } From 3953006792cca87800af18076e715715322c830b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:54:33 +0000 Subject: [PATCH 032/111] Fix spelling --- specification/modules/tags.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index e9b4435e0..3d45975c4 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -4,13 +4,13 @@ Room Tagging .. _module:tagging: Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "familly". A room may have multiple tags. Tags are only visible to the +"work", "family". A room may have multiple tags. Tags are only visible to the user that set them but are shared across all their devices. Events ------ -The tags on a room are receieved as single ``m.tag`` event in the +The tags on a room are received as single ``m.tag`` event in the ``private_user_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the From d53814097f11ca5f24bb6b1af0b1b9863243f173 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 15:57:06 +0000 Subject: [PATCH 033/111] Add example content to the tags in example tag events --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- event-schemas/examples/v1/m.tag | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index e7b93ad95..9b8b930f8 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 89b9400e9..b52908a99 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index b5465601c..4c1a7601b 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -270,7 +270,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {}}} + "content": {"tags": {"work": {"order": 1}}} } ] } diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/v1/m.tag index 1fd3158ed..00e81060b 100644 --- a/event-schemas/examples/v1/m.tag +++ b/event-schemas/examples/v1/m.tag @@ -2,7 +2,7 @@ "type": "m.tag", "content": { "tags": { - "work": {} + "work": {"order": 1} } } } From 30a4f17420f1c6b21648c7e5f729fb4d84468d54 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 3 Nov 2015 16:15:36 +0000 Subject: [PATCH 034/111] Don't mark 'aliases' on m.room.canonical_alias as required, because it isn't. --- event-schemas/schema/v1/m.room.canonical_alias | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias index 25cd00c08..49e0e669a 100644 --- a/event-schemas/schema/v1/m.room.canonical_alias +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -13,8 +13,7 @@ "type": "string", "description": "The canonical alias." } - }, - "required": ["alias"] + } }, "state_key": { "type": "string", From c00abe9f2faa80f6cfcf7fba028de870eca676ac Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:26:51 +0000 Subject: [PATCH 035/111] Fix msgtype display --- templating/matrix_templates/sections.py | 2 +- templating/matrix_templates/units.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 81b2bb3cc..78aabca76 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -136,7 +136,7 @@ class MatrixSections(Sections): if not event_name.startswith("m.room.message#m."): continue sections.append(template.render( - example=examples[event_name], + example=examples[event_name][0], event=schemas[event_name], title_kind=subtitle_title_char )) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 5ae117ca4..94435c52e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -562,7 +562,7 @@ class MatrixUnits(Units): event_name = filename.split("#")[0] example = json.loads(f.read()) - examples[filename] = examples.get(event_name, []) + examples[filename] = examples.get(filename, []) examples[filename].append(example) if filename != event_name: examples[event_name] = examples.get(event_name, []) From 4d3175fc8b5c01a41afb802c2e4bc0e191b822db Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 10:45:57 +0000 Subject: [PATCH 036/111] Note that invite_room_state is optional --- event-schemas/schema/v1/m.room.member | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 81057049c..25b301097 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], From d7c69fae438410e1b91030e60466f48f28e27360 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:33:34 +0000 Subject: [PATCH 037/111] Fix typo 'process' --- specification/server_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d26cc26ec..1c3e761de 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -555,8 +555,8 @@ sign, and emit a new ``m.room.member`` state event adding the user into that room. When the homeserver does not yet know about the room it cannot do this directly. Instead, it must take a longer multi-stage handshaking process by which it first selects a remote homeserver which is already participating in -that room, and uses it to assist in the joining procss. This is the remote join -handshake. +that room, and uses it to assist in the joining process. This is the remote +join handshake. This handshake involves the homeserver of the new member wishing to join (referred to here as the "joining" server), the directory server hosting the From aac45295ee9dc615cfc7290cc928c011ca0d4e79 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:34:22 +0000 Subject: [PATCH 038/111] Remark that the directory server step could be skipped in an invite case --- specification/server_server_api.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 1c3e761de..e158ae23d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -596,9 +596,12 @@ homeservers, though most in practice will use just two. | <---------- join response -The first part of the handshake involves using the directory server to request -the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. +The first part of the handshake usually involves using the directory server to +request the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver could +optimise this step away by picking the origin server of that invite message as +the join candidate. Once the joining server has the room ID and the join candidates, it then needs to obtain enough of the current state of the room to fill in the required From db5a90edcd7fcc33385d2145b2ad1ed93313182f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:36:26 +0000 Subject: [PATCH 039/111] Avoid using the words 'current state' when talking about the result of the /make_join request --- specification/server_server_api.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e158ae23d..6dfbd86f1 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -566,9 +566,8 @@ room members are already present (referred to as the "resident" server). In summary, the remote join handshake consists of the joining server querying the directory server for information about the room alias; receiving a room ID and a list of join candidates. The joining server then requests information -about the current state of the room from one of the residents. It uses this -information to construct a ``m.room.member`` event which it finally sends to -a resident server. +about the room from one of the residents. It uses this information to construct +a ``m.room.member`` event which it finally sends to a resident server. Conceptually these are three different roles of homeserver. In practice the directory server is likely to be resident in the room, and so may be selected @@ -604,11 +603,11 @@ optimise this step away by picking the origin server of that invite message as the join candidate. Once the joining server has the room ID and the join candidates, it then needs -to obtain enough of the current state of the room to fill in the required -fields of the ``m.room.member`` event. It obtains this by selecting a resident -from the candidate list, and requesting the ``make_join`` endpoint using a -``GET`` request, specifying the room ID and the user ID of the new member who -is attempting to join. +to obtain enough information about the room to fill in the required fields of +the ``m.room.member`` event. It obtains this by selecting a resident from the +candidate list, and requesting the ``make_join`` endpoint using a ``GET`` +request, specifying the room ID and the user ID of the new member who is +attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some From 885dd1e86cc8d6ee818ab32ab246eeec9a1f5ede Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 16:39:31 +0000 Subject: [PATCH 040/111] Explain the 'prev_events' join protoevent key --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6dfbd86f1..ebf55eb4b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -629,7 +629,8 @@ the assisting resident. The required fields are: resident ``origin`` String The name of the assisting resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver -``prev_events`` List (TODO(paul): ? - I notice these can be blank) +``prev_events`` List An event-reference list containing the immediate + predecessor events ``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member From 988d77347651ce2504ab1e52196166358dffe7f7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 13 Nov 2015 17:49:22 +0000 Subject: [PATCH 041/111] /make_join protoevent no longer needs the pointless 'prev_state' key (SYN-517) --- specification/server_server_api.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index ebf55eb4b..9cc10141f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -631,7 +631,6 @@ the assisting resident. The required fields are: ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events -``prev_state`` List (TODO(paul): ? - I notice these can be blank) ``room_id`` String The room ID of the room ``sender`` String The user ID of the joining member ``state_key`` String The user ID of the joining member @@ -688,8 +687,6 @@ integer 200, and whose second element contains the following keys: - (paul) I don't really understand why the full auth_chain events are given here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? - - (paul) the state seems to be entirely ignored by synapse, so I'm not really - sure what ought to be there. Backfilling ----------- From cc8ef691fb20331d135a39993e843d42588e0761 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 16:49:22 +0000 Subject: [PATCH 042/111] Rewrite the table templates Allow columns to stretch if they end up with wide content. Apart from the hassle of having to manually update the calculations, having the columns wide enough to hold the widest thing they might ever have leads to tables with lots of whitespace in the results. --- templating/build.py | 22 ++++ .../templates/common-event-fields.tmpl | 15 +-- .../matrix_templates/templates/events.tmpl | 15 +-- .../matrix_templates/templates/http-api.tmpl | 27 +---- .../matrix_templates/templates/msgtypes.tmpl | 15 +-- .../matrix_templates/templates/tables.tmpl | 104 ++++++++++++++++++ 6 files changed, 139 insertions(+), 59 deletions(-) create mode 100644 templating/matrix_templates/templates/tables.tmpl diff --git a/templating/build.py b/templating/build.py index a35d8a085..77fecf91f 100755 --- a/templating/build.py +++ b/templating/build.py @@ -93,6 +93,27 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return '\n\n'.join(output_lines) + def fieldwidths(input, keys, defaults=[], default_width=15): + """ + A template filter to help in the generation of tables. + + Given a list of rows, returns a list giving the maximum length of the + values in each column. + + :param list[dict[str, str]] input: a list of rows. Each row should be a + dict with the keys given in ``keys``. + :param list[str] keys: the keys corresponding to the table columns + :param list[int] defaults: for each column, the default column width. + :param int default_width: if ``defaults`` is shorter than ``keys``, this + will be used as a fallback + """ + def colwidth(key, default): + return reduce(max, (len(row[key]) for row in input), + default if default is not None else default_width) + + results = map(colwidth, keys, defaults) + return results + # make Jinja aware of the templates and filters env = Environment( loader=FileSystemLoader(in_mod.exports["templates"]), @@ -102,6 +123,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): env.filters["indent"] = indent env.filters["indent_block"] = indent_block env.filters["wrap"] = wrap + env.filters["fieldwidths"] = fieldwidths # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 3f16be3da..8d8c8f0ca 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,17 +1,8 @@ +{% import 'tables.tmpl' as tables -%} + {{common_event.title}} Fields {{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} -================== ================= =========================================== - Key Type Description -================== ================= =========================================== -{% for row in common_event.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc | indent(18 - (row.type|length)) |wrap(43) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440c..324a8f3e0 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.type}}`` {{(4 + event.type | length) * title_kind}} *{{event.typeof}}* @@ -7,18 +9,7 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ================= =========================================== - {{table.title or "Content"}} Key Type Description -======================= ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} -{% endfor -%} -======================= ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98c..1253e66eb 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} {% if "alias_for_path" in endpoint -%} @@ -13,17 +15,7 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== -{% for loc in endpoint.req_param_by_loc -%} -*{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- -{% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} -{% endfor -%} -{% endfor -%} -=========================================== ================= =========================================== +{{ tables.split_paramtable(endpoint.req_param_by_loc) }} {% else %} `No parameters` {% endif %} @@ -34,18 +26,7 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ========================= ========================================== - Param Type Description -======================= ========================= ========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (42 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} -{% endfor -%} -======================= ========================= ========================================== +{{ tables.paramtable(table.rows) }} {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 18d3492bc..87cf4a193 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,21 +1,12 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.msgtype}}`` {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/tables.tmpl b/templating/matrix_templates/templates/tables.tmpl new file mode 100644 index 000000000..aba6c0c49 --- /dev/null +++ b/templating/matrix_templates/templates/tables.tmpl @@ -0,0 +1,104 @@ +{# + # A set of macros for generating RST tables + #} + + +{# + # write a table for a list of parameters. + # + # 'rows' is the list of parameters. Each row should have the keys + # 'key', 'type', and 'desc'. + #} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} +{{ split_paramtable({None: rows}, titles) }} +{% endmacro %} + + +{# + # write a table for the request parameters, split by location. + # 'rows_by_loc' is a map from location to a list of parameters. + # + # As a special case, if a key of 'rows_by_loc' is 'None', no title row is + # written for that location. This is used by the standard 'paramtable' macro. + #} +{% macro split_paramtable(rows_by_loc, + titles=["Parameter", "Type", "Description"]) -%} + +{% set rowkeys = ['key', 'type', 'desc'] %} +{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %} + +{# We need the rows flattened into a single list. Abuse the 'sum' filter to + # join arrays instead of add numbers. -#} +{% set flatrows = rows_by_loc.values()|sum(start=[]) -%} + +{# Figure out the widths of the columns. The last column is always 50 characters + # wide; the others default to 10, but stretch if there is wider text in the + # column. -#} +{% set fieldwidths = (([titlerow] + flatrows) | + fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} + +{{ tableheader(fieldwidths) }} +{{ tablerow(fieldwidths, titlerow, rowkeys) }} +{{ tableheader(fieldwidths) }} +{% for loc in rows_by_loc -%} + +{% if loc != None -%} +{{ tablespan(fieldwidths, "*" ~ loc ~ " parameters*") }} +{% endif -%} + +{% for row in rows_by_loc[loc] -%} +{{ tablerow(fieldwidths, row, rowkeys) }} +{% endfor -%} +{% endfor -%} + +{{ tableheader(fieldwidths) }} +{% endmacro %} + + + +{# + # Write a table header row, for the given column widths + #} +{% macro tableheader(widths) -%} +{% for arg in widths -%} +{{"="*arg}} {% endfor -%} +{% endmacro %} + + + +{# + # Write a normal table row. Each of 'widths' and 'keys' should be sequences + # of the same length; 'widths' defines the column widths, and 'keys' the + # attributes of 'row' to look up for values to put in the columns. + #} +{% macro tablerow(widths, row, keys) -%} +{% for key in keys -%} +{% set value=row[key] -%} +{% if not loop.last -%} + {# the first few columns need space after them -#} + {{ value }}{{" "*(1+widths[loop.index0]-value|length) -}} +{% else -%} + {# the last column needs wrapping and indenting (by the sum of the widths of + the preceding columns, plus the number of preceding columns (for the + separators)) -#} + {{ value | wrap(widths[loop.index0]) | + indent_block(widths[0:-1]|sum + loop.index0) -}} +{% endif -%} +{% endfor -%} +{% endmacro %} + + + + +{# + # write a tablespan row. This is a single value which spans the entire table. + #} +{% macro tablespan(widths, value) -%} +{{value}} +{# we write a trailing space to stop the separator being misinterpreted + # as a header line. -#} +{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} + + + + From 838af2a23ec623b2a7f2bd18ac0af51a097dd2b5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 17:21:56 +0000 Subject: [PATCH 043/111] Updates to swagger table generation A bunch of related fixes to the code for parsing the state and API yaml files: 1. Some of our objects are {key: {key: value}} - style nested key/value dictionaries. Handle this by refactoring get_json_schema_object_fields so that such objects are handled wherever they appear, rather than when they are just subproperties of a 'proper' object. 2. Fix multi-level inheritance (so an object can have an 'allOf' property which can successfully refer to an object which itself has an 'allOf' property). 3. $ref fields in event schemas weren't being expanded correctly 4. sort type tables breadth-first rather than depth-first so that the ordering in complex structures like the /sync response makes a bit more sense. --- templating/build.py | 4 + templating/matrix_templates/units.py | 200 +++++++++++++++++---------- 2 files changed, 130 insertions(+), 74 deletions(-) diff --git a/templating/build.py b/templating/build.py index a35d8a085..eea6ac7ac 100755 --- a/templating/build.py +++ b/templating/build.py @@ -42,6 +42,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, met from argparse import ArgumentParser, FileType import importlib import json +import logging import os import sys from textwrap import TextWrapper @@ -188,6 +189,9 @@ if __name__ == '__main__': ) args = parser.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + if not args.input: raise Exception("Missing [i]nput python module.") diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427c..5acbcd568 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -8,6 +8,7 @@ For the actual conversion of data -> RST (including templates), see the sections file instead. """ from batesian.units import Units +import logging import inspect import json import os @@ -27,6 +28,7 @@ TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" +logger = logging.getLogger(__name__) def resolve_references(path, schema): if isinstance(schema, dict): @@ -46,6 +48,32 @@ def resolve_references(path, schema): return schema +def inherit_parents(obj): + """ + Recurse through the 'allOf' declarations in the object + """ + logger.debug("inherit_parents %r" % obj) + parents = obj.get("allOf", []) + if not parents: + return obj + + result = {} + + # settings defined in the child take priority over the parents, so we + # iterate through the parents first, and then overwrite with the settings + # from the child. + for p in map(inherit_parents, parents) + [obj]: + for key in ('title', 'type', 'required'): + if p.get(key): + result[key] = p[key] + + for key in ('properties', 'additionalProperties', 'patternProperties'): + if p.get(key): + result.setdefault(key, {}).update(p[key]) + + return result + + def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -53,22 +81,44 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals raise Exception( "get_json_schema_object_fields: Object %s isn't an object." % obj ) + + obj = inherit_parents(obj) + + logger.debug("Processing object with title '%s'", obj.get("title")) + if enforce_title and not obj.get("title"): # Force a default titile of "NO_TITLE" to make it obvious in the # specification output which parts of the schema are missing a title obj["title"] = 'NO_TITLE' - required_keys = obj.get("required") - if not required_keys: - required_keys = [] + additionalProps = obj.get("additionalProperties") + if additionalProps: + # not "really" an object, just a KV store + logger.debug("%s is a pseudo-object", obj.get("title")) + + key_type = additionalProps.get("x-pattern", "string") + + value_type = additionalProps["type"] + if value_type == "object": + nested_objects = get_json_schema_object_fields( + additionalProps, + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + tables = [x for x in nested_objects if not x.get("no-table")] + else: + key_type = "string" + tables = [] - fields = { - "title": obj.get("title"), - "rows": [] - } - tables = [fields] + tables = [{ + "title": "{%s: %s}" % (key_type, value_type), + "no-table": True + }]+tables + + logger.debug("%s done: returning %s", obj.get("title"), tables) + return tables - parents = obj.get("allOf") props = obj.get("properties") if not props: props = obj.get("patternProperties") @@ -79,73 +129,60 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if pretty_key: props[pretty_key] = props[key_name] del props[key_name] - if not props and not parents: - # Sometimes you just want to specify that a thing is an object without - # doing all the keys. Allow people to do that if they set a 'title'. - if obj.get("title"): - parents = [{ - "$ref": obj.get("title") - }] - if not props and not parents: - raise Exception( - "Object %s has no properties or parents." % obj - ) - if not props: # parents only - if include_parents: - if obj["title"] == "NO_TITLE" and parents[0].get("title"): - obj["title"] = parents[0].get("title") - props = parents[0].get("properties") - if not props: + # Sometimes you just want to specify that a thing is an object without + # doing all the keys. Allow people to do that if they set a 'title'. + if not props and obj.get("title"): return [{ "title": obj["title"], - "parent": parents[0].get("$ref"), "no-table": True }] + if not props: + raise Exception( + "Object %s has no properties and no title" % obj + ) + + required_keys = set(obj.get("required", [])) + + fields = { + "title": obj.get("title"), + "rows": [] + } + + tables = [fields] + for key_name in sorted(props): + logger.debug("Processing property %s.%s", obj.get('title'), key_name) value_type = None required = key_name in required_keys desc = props[key_name].get("description", "") - - if props[key_name]["type"] == "object": - if props[key_name].get("additionalProperties"): - # not "really" an object, just a KV store - prop_val = props[key_name]["additionalProperties"]["type"] - if prop_val == "object": - nested_object = get_json_schema_object_fields( - props[key_name]["additionalProperties"], - enforce_title=True, - include_parents=include_parents, - ) - key = props[key_name]["additionalProperties"].get( - "x-pattern", "string" - ) - value_type = "{%s: %s}" % (key, nested_object[0]["title"]) - if not nested_object[0].get("no-table"): - tables += nested_object - else: - value_type = "{string: %s}" % prop_val - else: - nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True, - include_parents=include_parents, - ) - value_type = "{%s}" % nested_object[0]["title"] - - if not nested_object[0].get("no-table"): - tables += nested_object - elif props[key_name]["type"] == "array": + prop_type = props[key_name].get('type') + + if prop_type is None: + raise KeyError("Property '%s' of object '%s' missing 'type' field" + % (key_name, obj)) + logger.debug("%s is a %s", key_name, prop_type) + + if prop_type == "object": + nested_objects = get_json_schema_object_fields( + props[key_name], + enforce_title=True, + include_parents=include_parents, + ) + value_type = nested_objects[0]["title"] + + tables += [x for x in nested_objects if not x.get("no-table")] + elif prop_type == "array": # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": - nested_object = get_json_schema_object_fields( + nested_objects = get_json_schema_object_fields( props[key_name]["items"], enforce_title=True, include_parents=include_parents, ) - value_type = "[%s]" % nested_object[0]["title"] - tables += nested_object + value_type = "[%s]" % nested_objects[0]["title"] + tables += nested_objects else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): @@ -163,7 +200,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals " Must be '%s'." % array_enums[0] ) else: - value_type = props[key_name]["type"] + value_type = prop_type if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -188,15 +225,32 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) + logger.debug("Done property %s" % key_name) + + return tables + + +def get_tables_for_schema(path, schema, include_parents=False): + resolved_schema = resolve_references(path, schema) + tables = get_json_schema_object_fields(resolved_schema, + include_parents=include_parents, + ) + + # the result may contain duplicates, if objects are referred to more than + # once. Filter them out. + # + # Go through the tables backwards so that we end up with a breadth-first + # rather than depth-first ordering. titles = set() filtered = [] - for table in tables: + for table in reversed(tables): if table.get("title") in titles: continue titles.add(table.get("title")) filtered.append(table) + filtered.reverse() return filtered @@ -306,16 +360,14 @@ class MatrixUnits(Units): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields( - resolve_references(filepath, req_obj), - include_parents=True, - ) + req_tables = get_tables_for_schema( + filepath, req_obj, include_parents=True) if req_tables > 1: for table in req_tables[1:]: nested_key_name = [ s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) + s["type"] == ("%s" % (table["title"],)) ][0] for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"]) @@ -431,8 +483,7 @@ class MatrixUnits(Units): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields( - resolve_references(filepath, schema), + res_tables = get_tables_for_schema(filepath, schema, include_parents=True, ) for table in res_tables: @@ -571,8 +622,9 @@ class MatrixUnits(Units): for filename in os.listdir(path): if not filename.startswith("m."): continue - self.log("Reading %s" % os.path.join(path, filename)) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + self.log("Reading %s" % filepath) + with open(filepath, "r") as f: json_schema = json.loads(f.read()) schema = { "typeof": None, @@ -614,15 +666,15 @@ class MatrixUnits(Units): schema["desc"] = json_schema.get("description", "") # walk the object for field info - schema["content_fields"] = get_json_schema_object_fields( + schema["content_fields"] = get_tables_for_schema(filepath, Units.prop(json_schema, "properties/content") ) # This is horrible because we're special casing a key on m.room.member. # We need to do this because we want to document a non-content object. if schema["type"] == "m.room.member": - invite_room_state = get_json_schema_object_fields( - json_schema["properties"]["invite_room_state"]["items"] + invite_room_state = get_tables_for_schema(filepath, + json_schema["properties"]["invite_room_state"]["items"], ) schema["content_fields"].extend(invite_room_state) From 3f0262081c7a900cd1e5bcd7ca9a2991facb6ef9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 11 Nov 2015 19:22:38 +0000 Subject: [PATCH 044/111] Update sync API defn to reflect SPEC-254 changes State now corresponds to the start of the timeline, not the end. --- api/client-server/v2_alpha/sync.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index a2d5a2b89..9e921610c 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -119,7 +119,11 @@ paths: title: State type: object description: |- - The state updates for the room. + Updates to the state, between the time indicated by + the ``since`` parameter, and the start of the + ``timeline`` (or all state up to the start of the + ``timeline``, if ``since`` is not given, or + ``full_state`` is true). allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -193,8 +197,7 @@ paths: title: State type: object description: |- - The state updates for the room up to the point when - the user left. + The state updates for the room up to the start of the timeline. allOf: - $ref: "definitions/room_event_batch.json" timeline: @@ -260,7 +263,6 @@ paths: "state": { "events": [ "$66697273743031:example.com", - "$7365636s6r6432:example.com" ] }, "timeline": { From 57995a815a83ec7496283eae5511ebd89ddeb747 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:30:29 +0000 Subject: [PATCH 045/111] Fix a number of untruths in the documentation about /sync. Attempts to make the /sync documentation better represent fact as it currently stands - in particular document the structure of the returned events. --- api/client-server/v2_alpha/core-event-schema | 1 - .../v2_alpha/definitions/event.json | 53 +++++++++++++++++++ .../v2_alpha/definitions/event_batch.json | 4 +- .../v2_alpha/definitions/timeline_batch.json | 4 +- api/client-server/v2_alpha/sync.yaml | 16 +++--- specification/events.rst | 5 +- 6 files changed, 68 insertions(+), 15 deletions(-) delete mode 120000 api/client-server/v2_alpha/core-event-schema create mode 100644 api/client-server/v2_alpha/definitions/event.json diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema deleted file mode 120000 index b020e6da4..000000000 --- a/api/client-server/v2_alpha/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json new file mode 100644 index 000000000..6a01c6885 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "title": "Event", + "properties": { + "content": { + "type": "object", + "title": "EventContent", + "description": "The content of this event. The fields in this object will vary depending on the type of event." + }, + "origin_server_ts": { + "type": "integer", + "format": "int64", + "description": "Timestamp in milliseconds on originating homeserver when this event was sent." + }, + "sender": { + "type": "string", + "description": "The MXID of the user who sent this event." + }, + "state_key": { + "type": "string", + "description": "Optional. This key will only be present for state events. A unique key which defines the overwriting semantics for this piece of room state." + }, + "type": { + "type": "string", + "description": "The type of event." + }, + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 395aed13d..7f4894234 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,8 +5,8 @@ "type": "array", "description": "List of events", "items": { - "title": "Event", - "type": "object" + "type": "object", + "allOf": [{"$ref": "event.json" }] } } } diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index ddf8d3416..15ca1ead6 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -4,11 +4,11 @@ "properties": { "limited": { "type": "boolean", - "description": "Whether there are more events on the server" + "description": "True if the number of events returned was limited by the ``limit`` on the filter" }, "prev_batch": { "type": "string", - "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve earlier events" } } } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 9e921610c..06763a08c 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -96,8 +96,10 @@ paths: Updates to rooms. properties: joined: - title: Joined + title: Joined Rooms type: object + description: |- + The rooms that the user has joined. additionalProperties: title: Joined Room type: object @@ -114,7 +116,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -144,7 +146,7 @@ paths: allOf: - $ref: "definitions/event_batch.json" invited: - title: Invited + title: Invited Rooms type: object description: |- The rooms that the user has been invited to. @@ -171,11 +173,10 @@ paths: allOf: - $ref: "definitions/event_batch.json" archived: - title: Archived + title: Archived rooms type: object description: |- - The rooms that the user has left or been banned from. The - entries in the room_map will lack an ``ephemeral`` key. + The rooms that the user has left or been banned from. additionalProperties: title: Archived Room type: object @@ -192,7 +193,7 @@ paths: description: An event object. type: object allOf: - - $ref: "core-event-schema/event.json" + - $ref: "definitions/event.json" state: title: State type: object @@ -276,7 +277,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } diff --git a/specification/events.rst b/specification/events.rst index a1aece1c4..7bf080125 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -30,12 +30,13 @@ formatted for federation by: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, ``origin``, ``prev_state``. * Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has ellapsed since the event was sent. + milliseconds that has elapsed since the event was sent. * Adding a ``prev_content`` to the ``unsigned`` object if the event is a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``transaction_id`` if the event was sent by the client requesting it. +* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the + client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From b41d771c1586a06e387222030f923ed5fc95ecf0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:58:04 +0000 Subject: [PATCH 046/111] Fix typos in /sync example response --- api/client-server/v2_alpha/sync.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 06763a08c..2290b0700 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, + "unsigned": {"age": 124524, "txn_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" @@ -263,7 +263,7 @@ paths: }, "state": { "events": [ - "$66697273743031:example.com", + "$66697273743031:example.com" ] }, "timeline": { From 29bd4d45ee9155bd80e61ce50ab34745be1fd8a4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2015 23:40:41 +0000 Subject: [PATCH 047/111] Flatten the response to /sync Now that we don't expect there to be duplication betwen the 'timeline' and 'state' results, there's no point in having the state map. (That does mean the events themselves need event_id fields though). Also: - move the contents of the 'unsigned' dictionary into the events themselves - replace the state list with two layers of dictionary keyed on type and state_key - rename the children of the 'rooms' dict from "joined/invited/archived" to "join/invite/leave" to match the membership states --- .../v2_alpha/definitions/event.json | 45 +++--- .../definitions/room_event_batch.json | 12 -- .../v2_alpha/definitions/timeline_batch.json | 2 +- api/client-server/v2_alpha/sync.yaml | 134 +++++++----------- specification/events.rst | 17 +-- 5 files changed, 84 insertions(+), 126 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/room_event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c6885..3a15357aa 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,16 +2,34 @@ "type": "object", "title": "Event", "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, + "event_id": { + "type": "string", + "description": "Globally unique identifier for this event." + }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "prev_sender": { + "type": "string", + "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." + }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -24,30 +42,9 @@ "type": "string", "description": "The type of event." }, - "unsigned": { - "type": "object", - "title": "Unsigned", - "description": "Information about this event which was not sent by the originating homeserver", - "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "replaces_state": { - "type": "string", - "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." - } - } + "txn_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json deleted file mode 100644 index fcf148f36..000000000 --- a/api/client-server/v2_alpha/definitions/room_event_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "events": { - "type": "array", - "description": "List of event ids", - "items": { - "type": "string" - } - } - } -} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index 15ca1ead6..f27a5746c 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "allOf": [{"$ref":"definitions/event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b0700..c63f3432d 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -95,7 +95,7 @@ paths: description: |- Updates to rooms. properties: - joined: + join: title: Joined Rooms type: object description: |- @@ -104,19 +104,6 @@ paths: title: Joined Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object @@ -127,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -145,7 +132,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - invited: + invite: title: Invited Rooms type: object description: |- @@ -171,36 +158,23 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/event_batch.json" - archived: - title: Archived rooms + - $ref: "definitions/state_batch.json" + leave: + title: Left rooms type: object description: |- The rooms that the user has left or been banned from. additionalProperties: - title: Archived Room + title: Left Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "definitions/event.json" state: title: State type: object description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/state_batch.json" timeline: title: Timeline type: object @@ -230,46 +204,43 @@ paths: ] }, "rooms": { - "joined": { + "join": { "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "unsigned": { - "prev_content": {"membership": "invite"} - }, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, - "content": { - "body": "I am a fish", - "msgtype": "m.text" - }, - "origin_server_ts": 1417731086797 - } - }, "state": { - "events": [ - "$66697273743031:example.com" - ] + "m.room.member": { + "@alice:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795, + "event_id": "$66697273743031:example.com" + } + } }, "timeline": { "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" + { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "prev_content": {"membership": "invite"}, + "origin_server_ts": 1417731086795, + "event_id": "$7365636s6r6432:example.com": + }, + { + "sender": "@alice:example.com", + "type": "m.room.message", + "age": 124524, + "txn_id": "1234", + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797, + "event_id": "$74686972643033:example.com" + } ], "limited": true, "prev_batch": "t34-23535_0_0" @@ -277,6 +248,7 @@ paths: "ephemeral": { "events": [ { + "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -284,26 +256,30 @@ paths: } } }, - "invited": { + "invite": { "!696r7674:example.com": { "invite_state": { - "events": [ - { + "m.room.name": { + "": { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"} - }, - { + "content": {"name": "My Room Name"}, + "event_id": "$asdkgjrsfg2314375:example.com", + + } + }, + "m.room.member": { + "@bob:example.com": { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"} - } - ] + "content": {"membership": "invite"}, + "event_id": "$257kasjdg315324akhg:example.com", + } } } }, - "archived": {} + "leave": {} } } diff --git a/specification/events.rst b/specification/events.rst index 7bf080125..3d069be8a 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,22 +28,19 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``. -* Adding an ``age`` to the ``unsigned`` object which gives the time in + ``origin``, ``prev_state``, ``unsigned``. +* Adding an ``age`` to the event object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding a ``prev_content`` to the ``unsigned`` object if the event is - a ``state event`` which gives previous content of that state key. -* Adding a ``redacted_because`` to the ``unsigned`` object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the event object if the event + is a ``state event``, which give the previous content and previous sender of + that state key +* Adding a ``redacted_because`` to event object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``txn_id`` if the event was sent by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: -* Moving the folling keys from the ``unsigned`` object to the top level event - object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. -* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From e1b12a753ecd96227effef3612f70d9adf48c0a9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 00:17:22 +0000 Subject: [PATCH 048/111] Fix typos and missing file --- .../v2_alpha/definitions/state_batch.json | 12 ++++++++++++ api/client-server/v2_alpha/sync.yaml | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json new file mode 100644 index 000000000..45728f403 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/state_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "additionalProperties": { + "type": "object", + "x-pattern": "$EVENT_TYPE", + "additionalProperties": { + "type": "object", + "x-pattern": "$STATE_KEY", + "allOf": [{"$ref": "event.json" }] + } + } +} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c63f3432d..198f48337 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -227,7 +227,7 @@ paths: "content": {"membership": "join"}, "prev_content": {"membership": "invite"}, "origin_server_ts": 1417731086795, - "event_id": "$7365636s6r6432:example.com": + "event_id": "$7365636s6r6432:example.com" }, { "sender": "@alice:example.com", @@ -265,8 +265,7 @@ paths: "type": "m.room.name", "state_key": "", "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com", - + "event_id": "$asdkgjrsfg2314375:example.com" } }, "m.room.member": { @@ -275,7 +274,8 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com", + "event_id": "$257kasjdg315324akhg:example.com" + } } } } From 36af793f05992d0bef5ef325c303a753f16f24e0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:30:22 +0000 Subject: [PATCH 049/111] s/full object/full event/ --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 9cc10141f..127a35721 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -612,7 +612,7 @@ attempting to join. The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this -object is not a full object; notably it does not need to be hashed or signed by +object is not a full event; notably it does not need to be hashed or signed by the assisting resident. The required fields are: ==================== ======== ============ From 923f05e5541f354c2003acf77dbe4e2c3d6f90ec Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 15:34:11 +0000 Subject: [PATCH 050/111] More consistency around 'resident homeserver' --- specification/server_server_api.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 127a35721..4c315edcb 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -613,7 +613,7 @@ The resident server replies to this request with a JSON-encoded object having a single key called ``event``; within this is an object whose fields contain some of the information that the joining server will need. Despite its name, this object is not a full event; notably it does not need to be hashed or signed by -the assisting resident. The required fields are: +the resident homeserver. The required fields are: ==================== ======== ============ Key Type Description @@ -625,9 +625,9 @@ the assisting resident. The required fields are: ``content`` Object The event content ``depth`` Integer (this field must be present but is ignored; it may be 0) -``event_id`` String A new event ID specified by the assisting - resident -``origin`` String The name of the assisting resident homeserver +``event_id`` String A new event ID specified by the resident + homeserver +``origin`` String The name of the resident homeserver ``origin_server_ts`` Integer A timestamp added by the resident homeserver ``prev_events`` List An event-reference list containing the immediate predecessor events @@ -665,12 +665,12 @@ algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` fields. To complete the join handshake, the joining server must now submit this new -event to an assisting resident, by using the ``send_join`` endpoint. This is +event to an resident homeserver, by using the ``send_join`` endpoint. This is invoked using the room ID and the event ID of the new member event. -The assisting resident then accepts this event into the room's event graph, and -responds to the joining server with the full set of state for the newly-joined -room. This is returned as a two-element list, whose first element is the +The resident homeserver then accepts this event into the room's event graph, +and responds to the joining server with the full set of state for the newly- +joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element contains the following keys: ============== ===== ============ @@ -919,6 +919,6 @@ Querying directory information:: The list of join candidates is a list of server names that are likely to hold the given room; these are servers that the requesting server may wish to use as -assisting resident servers as part of the remote join handshake. This list may -or may not include the server answering the query. +resident servers as part of the remote join handshake. This list may or may not +include the server answering the query. From 122c082fcf0ef14cb93e09d1b8d2b7469866c786 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:29:43 +0000 Subject: [PATCH 051/111] Comment about origin servers of invites having subsequently left the room --- specification/server_server_api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4c315edcb..20026890d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -600,7 +600,9 @@ request the room ID and join candidates. This is covered in more detail on the directory server documentation, below. In the case of a new user joining a room as a result of a received invite, the joining user's homeserver could optimise this step away by picking the origin server of that invite message as -the join candidate. +the join candidate. However, the joining server should be aware that the origin +server of the invite might since have left the room, so should be prepared to +fall back on the regular join flow if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of From 6cbfba70113e302985c4f93af181141e7b0c84f1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:33:26 +0000 Subject: [PATCH 052/111] 'auth_events' is a List, not a String --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 20026890d..6c66b3d9c 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -621,7 +621,7 @@ the resident homeserver. The required fields are: Key Type Description ==================== ======== ============ ``type`` String The value ``m.room.member`` -``auth_events`` String An event-reference list containing the +``auth_events`` List An event-reference list containing the authorization events that would allow this member to join ``content`` Object The event content From 769c5285ab75243d247afd2b6bdb806224d0309d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:33:57 +0000 Subject: [PATCH 053/111] Add API for setting client config --- api/client-server/v2_alpha/client-config.yaml | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 api/client-server/v2_alpha/client-config.yaml diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml new file mode 100644 index 000000000..91be1686a --- /dev/null +++ b/api/client-server/v2_alpha/client-config.yaml @@ -0,0 +1,105 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server Client Config API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client. This config is only visible to the user + that set the config. The config will be synced to clients in the + top-level ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. + "/user/{userId}/rooms/{roomId}/config/{configType}": + put: + summary: Set some config for the user. + description: |- + Set some config for the client on a given room. This config is only + visible to the user that set the config. The config will be synced to + clients in the per-room ``private_user_data``. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: |- + The id of the user to set config for. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + required: true + description: |- + The id of the room to set config on. + x-example: "!726s6s6q:example.com" + - in: path + type: string + name: configType + required: true + description: |- + The type of the config to set. Custom types should be namespaced to + avoid clashes. + x-example: "org.example.custom.room.config" + - in: body + name: content + required: true + description: |- + The content of the config + schema: + type: object + example: |- + {"custom_config_key": "custom_config_value"} + responses: + 200: + description: + The config was successfully added. From 22b3159a3983a98762961ff5374deb3eff6c6d99 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:34:39 +0000 Subject: [PATCH 054/111] Add examples of v1 initialSync and v2 /sync returning the client config --- api/client-server/v1/sync.yaml | 24 ++++++++++++++++++++---- api/client-server/v2_alpha/sync.yaml | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a99..57a155837 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,6 +131,14 @@ paths: "type": "m.presence" } ], + "private_user_data": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ], "rooms": [ { "membership": "join", @@ -246,10 +254,18 @@ paths: } ], "visibility": "private", - "private_user_data": [{ - "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} - }] + "private_user_data": [ + { + "type": "m.tag", + "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] } ] } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 4c1a7601b..e180c5ff0 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -215,6 +215,16 @@ paths: } ] }, + "private_user_data": { + "events": [ + { + "type": "org.example.custom.config", + "content": { + "custom_config_key": "custom_config_value" + } + } + ] + }, "rooms": { "joined": { "!726s6s6q:example.com": { @@ -271,6 +281,12 @@ paths: { "type": "m.tags", "content": {"tags": {"work": {"order": 1}}} + }, + { + "type": "org.example.custom.room.config", + "content": { + "custom_config_key": "custom_config_value" + } } ] } From c3769ef75ca039ad2bfe2532a17ad979c631c71c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 16:49:32 +0000 Subject: [PATCH 055/111] Add a module for client config to the specification --- specification/modules/client_config.rst | 26 +++++++++++++++++++++++++ specification/targets.yaml | 1 + 2 files changed, 27 insertions(+) create mode 100644 specification/modules/client_config.rst diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst new file mode 100644 index 000000000..837f174d7 --- /dev/null +++ b/specification/modules/client_config.rst @@ -0,0 +1,26 @@ +Client Config +============= + +.. _module:client_config: + +Clients can store their config on their homeserver. This config will be synced +between different devices and can persist across installations on a particular +device. + +The config may be either global or scoped to a particular rooms. + +Events +------ + +The client recieves the config as a event in the ``private_user_data`` sections +of a v2 /sync. + +These events can also be received in a v1 /events response or in the +``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +events appearing in v1 /events will have a ``room_id`` with the room +the tags are for. + +Client Behaviour +---------------- + +{{v2_client_config_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 2c6a8f205..b6ac6a65f 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -26,6 +26,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/third_party_invites.rst - modules/search.rst - modules/tags.rst + - modules/client_config.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 83168813939ea1dc4c5a778781cb12e1f8eb9d87 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:50:54 +0000 Subject: [PATCH 056/111] txn_id field in events is called transaction_id --- api/client-server/v2_alpha/definitions/event.json | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 6a01c6885..5a8f52f6f 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -43,7 +43,7 @@ "type": "string", "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." }, - "txn_id": { + "transaction_id": { "type": "string", "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 2290b0700..d23929df3 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -253,7 +253,7 @@ paths: "$74686972643033:example.com": { "sender": "@alice:example.com", "type": "m.room.message", - "unsigned": {"age": 124524, "txn_id": "1234"}, + "unsigned": {"age": 124524, "transaction_id": "1234"}, "content": { "body": "I am a fish", "msgtype": "m.text" From 233e8486bc41e4ad87fc8e06a57bbcd4b8579974 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 16 Nov 2015 16:50:58 +0000 Subject: [PATCH 057/111] Wording fix - objects contain keys, not list elements directly --- specification/server_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6c66b3d9c..cc9426e2b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -673,7 +673,8 @@ invoked using the room ID and the event ID of the new member event. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the newly- joined room. This is returned as a two-element list, whose first element is the -integer 200, and whose second element contains the following keys: +integer 200, and whose second element is an object which contains the +following keys: ============== ===== ============ Key Type Description From 0db055b4ea22f4d29064728eaaaf2c37a2f9d926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 16:54:57 +0000 Subject: [PATCH 058/111] Fix another reference to 'txn_id' --- specification/events.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/events.rst b/specification/events.rst index 7bf080125..9d0be55f8 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -35,8 +35,8 @@ formatted for federation by: a ``state event`` which gives previous content of that state key. * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the ``unsigned`` object if the event was sent by the - client requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: From 299af673da3d4a698dc943aaabe3c2e961f2da49 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:04:19 +0000 Subject: [PATCH 059/111] Specify how ordering of tags is supposed to work --- api/client-server/v1/rooms.yaml | 2 +- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- api/client-server/v2_alpha/tags.yaml | 4 ++-- specification/modules/tags.rst | 5 +++++ 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 1cce76f30..bafcb98b6 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -314,7 +314,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } schema: diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b52908a99..8050ede39 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -248,7 +248,7 @@ paths: "visibility": "private", "private_user_data": [{ "type": "m.tag", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} }] } ] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index ec34c085c..7a597f9e4 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -291,7 +291,7 @@ paths: "events": [ { "type": "m.tags", - "content": {"tags": {"work": {"order": 1}}} + "content": {"tags": {"work": {"order": "1"}}} } ] } diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/v2_alpha/tags.yaml index 009e6000c..91945dca2 100644 --- a/api/client-server/v2_alpha/tags.yaml +++ b/api/client-server/v2_alpha/tags.yaml @@ -55,7 +55,7 @@ paths: application/json: |- { "tags": { - "work": {"order": 1}, + "work": {"order": "1"}, "pinned": {} } } @@ -97,7 +97,7 @@ paths: schema: type: object example: |- - {"order": 1} + {"order": "1"} responses: 200: description: diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 3d45975c4..15982a0af 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -21,6 +21,11 @@ the tags are for. Each tag has an associated JSON object with information about the tag, e.g how to order the rooms with a given tag. +Ordering information is given under the ``order`` key as a string. The string +are compared lexicographically by unicode codepoint to determine which should +displayed first. So a tag with an ``order`` key of ``"apples"`` would appear +before a tag with an ``order`` key of ``"oranges"``. + {{m_tag_event}} Client Behaviour From 48f35e15cba959e28aeb897e3339e5d6dfbfddc7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:08:59 +0000 Subject: [PATCH 060/111] describe how to order rooms that don't have an order in their tags --- specification/modules/tags.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 15982a0af..56d216e7c 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -23,10 +23,13 @@ to order the rooms with a given tag. Ordering information is given under the ``order`` key as a string. The string are compared lexicographically by unicode codepoint to determine which should -displayed first. So a tag with an ``order`` key of ``"apples"`` would appear -before a tag with an ``order`` key of ``"oranges"``. +displayed first. So a room with a tag with an ``order`` key of ``"apples"`` +would appear before a room with a tag with an ``order`` key of ``"oranges"``. +If a room has a tag without an ``order`` key then it should appear after the +rooms with that tag that have an ``order`` key. -{{m_tag_event}} + +{{m_tag_event} Client Behaviour ---------------- From e7fbe6f13b12797817eed767cd7306498dcddcec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:13:02 +0000 Subject: [PATCH 061/111] Limit the size of a tag --- specification/modules/tags.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 56d216e7c..4d88c7715 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -28,6 +28,7 @@ would appear before a room with a tag with an ``order`` key of ``"oranges"``. If a room has a tag without an ``order`` key then it should appear after the rooms with that tag that have an ``order`` key. +The name of a tag MUST not exceed 255 bytes. {{m_tag_event} From 25769493b11cd95ef73b5cfe936971a8bc7565ea Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:14:41 +0000 Subject: [PATCH 062/111] Fix template --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 4d88c7715..5fe6345f4 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,7 +30,7 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -{{m_tag_event} +{{m_tag_event}} Client Behaviour ---------------- From c77b22778fab74d7bae2ebc25b1d635d17e8afb3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 17:48:41 +0000 Subject: [PATCH 063/111] Add some documentation on names of tags --- specification/modules/tags.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 5fe6345f4..681d33d71 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -30,6 +30,16 @@ rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. +The name of a tag should be human readable. When displaying tags for a room a +client should display this human readable name. When adding a tag for a room +a client may offer a list to choose from that includes all the tags that the +user has previously set on any of their rooms. + +Two special names are listed in the specification: + +* ``m.favourite`` +* ``m.lowpriority`` + {{m_tag_event}} Client Behaviour From 03a0377c76861eb1c9af503b718b31508dcdd7dc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 16 Nov 2015 20:07:42 +0000 Subject: [PATCH 064/111] Plans for end-to-end in matrix --- drafts/markjh_end_to_end.rst | 142 +++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 drafts/markjh_end_to_end.rst diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst new file mode 100644 index 000000000..1699f79bd --- /dev/null +++ b/drafts/markjh_end_to_end.rst @@ -0,0 +1,142 @@ +Goals of Key-Distribution in Matrix +=================================== + +* No Central Authority: Users should not need to trust a central authority + when determining the authenticity of keys. + +* Easy to Add New Devices: It should be easy for a user to start using a + new device. + +* Possible to discover MITM: It should be possible for a user to determine if + they are being MITM. + +* Lost Devices: It should be possible for a user to recover if they lose all + their devices. + +* No Copying Keys: Keys should be per device and shouldn't leave the device + they were created on. + +A Possible Mechanism for Key Distribution +========================================= + +Basic API for setting up keys on a server: + +https://github.com/matrix-org/matrix-doc/pull/24 + +Client shouldn't trust the keys unless they have been verified, e.g by +comparing fingerprints. + +If a user adds a new device it should some yet to be specified protocol +communicate with an old device and obtain a cross-signature from the old +device for its public key. + +The new device can then present the cross-signed key to all the devices +that the user is in conversations with. Those devices should then include +the new device into those conversations. + +If the user cannot cross-sign the new key, e.g. because their old device +is lost or stolen. Then they will need to reauthenticate their conversations +out of band, e.g by comparing fingerprints. + + +Goals of End-to-end encryption in Matrix +======================================== + +* Access to Chat History: Users should be able to see the history of a + conversation on a new device. User should be able to control who can + see their chat history and how much of the chat history they can see. + +* Forward Secrecy of Discarded Chat History: Users should be able to discard + history from their device, once they have discarded the history it should be + impossible for an adversary to recover that history. + +* Forward Secrecy of Future Messages: Users should be able to recover from + disclosure of the chat history on their device. + +* Deniablity of Chat History: It should not be possible to prove to a third + party that a given user sent a message. + +* Authenticity of Chat History: It should be possible to prove amoungst + the members of a chat that a message sent by a user was authored by that + user. + + +Bonus Goals: + +* Traffic Analysis: It would be nice if the protocol was resilient to traffic + or metadata analysis. However it's not something we want to persue if it + harms the usability of the protocol. It might be cool if there was a + way for the user to could specify the trade off between performance and + resilience to traffic analysis that they wanted. + + +A Possible Design for Group Chat using Olm +========================================== + +Protecting the secrecy of history +--------------------------------- + +Each message sent by a client has a 32-bit counter. This counter increments +by one for each message sent by the client. This counter is used to advance a +ratchet. The ratchet is split into a vector four 256-bit values, +:math:`R_{n,j}` for :math:`j \in {0,1,2,3}`. The ratchet can be advanced as +follows: + +.. math:: + \begin{align} + R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ + R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_1\left(R_{(i-1),3}\right) + \end{align} + +Where :math:`H_1` and :math:`H_2` are different hash functions. For example +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and +:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. + +So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. +Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. +Every :math:`2^8` iterations :math:`R_{n,3}` is reseeded from :math:`R_{n,2}`. + +This scheme allows the ratchet to be advanced an arbitrary amount forwards +while needing only 1024 hash computations. + +This the value of the ratchet is hashed to generate the keys used to encrypt +each mesage. + +A client can decrypt chat history onwards from the earliest value of the +ratchet it is aware of. But cannot decrypt history from before that point +without reversing the hash function. + +This allows a client to share its ability to decrypt chat history with another +from a point in the conversation onwards by giving a copy of the ratchet at +that point in the conversation. + +A client can discard history by advancing a ratchet to beyond the last message +they want to discard and then forgetting all previous values of the ratchet. + +Proving and denying the authenticity of history +----------------------------------------------- + +Client sign the messages they send using a Ed25519 key generated per +conversation. That key, along with the ratchet key, is distributed +to other clients using 1:1 olm ratchets. Those 1:1 ratchets are started using +Triple Diffie-Hellman which provides authenticity of the messages to the +participants and deniability of the messages to third parties. Therefore +any keys shared over those keys inherit the same levels of deniability and +authenticity. + +Protecting the secrecy of future messages +----------------------------------------- + +A client would need to generate new keys if it wanted to prevent access to +messages beyond a given point in the conversation. It must generate new keys +whenever someone leaves the room. It should generate new keys periodically +anyway. + +The frequency of key generation in a large room may need to be restricted to +keep the frequency of messages broadcast over the individual 1:1 channels +low. From 8201eaa042875feb69389f14262356fd3cfa73ed Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 15:31:10 +0000 Subject: [PATCH 065/111] Swaggerify /rooms/:room_id/leave --- .../v1/{membership.yaml => inviting.yaml} | 51 +------------- api/client-server/v1/joining.yaml | 68 +++++++++++++++++++ api/client-server/v1/leaving.yaml | 57 ++++++++++++++++ specification/client_server_api.rst | 32 ++------- 4 files changed, 132 insertions(+), 76 deletions(-) rename api/client-server/v1/{membership.yaml => inviting.yaml} (61%) create mode 100644 api/client-server/v1/joining.yaml create mode 100644 api/client-server/v1/leaving.yaml diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/inviting.yaml similarity index 61% rename from api/client-server/v1/membership.yaml rename to api/client-server/v1/inviting.yaml index c089e154a..0a0287103 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/inviting.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - title: "Matrix Client-Server v1 Room Membership API" + title: "Matrix Client-Server v1 Room Joining API" version: "1.0.0" host: localhost:8008 schemes: @@ -18,55 +18,6 @@ securityDefinitions: name: access_token in: query paths: - "/rooms/{roomId}/join": - post: - summary: Start the requesting user participating in a particular room. - description: |- - This API starts a user participating in a particular room, if that user - is allowed to participate in that room. After this call, the client is - allowed to see all current state events in the room, and all subsequent - events associated with the room until the user leaves the room. - - After a user has joined a room, the room will appear as an entry in the - response of the |initialSync| API. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier or room alias to join. - required: true - x-example: "#monkeys:matrix.org" - responses: - 200: - description: |- - The room has been joined. - - The joined room ID must be returned in the ``room_id`` field. - examples: - application/json: |- - {"room_id": "!d41d8cd:matrix.org"} - schema: - type: object - 403: - description: |- - You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - - The room is invite-only and the user was not invited. - - The user has been banned from the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" - x-alias: - canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" - aliases: - - /join/{roomId} - # With an extra " " to disambiguate from the 3pid invite endpoint # The extra space makes it sort first for what I'm sure is a good reason. "/rooms/{roomId}/invite ": diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml new file mode 100644 index 000000000..20fdbd654 --- /dev/null +++ b/api/client-server/v1/joining.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Inviting API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml new file mode 100644 index 000000000..376b2060c --- /dev/null +++ b/api/client-server/v1/leaving.yaml @@ -0,0 +1,57 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Leaving API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/leave": + post: + summary: Stop the requesting user participating in a particular room. + description: |- + This API stops a user participating in a particular room. + + If the user was already in the room, they will no longer be able to see + new events in the room. If the room requires an invite to join, they + will need to be re-invited before they can re-join. + + If the user was invited to the room, but had not joined, this call + serves to reject the invite. + + The user will still be allowed to retrieve history from the room which + they were previously allowed to see. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to leave. + required: true + x-example: "!nkl290a:matrix.org" + responses: + 200: + description: |- + The room has been left. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 05b6ff3c2..ba5578bcb 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -870,42 +870,22 @@ following values: ``invite`` This room can only be joined if you were invited. -{{membership_http_api}} +{{inviting_http_api}} + +{{joining_http_api}} Leaving rooms ~~~~~~~~~~~~~ -.. TODO-spec - HS deleting rooms they are no longer a part of. Not implemented. - - This is actually Very Tricky. If all clients a HS is serving leave a room, - the HS will no longer get any new events for that room, because the servers - who get the events are determined on the *membership list*. There should - probably be a way for a HS to lurk on a room even if there are 0 of their - members in the room. - - Grace period before deletion? - - Under what conditions should a room NOT be purged? - - A user can leave a room to stop receiving events for that room. A user must have been invited to or have joined the room before they are eligible to leave the room. Leaving a room to which the user has been invited rejects the invite. +Once a user leaves a room, it will no longer appear on the |initialSync|_ API. Whether or not they actually joined the room, if the room is an "invite-only" room they will need to be re-invited before they can re-join -the room. To leave a room, a request should be made to -|/rooms//leave|_ with:: - - {} - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "leave" - } +the room. -See the `Room events`_ section for more information on ``m.room.member``. Once a -user has left a room, that room will no longer appear on the |initialSync|_ API. -If all members in a room leave, that room becomes eligible for deletion. +{{leaving_http_api}} Banning users in a room ~~~~~~~~~~~~~~~~~~~~~~~ From 6763317e64645945224b7c68ac385ed055fe80c3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 17 Nov 2015 10:33:46 -0500 Subject: [PATCH 066/111] Specify /rooms/:room_id/forget --- api/client-server/v1/leaving.yaml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml index 376b2060c..e81d812b5 100644 --- a/api/client-server/v1/leaving.yaml +++ b/api/client-server/v1/leaving.yaml @@ -55,3 +55,38 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" + "/rooms/{roomId}/forget": + post: + summary: Stop the requesting user remembering about a particular room. + description: |- + This API stops a user remembering about a particular room. + + In general, history is a first class citizen in Matrix. After this API + is called, however, a user will no longer be able to retrieve history + for this room. If all users on a homeserver forget a room, the room is + eligible for deletion from that homeserver. + + If the user is currently joined to the room, they will implicitly leave + the room as part of this API call. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to forget. + required: true + x-example: "!au1ba7o:matrix.org" + responses: + 200: + description: |- + The room has been forgotten. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" From 6653362f315d1379d1dda248dfd299c8c3c8530d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 15:15:21 +0000 Subject: [PATCH 067/111] Unflatten 'unsigned' It turns out that flattening 'unsigned' comes with too many downsides. Let's stick with the status quo. --- .../v2_alpha/definitions/event.json | 45 ++++++++++--------- specification/events.rst | 19 ++++---- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json index 3a15357aa..5a8f52f6f 100644 --- a/api/client-server/v2_alpha/definitions/event.json +++ b/api/client-server/v2_alpha/definitions/event.json @@ -2,34 +2,16 @@ "type": "object", "title": "Event", "properties": { - "age": { - "type": "integer", - "format": "int64", - "description": "Time in milliseconds since the event was sent." - }, "content": { "type": "object", "title": "EventContent", "description": "The content of this event. The fields in this object will vary depending on the type of event." }, - "event_id": { - "type": "string", - "description": "Globally unique identifier for this event." - }, "origin_server_ts": { "type": "integer", "format": "int64", "description": "Timestamp in milliseconds on originating homeserver when this event was sent." }, - "prev_content": { - "title": "EventContent", - "type": "object", - "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." - }, - "prev_sender": { - "type": "string", - "description": "Optional. The ``sender`` of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there was no previous event for this state, this key will be missing." - }, "sender": { "type": "string", "description": "The MXID of the user who sent this event." @@ -42,9 +24,30 @@ "type": "string", "description": "The type of event." }, - "txn_id": { - "type": "string", - "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "transaction_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } } } } diff --git a/specification/events.rst b/specification/events.rst index eb48ca17d..5a0031155 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -28,20 +28,23 @@ formatted for federation by: * Removing the following keys: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, - ``origin``, ``prev_state``, ``unsigned``. -* Adding an ``age`` to the event object which gives the time in + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in milliseconds that has elapsed since the event was sent. -* Adding ``prev_content`` and ``prev_sender`` to the event object if the event - is a ``state event``, which give the previous content and previous sender of - that state key -* Adding a ``redacted_because`` to event object if the event was +* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the + event is a ``state event``, which give the previous content and previous + sender of that state key +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``txn_id`` to the event object if the event was sent by the client - requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. * Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From fcbb985073e82368894a41285dab86feb53daa21 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:27:26 +0000 Subject: [PATCH 068/111] s/private_user_data/account_data/ --- api/client-server/v1/rooms.yaml | 4 ++-- api/client-server/v1/sync.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 4 ++-- specification/modules/tags.rst | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index bafcb98b6..ff5587a90 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -312,7 +312,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -375,7 +375,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to this room. diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 8050ede39..973ccc6e3 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -246,7 +246,7 @@ paths: } ], "visibility": "private", - "private_user_data": [{ + "account_data": [{ "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} }] @@ -336,7 +336,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - private_user_data: + account_data: type: array description: |- The private data that this user has attached to diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7a597f9e4..28358f558 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -139,7 +139,7 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - private_user_data: + account_data: title: Private User Data type: object description: |- @@ -287,7 +287,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "m.tags", diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 681d33d71..bb467bc05 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -11,10 +11,10 @@ Events ------ The tags on a room are received as single ``m.tag`` event in the -``private_user_data`` section of a room in a v2 /sync. +``account_data`` section of a room in a v2 /sync. The ``m.tag`` can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From 40f7eab73f1a3055c3c4874b8d122f6b9c57e167 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Nov 2015 15:44:18 +0000 Subject: [PATCH 069/111] s/private_user_data/account_data/ --- api/client-server/v1/sync.yaml | 2 +- api/client-server/v2_alpha/client-config.yaml | 4 ++-- api/client-server/v2_alpha/sync.yaml | 2 +- specification/modules/client_config.rst | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b8608171c..58cd55443 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -131,7 +131,7 @@ paths: "type": "m.presence" } ], - "private_user_data": [ + "account_data": [ { "type": "org.example.custom.config", "content": { diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/client-config.yaml index 91be1686a..76b6219ee 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/client-config.yaml @@ -24,7 +24,7 @@ paths: description: |- Set some config for the client. This config is only visible to the user that set the config. The config will be synced to clients in the - top-level ``private_user_data``. + top-level ``account_data``. security: - accessToken: [] parameters: @@ -63,7 +63,7 @@ paths: description: |- Set some config for the client on a given room. This config is only visible to the user that set the config. The config will be synced to - clients in the per-room ``private_user_data``. + clients in the per-room ``account_data``. security: - accessToken: [] parameters: diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index e2016d68d..f5c99508e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -233,7 +233,7 @@ paths: } ] }, - "private_user_data": { + "account_data": { "events": [ { "type": "org.example.custom.config", diff --git a/specification/modules/client_config.rst b/specification/modules/client_config.rst index 837f174d7..581a58281 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/client_config.rst @@ -12,11 +12,11 @@ The config may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``private_user_data`` sections +The client recieves the config as a event in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the -``private_user_data`` section of a room in v1 /initialSync. ``m.tag`` +``account_data`` section of a room in v1 /initialSync. ``m.tag`` events appearing in v1 /events will have a ``room_id`` with the room the tags are for. From d7d59d78e1dca06407dea20ad875f64569844ad3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Nov 2015 16:17:29 +0000 Subject: [PATCH 070/111] /sync: Put state dict back to being a list Turning the state into a dict-of-dicts caused more pain than it solved. Put it back to a list. --- .../v2_alpha/definitions/state_batch.json | 12 ------- api/client-server/v2_alpha/sync.yaml | 31 ++++++++----------- 2 files changed, 13 insertions(+), 30 deletions(-) delete mode 100644 api/client-server/v2_alpha/definitions/state_batch.json diff --git a/api/client-server/v2_alpha/definitions/state_batch.json b/api/client-server/v2_alpha/definitions/state_batch.json deleted file mode 100644 index 45728f403..000000000 --- a/api/client-server/v2_alpha/definitions/state_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "additionalProperties": { - "type": "object", - "x-pattern": "$EVENT_TYPE", - "additionalProperties": { - "type": "object", - "x-pattern": "$STATE_KEY", - "allOf": [{"$ref": "event.json" }] - } - } -} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 198f48337..0c9d6186f 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -114,7 +114,7 @@ paths: ``timeline``, if ``since`` is not given, or ``full_state`` is true). allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -158,7 +158,7 @@ paths: delta against the archived ``state`` not the ``invite_state``. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" leave: title: Left rooms type: object @@ -174,7 +174,7 @@ paths: description: |- The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/state_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -207,8 +207,8 @@ paths: "join": { "!726s6s6q:example.com": { "state": { - "m.room.member": { - "@alice:example.com": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@alice:example.com", @@ -216,7 +216,7 @@ paths: "origin_server_ts": 1417731086795, "event_id": "$66697273743031:example.com" } - } + ] }, "timeline": { "events": [ @@ -248,7 +248,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -259,24 +258,20 @@ paths: "invite": { "!696r7674:example.com": { "invite_state": { - "m.room.name": { - "": { + "events": [ + { "sender": "@alice:example.com", "type": "m.room.name", "state_key": "", - "content": {"name": "My Room Name"}, - "event_id": "$asdkgjrsfg2314375:example.com" - } - }, - "m.room.member": { - "@bob:example.com": { + "content": {"name": "My Room Name"} + }, + { "sender": "@alice:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"}, - "event_id": "$257kasjdg315324akhg:example.com" + "content": {"membership": "invite"} } - } + ] } } }, From 5aeaa42a5039bd0435f66be0a9787cb3efc24985 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:41:25 -0500 Subject: [PATCH 071/111] Changelog for 0.3.0 --- CHANGELOG.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e3198ef9..4916c771f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,35 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.3.0 (2015-11-18) +============================================ + +Many sections have been filled in, or added to reflect long-standing status +quos. Several API endpoints have also been converted to Swagger. Several +presentation improvements (e.g. syntax highlighting) have also been added. + +Additions: + - Extensive narrative on instant messaging + - Documentation of event sending + - Documentation of state event retrieval + - Redaction algorithm + - Algorithm for calculating display names for rooms and users + - New endpoint: ``/publicRooms`` + - New event: ``m.room.avatar`` + - invite_room_state on ``m.room.member`` events + - New section: Third party invitations + - Ability to reject invitations + - New API: Search + - Guest access to rooms + - Descriptions of server-server API endpoints for joining rooms + - Sample expected crypto output + - Several security warnings + + Modifications: + - Fixed errors in parameters of /login + - Significant clarification and improvements to description of pagination + - Changes to v2_alpha/sync API (Note: this API is still not stable) + Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -38,4 +67,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file +- Restructure the format of the Client-Server HTTP APIs. From 1a1a7d87dcd1f8ada30592cae7b4da86b8b2468f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 18 Nov 2015 18:53:20 -0500 Subject: [PATCH 072/111] Revert "Changelog for 0.3.0" This reverts commit 5aeaa42a5039bd0435f66be0a9787cb3efc24985. --- CHANGELOG.rst | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4916c771f..6e3198ef9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,35 +6,6 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. -Specification changes in v0.3.0 (2015-11-18) -============================================ - -Many sections have been filled in, or added to reflect long-standing status -quos. Several API endpoints have also been converted to Swagger. Several -presentation improvements (e.g. syntax highlighting) have also been added. - -Additions: - - Extensive narrative on instant messaging - - Documentation of event sending - - Documentation of state event retrieval - - Redaction algorithm - - Algorithm for calculating display names for rooms and users - - New endpoint: ``/publicRooms`` - - New event: ``m.room.avatar`` - - invite_room_state on ``m.room.member`` events - - New section: Third party invitations - - Ability to reject invitations - - New API: Search - - Guest access to rooms - - Descriptions of server-server API endpoints for joining rooms - - Sample expected crypto output - - Several security warnings - - Modifications: - - Fixed errors in parameters of /login - - Significant clarification and improvements to description of pagination - - Changes to v2_alpha/sync API (Note: this API is still not stable) - Specification changes in v0.2.0 (2015-10-02) ============================================ @@ -67,4 +38,4 @@ Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. - Restructure the format of Event information. Add more information. -- Restructure the format of the Client-Server HTTP APIs. +- Restructure the format of the Client-Server HTTP APIs. \ No newline at end of file From 05c00926641a7a1156b2f239efc153b3aa39e9eb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:36:27 +0000 Subject: [PATCH 073/111] s/config/account_data/ --- .../{client-config.yaml => account-data.yaml} | 44 +++++++++---------- .../{client_config.rst => account_data.rst} | 4 +- specification/targets.yaml | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) rename api/client-server/v2_alpha/{client-config.yaml => account-data.yaml} (60%) rename specification/modules/{client_config.rst => account_data.rst} (91%) diff --git a/api/client-server/v2_alpha/client-config.yaml b/api/client-server/v2_alpha/account-data.yaml similarity index 60% rename from api/client-server/v2_alpha/client-config.yaml rename to api/client-server/v2_alpha/account-data.yaml index 76b6219ee..98134d6fe 100644 --- a/api/client-server/v2_alpha/client-config.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -18,12 +18,12 @@ securityDefinitions: name: access_token in: query paths: - "/user/{userId}/config/{configType}": + "/user/{userId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client. This config is only visible to the user - that set the config. The config will be synced to clients in the + Set some account_data for the client. This config is only visible to the user + that set the account_data. The config will be synced to clients in the top-level ``account_data``. security: - accessToken: [] @@ -33,36 +33,36 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to + The type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_config_value"} responses: 200: description: - The config was successfully added. - "/user/{userId}/rooms/{roomId}/config/{configType}": + The account_data was successfully added. + "/user/{userId}/rooms/{roomId}/account_data/{type}": put: - summary: Set some config for the user. + summary: Set some account_data for the user. description: |- - Set some config for the client on a given room. This config is only - visible to the user that set the config. The config will be synced to + Set some account_data for the client on a given room. This config is only + visible to the user that set the account_data. The config will be synced to clients in the per-room ``account_data``. security: - accessToken: [] @@ -72,7 +72,7 @@ paths: name: userId required: true description: |- - The id of the user to set config for. The access token must be + The id of the user to set account_data for. The access token must be authorized to make requests for this user id. x-example: "@alice:example.com" - in: path @@ -80,26 +80,26 @@ paths: name: roomId required: true description: |- - The id of the room to set config on. + The id of the room to set account_data on. x-example: "!726s6s6q:example.com" - in: path type: string - name: configType + name: type required: true description: |- - The type of the config to set. Custom types should be namespaced to - avoid clashes. + The type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body name: content required: true description: |- - The content of the config + The content of the account_data schema: type: object example: |- - {"custom_config_key": "custom_config_value"} + {"custom_account_data_key": "custom_account_data_value"} responses: 200: description: - The config was successfully added. + The account_data was successfully added. diff --git a/specification/modules/client_config.rst b/specification/modules/account_data.rst similarity index 91% rename from specification/modules/client_config.rst rename to specification/modules/account_data.rst index 581a58281..fc0e2e687 100644 --- a/specification/modules/client_config.rst +++ b/specification/modules/account_data.rst @@ -1,7 +1,7 @@ Client Config ============= -.. _module:client_config: +.. _module:account_data: Clients can store their config on their homeserver. This config will be synced between different devices and can persist across installations on a particular @@ -23,4 +23,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_client_config_http_api}} +{{v2_account_data_http_api}} diff --git a/specification/targets.yaml b/specification/targets.yaml index 418515c3e..afb7f8d91 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -27,7 +27,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/search.rst - modules/guest_access.rst - modules/tags.rst - - modules/client_config.rst + - modules/account_data.rst title_styles: ["=", "-", "~", "+", "^", "`"] From 032ee75537fd8d67db0e7e930ada4cea6d428faf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 19 Nov 2015 10:42:22 +0000 Subject: [PATCH 074/111] Update specification wording to match s/config/account_data/ --- specification/modules/account_data.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index fc0e2e687..784bc874f 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -3,16 +3,17 @@ Client Config .. _module:account_data: -Clients can store their config on their homeserver. This config will be synced -between different devices and can persist across installations on a particular -device. +Clients can store custom config data for their account on their homeserver. +This account data will be synced between different devices and can persist +across installations on a particular device. Users may only view the account +data for their own account -The config may be either global or scoped to a particular rooms. +The account_data may be either global or scoped to a particular rooms. Events ------ -The client recieves the config as a event in the ``account_data`` sections +The client recieves the account data as events in the ``account_data`` sections of a v2 /sync. These events can also be received in a v1 /events response or in the From 9ad64b02d183e72b6548d7a36f9b96949f624e70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 15:41:15 -0500 Subject: [PATCH 075/111] speculator: guard against concurrent git commands --- scripts/speculator/main.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 97e67c8c2..4472164c0 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -136,17 +137,26 @@ func writeError(w http.ResponseWriter, code int, err error) { } type server struct { + mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL matrixDocCloneURL string } +func (s *server) updateBase() error { + s.mu.Lock() + defer s.mu.Unlock() + return gitFetch(s.matrixDocCloneURL) +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = s.updateBase() if err != nil { return } + s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) + s.mu.Unlock() if err != nil { return } @@ -164,7 +174,10 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - if err := cmd.Run(); err != nil { + s.mu.Lock() + err := cmd.Run() + s.mu.Unlock() + if err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil @@ -174,7 +187,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := gitFetch(s.matrixDocCloneURL); err != nil { + if err := s.updateBase(); err != nil { writeError(w, 500, err) return } @@ -383,7 +396,7 @@ func main() { if err != nil { log.Fatal(err) } - s := server{masterCloneDir} + s := server{matrixDocCloneURL: masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) From 757b0bcd12269b0a9905efee7520e61abc81ec7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:08:37 -0500 Subject: [PATCH 076/111] Try to build continuserv and speculator --- jenkins.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins.sh b/jenkins.sh index 0b217e58a..b2aa84891 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -7,3 +7,8 @@ set -ex (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) + +if which go >/dev/null 2>/dev/null; then + (cd scripts/continuserv && go build) + (cd scripts/speculator && go build) +fi From dd53847211c57553180efca9d843c1c8741d6de3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:11:19 -0500 Subject: [PATCH 077/111] Include command stderr in error text --- scripts/speculator/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4472164c0..00a59eeaa 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -70,13 +70,15 @@ const ( func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, directory) + if err := os.MkdirAll(directory, 0755); err != nil { + return "", fmt.Errorf("error making directory %s: %v", directory, err) + } + args := []string{"clone", url, directory} if shared { - cmd.Args = append(cmd.Args, "--shared") + args = append(args, "--shared") } - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("error cloning repo: %v", err) + if err := runGitCommand(directory, args); err != nil { + return "", err } return directory, nil } @@ -92,8 +94,10 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path + var b bytes.Buffer + cmd.Stderr = &b if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String()) } return nil } From 8872e17f9355a47f9b7a2822cfd1303d8e56e2bc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:14:22 -0500 Subject: [PATCH 078/111] Fall back to last known HEAD sha if fetch fails --- scripts/speculator/main.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 00a59eeaa..380cd2bc9 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -87,10 +87,6 @@ func gitCheckout(path, sha string) error { return runGitCommand(path, []string{"checkout", sha}) } -func gitFetch(path string) error { - return runGitCommand(path, []string{"fetch"}) -} - func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path @@ -148,7 +144,7 @@ type server struct { func (s *server) updateBase() error { s.mu.Lock() defer s.mu.Unlock() - return gitFetch(s.matrixDocCloneURL) + return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } // generateAt generates spec from repo at sha. @@ -191,16 +187,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - if err := s.updateBase(); err != nil { + if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return + } else { + sha = headSha } - originHead, err := s.getSHAOf("origin/master") - if err != nil { - writeError(w, 500, err) - return - } - sha = originHead } else { pr, err := lookupPullRequest(*req.URL, "/spec") if err != nil { @@ -237,6 +229,25 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { specCache.Add(sha, b) } +// lookupHeadSHA looks up what origin/master's HEAD SHA is. +// It attempts to `git fetch` before doing so. +// If this fails, it may still return a stale sha, but will also return an error. +func (s *server) lookupHeadSHA() (sha string, retErr error) { + retErr = s.updateBase() + if retErr != nil { + log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr) + } + originHead, err := s.getSHAOf("origin/master") + if err != nil { + retErr = err + } + sha = originHead + if retErr != nil && originHead != "" { + log.Printf("Successfully fell back to possibly stale sha: %s", sha) + } + return +} + func checkAuth(pr *PullRequest) error { if !pr.User.IsTrusted() { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login) From 6f1d00097be403998e5490845835ecb23d2c741c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:15:13 -0500 Subject: [PATCH 079/111] Only bother trying to fetch if we need to --- scripts/speculator/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 380cd2bc9..4972210eb 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -147,12 +147,20 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } +func (s *server) knowsAbout(sha string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = s.updateBase() - if err != nil { - return + if !s.knowsAbout(sha) { + err = s.updateBase() + if err != nil { + return + } } s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) From 858674477111bfc80adb299ad73715a98099c296 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 16:41:58 -0500 Subject: [PATCH 080/111] Add anchors to spec This is currently done by a script on the prod serving machine. We might as well keep the matrix.org spec and dev spec as similar as possible. --- scripts/gendoc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed36a5a78..28a115284 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -238,6 +238,18 @@ def rst2html(i, o): ) +def addAnchors(path): + with open(path, "r") as f: + lines = f.readlines() + + replacement = replacement = r'

\n\1' + with open(path, "w") as f: + for line in lines: + line = re.sub(r'()', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write(line + "\n") + + def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: @@ -387,6 +399,7 @@ def main(target_name, keep_intermediates): shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") + addAnchors("gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() From e0410330484257e6d0c8da481d8d7e31a2b05053 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:17:50 -0500 Subject: [PATCH 081/111] Rename file --- scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{matrix-org-gendoc.sh => add-matrix-org-stylings.sh} (100%) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/add-matrix-org-stylings.sh similarity index 100% rename from scripts/matrix-org-gendoc.sh rename to scripts/add-matrix-org-stylings.sh From ca3a9e3562492964e314d7f9173d320281fbddfd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 17:18:33 -0500 Subject: [PATCH 082/111] exec gendoc outside of script --- scripts/add-matrix-org-stylings.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 5716786d2..b02cb306f 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -35,8 +35,6 @@ if [ ! -f $FOOTER ]; then exit 1 fi -python gendoc.py - perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header #' gen/specification.html gen/howtos.html From da93317a7874096c2c13974542e2f2272e5d2a60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:13:40 -0500 Subject: [PATCH 083/111] Take dir not files as args --- scripts/add-matrix-org-stylings.sh | 43 +++++++++--------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index b02cb306f..83888346c 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -1,39 +1,20 @@ -#! /bin/bash +#!/bin/bash -eu -if [ -z "$1" ]; then - echo "Expected /includes/head.html file as 1st arg." - exit 1 -fi - -if [ -z "$2" ]; then - echo "Expected /includes/nav.html file as 2nd arg." - exit 1 +if [[ $# != 1 || ! -d $1 ]]; then + echo >&2 "Usage: $0 include_dir" + exit 1 fi -if [ -z "$3" ]; then - echo "Expected /includes/footer.html file as 3rd arg." - exit 1 -fi - - -HEADER=$1 -NAV_BAR=$2 -FOOTER=$3 +HEADER="$1/head.html" +NAV_BAR="$1/nav.html" +FOOTER="$1/footer.html" -if [ ! -f $HEADER ]; then - echo $HEADER " does not exist" +for f in "${HEADER}"/{head,nav,footer}.html; do + if [[ ! -e "${f}" ]]; then + echo >&2 "Need ${f} to exist" exit 1 -fi - -if [ ! -f $NAV_BAR ]; then - echo $NAV_BAR " does not exist" - exit 1 -fi - -if [ ! -f $FOOTER ]; then - echo $FOOTER " does not exist" - exit 1 -fi + fi +done perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header From 4ac85997f52303b5fa75241f9bc50cdc227af3fa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 19 Nov 2015 18:16:02 -0500 Subject: [PATCH 084/111] Fix check --- scripts/add-matrix-org-stylings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index 83888346c..ef4e1014a 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -9,7 +9,7 @@ HEADER="$1/head.html" NAV_BAR="$1/nav.html" FOOTER="$1/footer.html" -for f in "${HEADER}"/{head,nav,footer}.html; do +for f in "$1"/{head,nav,footer}.html; do if [[ ! -e "${f}" ]]; then echo >&2 "Need ${f} to exist" exit 1 From 181d3f976d4f236c34ba0e86f58f729690d82e34 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2015 13:01:20 +0000 Subject: [PATCH 085/111] Initial proposal for websockets support. --- drafts/websockets.rst | 285 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 drafts/websockets.rst diff --git a/drafts/websockets.rst b/drafts/websockets.rst new file mode 100644 index 000000000..bd0ff081d --- /dev/null +++ b/drafts/websockets.rst @@ -0,0 +1,285 @@ +WebSockets API +============== + +Introduction +------------ +This document is a proposal for a WebSockets-based client-server API. It is not +intended to replace the REST API, but rather to complement it and provide an +alternative interface for certain operations. + +The primary goal is to offer a more efficient interface than the REST API: by +using a bidirectional protocol such as WebSockets we can avoid the overheads +involved in long-polling (SSL negotiation, HTTP headers, etc). In doing so we +will reduce the latency between server and client by allowing the server to +send events as soon as they arrive, rather than having to wait for a poll from +the client. + +Handshake +--------- +1. Instead of calling ``/sync``, the client makes a websocket request to + ``/_matrix/client/rN/stream``, passing the query parameters ``access_token`` + and ``since``, and optionally ``filter`` - all of which have the same + meaning as for ``/sync``. + + * The client sets the ``Sec-WebSocket-Protocol`` to ``m.json``. (Servers may + offer alternative encodings; at present only the JSON encoding is + specified but in future we will specify alternative encodings.) + +#. The server returns the websocket handshake; the socket is then connected. + +If the server does not return a valid websocket handshake, this indicates that +the server or an intermediate proxy does not support WebSockets. In this case, +the client should fall back to polling the ``/sync`` REST endpoint. + +Example +~~~~~~~ + +Client request: + +.. code:: http + + GET /_matrix/client/v2_alpha/stream?access_token=123456&since=s72594_4483_1934 HTTP/1.1 + Host: matrix.org + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: m.json + Sec-WebSocket-Version: 13 + Origin: https://matrix.org + +Server response: + +.. code:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: m.json + + +Update Notifications +-------------------- +Once the socket is connected, the server begins streaming updates over the +websocket. The server sends Update notifications about new messages or state +changes. To make it easy for clients to parse, Update notifications have the +same structure as the response to ``/sync``: an object with the following +members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +next_batch string The batch token to supply in the ``since`` param of + the next /sync request. This is not required for + streaming of events over the WebSocket, but is + provided so that clients can reconnect if the + socket is disconnected. +presence Presence The updates to the presence status of other users. +rooms Rooms Updates to rooms. +============= ========== =================================================== + +Example +~~~~~~~ +Message from the server: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [] + }, + "rooms": { + "join": {}, + "invite": {}, + "leave": {} + } + } + + +Client-initiated operations +--------------------------- + +The client can perform certain operations by sending a websocket message to +the server. Such a "Request" message should be a JSON-encoded object with +the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string A unique identifier for this request +method string Specifies the name of the operation to be + performed; see below for available operations +param object The parameters for the requested operation. +============= ========== =================================================== + +The server responds to a client Request with a Response message. This is a +JSON-encoded object with the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string The same as the value in the corresponding Request + object. The presence of the ``id`` field + distinguishes a Response message from an Update + notification. +result object On success, the results of the request. +error object On error, an object giving the resons for the + error. This has the same structure as the "standard + error response" for the Matrix API: an object with + the fields ``errcode`` and ``error``. +============= ========== =================================================== + +Request methods +~~~~~~~~~~~~~~~ +It is not intended that all operations which are available via the REST API +will be available via the WebSockets API, but a few simple, common operations +will be exposed. The initial operations will be as follows. + +``ping`` +^^^^^^^^ +This is a no-op which clients may use to keep their connection alive. + +The request ``params`` and the response ``result`` should be empty. + +``send`` +^^^^^^^^ +Send a message event to a room. The parameters are as follows: + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to send the event to +event_type string **Required.** The type of event to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + +The ``id`` from the Request message is used as the transaction ID by the +server. + +``state`` +^^^^^^^^^ +Update the state on a room. + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to set the state in +event_type string **Required.** The type of event to send. +state_key string **Required.** The state_key for the state to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + + +Example +~~~~~~~ +Client request: + +.. code:: json + + { + "id": "12345", + "method": "send", + "params": { + "room_id": "!d41d8cd:matrix.org", + "event_type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "hello" + } + } + } + +Server response: + +.. code:: json + + { + "id": "12345", + "result": { + "event_id": "$66697273743031:matrix.org" + } + } + +Alternative server response, in case of error: + +.. code:: json + + { + "id": "12345", + "error": { + "errcode": "M_MISSING_PARAM", + "error": "Missing parameter: event_type" + } + } + + +Rationale +--------- +Alternatives to WebSockets include HTTP/2, CoAP, and simply rolling our own +protocol over raw TCP sockets. However, the need to implement browser-based +clients essentially reduces our choice to WebSockets. HTTP/2 streams will +probably provide an interesting alternative in the future, but current browsers +do not appear to give javascript applications low-level access to the protocol. + +Concerning the continued use of the JSON encoding: we prefer to focus on the +transition to WebSockets initially. Replacing JSON with a compact +representation such as CBOR, MessagePack, or even just compressed JSON will be +a likely extension for the future. The support for negotiation of subprotocols +within WebSockets should make this a simple transition once time permits. + +The number of methods available for client requests is deliberately limited, as +each method requires code to be written to map it onto the equivalent REST +implementation. Some REST methods - for instance, user registration and login - +would be pointless to expose via WebSockets. It is likely, however, that we +will increate the number of methods available via the WebSockets API as it +becomes clear which would be most useful. + +Open questions +-------------- + +Throttling +~~~~~~~~~~ +At least in v2 sync, clients are inherently self-throttling - if they do not +poll quickly enough, events will be dropped from the next result. This proposal +raises the possibility that events will be produced more quickly than they can +be sent to the client; backlogs will build up on the server and/or in the +intermediate network, which will not only lead to high latency on events being +delivered, but will lead to responses to client requests also being delayed. + +We may need to implement some sort of throttling mechanism by which the server +can start to drop events. The difficulty is in knowing when to start dropping +events. A few ideas: + +* Use websocket pings to measure the RTT; if it starts to increase, start + dropping events. But this requires knowledge of the base RTT, and a useful + model of what constitutes an excessive increase. + +* Have the client acknowledge each batch of events, and use a window to ensure + the number of outstanding batches is limited. This is annoying as it requires + the client to have to acknowledge batches - and it's not clear what the right + window size is: we want a big window for long fat networks (think of mobile + clients), but a small one for one with lower latency. + +* Start dropping events if the server's TCP buffer starts filling up. This has + the advantage of delegating the congestion-detection to TCP (which already + has a number of algorithms to deal with it, to greater or lesser + effectiveness), but relies on homeservers being hosted on OSes which use + sensible TCP congestion-avoidance algorithms, and more critically, an ability + to read the fill level of the TCP send buffer. From 5ccc39b850a13874774782f2454f5f136f6e9e1d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 20 Nov 2015 18:45:09 +0000 Subject: [PATCH 086/111] Say that type is an event type --- api/client-server/v2_alpha/account-data.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/v2_alpha/account-data.yaml index 98134d6fe..b634dddc5 100644 --- a/api/client-server/v2_alpha/account-data.yaml +++ b/api/client-server/v2_alpha/account-data.yaml @@ -41,8 +41,8 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be namespaced to - avoid clashes. + The event type of the account_data to set. Custom types should be + namespaced to avoid clashes. x-example: "org.example.custom.config" - in: body name: content @@ -87,7 +87,7 @@ paths: name: type required: true description: |- - The type of the account_data to set. Custom types should be + The event type of the account_data to set. Custom types should be namespaced to avoid clashes. x-example: "org.example.custom.room.config" - in: body From e045f28b44137303acccc833ffb486008c46be06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:20:54 +0000 Subject: [PATCH 087/111] Pull out constant for permissions Also, drop permissions from 0755 to 0700 --- scripts/speculator/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 4972210eb..cfe0f4ee7 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -64,13 +64,14 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" - matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + permissionsOwnerFull = 0700 ) func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - if err := os.MkdirAll(directory, 0755); err != nil { + if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil { return "", fmt.Errorf("error making directory %s: %v", directory, err) } args := []string{"clone", url, directory} From 866fa582763279063856ea89c3459e4eeb5ae2b7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:22:53 +0000 Subject: [PATCH 088/111] Rename --- scripts/speculator/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index cfe0f4ee7..e18743834 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -148,7 +148,8 @@ func (s *server) updateBase() error { return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) } -func (s *server) knowsAbout(sha string) bool { +// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL. +func (s *server) canCheckout(sha string) bool { s.mu.Lock() defer s.mu.Unlock() return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil @@ -157,7 +158,7 @@ func (s *server) knowsAbout(sha string) bool { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - if !s.knowsAbout(sha) { + if !s.canCheckout(sha) { err = s.updateBase() if err != nil { return From c432396079645052c8a9c60eb5be822889d25117 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:26:32 +0000 Subject: [PATCH 089/111] Add comment --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e18743834..0c9a2bfa7 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -197,6 +197,8 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. + // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return From 6a6cbd9d24a7cbef9db04a57b6d273e09d51ddf1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 23 Nov 2015 17:28:58 +0000 Subject: [PATCH 090/111] Always try to build continuserv & speculator on jenkins --- jenkins.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins.sh b/jenkins.sh index b2aa84891..f5ed3b15a 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -8,7 +8,11 @@ set -ex (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) -if which go >/dev/null 2>/dev/null; then - (cd scripts/continuserv && go build) - (cd scripts/speculator && go build) -fi +: ${GOPATH:=${WORKSPACE}/.gopath} +mkdir -p "${GOPATH}" +export GOPATH +go get github.com/hashicorp/golang-lru +go get gopkg.in/fsnotify.v1 + +(cd scripts/continuserv && go build) +(cd scripts/speculator && go build) From 4e42aab245514a0d7373e23b282c1a284b100dec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 25 Nov 2015 15:13:00 +0000 Subject: [PATCH 091/111] Fix up the multi-hash ratchet thing --- drafts/markjh_end_to_end.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst index 1699f79bd..07390b5e5 100644 --- a/drafts/markjh_end_to_end.rst +++ b/drafts/markjh_end_to_end.rst @@ -84,18 +84,22 @@ follows: .. math:: \begin{align} - R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ - R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\ R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ - R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ - R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ - R_{i,3} &= H_1\left(R_{(i-1),3}\right) + R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_3\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_3\left(R_{(i-1),3}\right) \end{align} -Where :math:`H_1` and :math:`H_2` are different hash functions. For example -:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and -:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. +Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3` +are different hash functions. For example +:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`. So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. From ec31c0f5184fcf932981b854d770e90011e8fd24 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 12:04:37 +0000 Subject: [PATCH 092/111] speculator: allow styling like matrix.org --- scripts/speculator/main.go | 47 ++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 0c9a2bfa7..74731ff63 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -54,9 +54,11 @@ type User struct { } var ( - port = flag.Int("port", 9000, "Port on which to listen for HTTP") - allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") + allowedMembers map[string]bool + specCache *lru.Cache // string -> []byte + styledSpecCache *lru.Cache // string -> []byte ) func (u *User) IsTrusted() bool { @@ -196,6 +198,13 @@ func (s *server) getSHAOf(ref string) (string, error) { func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string + var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != "" + + if styleLikeMatrixDotOrg && *includesDir == "" { + writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified")) + return + } + if strings.ToLower(req.URL.Path) == "/spec/head" { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. @@ -220,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { } sha = pr.Head.SHA } - if cached, ok := specCache.Get(sha); ok { + + var cache = specCache + if styleLikeMatrixDotOrg { + cache = styledSpecCache + } + + if cached, ok := cache.Get(sha); ok { w.Write(cached.([]byte)) return } @@ -232,13 +247,24 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) if err != nil { writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) return } w.Write(b) - specCache.Add(sha, b) + cache.Add(sha, b) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -384,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) { pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } s += `` + if *includesDir != "" { + s += `` + } + io.WriteString(w, s) } @@ -448,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) { } func initCache() error { - c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) - specCache = c + c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + specCache = c1 + + c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s) + styledSpecCache = c2 return err } From c25a806cef3f5129141449d21fbb888fffa6b266 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 15:03:39 +0000 Subject: [PATCH 093/111] Fix and include /directory api docs --- api/client-server/v1/directory.yaml | 48 ++++++++++++++++++++++++++--- specification/client_server_api.rst | 24 +++------------ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/api/client-server/v1/directory.yaml b/api/client-server/v1/directory.yaml index c70b9f6ba..4966a9201 100644 --- a/api/client-server/v1/directory.yaml +++ b/api/client-server/v1/directory.yaml @@ -29,6 +29,7 @@ paths: name: roomAlias description: The room alias to set. required: true + x-example: "#monkeys:matrix.org" - in: body name: roomInfo description: Information about this room alias. @@ -39,11 +40,18 @@ paths: room_id: type: string description: The room ID to set. + example: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com" + } responses: 200: description: The mapping was created. + examples: + application/json: |- + {} schema: - type: object # empty json object + type: object get: summary: Get the room ID corresponding to this room alias. parameters: @@ -52,6 +60,7 @@ paths: name: roomAlias description: The room alias. required: true + x-example: "#monkeys:matrix.org" responses: 200: description: The room ID and other information for this alias. @@ -67,10 +76,38 @@ paths: items: type: string description: A server which is aware of this room ID. + examples: + application/json: |- + { + "room_id": "!abnjk1jdasj98:capuchins.com", + "servers": [ + "capuchins.com", + "matrix.org", + "another.com" + ] + } 404: description: There is no mapped room ID for this room alias. + examples: + application/json: |- + { + "errcode": "M_NOT_FOUND", + "error": "Room ID !abnjk1jdasj98:capuchins.com not found." + } + 409: + description: A room alias with that name already exists. + examples: + application/json: |- + { + "errcode": "M_UNKNOWN", + "error": "Room alias #monkeys:matrix.org already exists." + } delete: summary: Remove a mapping of room alias to room ID. + description: |- + Remove a mapping of room alias to room ID. + + Servers may choose to implement additional access control checks here, for instance that room aliases can only be deleted by their creator or a server administrator. security: - accessToken: [] parameters: @@ -79,9 +116,12 @@ paths: name: roomAlias description: The room alias to remove. required: true + x-example: "#monkeys:matrix.org" responses: 200: - description: The mapping was removed. + description: The mapping was deleted. + examples: + application/json: |- + {} schema: - type: object # empty json object - + type: object diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bcb..245a9eab0 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -783,19 +783,9 @@ client has to use the the following API. Room aliases ~~~~~~~~~~~~ -.. NOTE:: - This section is a work in progress. - -Room aliases can be created by sending a ``PUT /directory/room/``:: - - { - "room_id": - } -They can be deleted by sending a ``DELETE /directory/room/`` with -no content. Only some privileged users may be able to delete room aliases, e.g. -server admins, the creator of the room alias, etc. This specification does not -outline the privilege level required for deleting room aliases. +Servers may host aliases for rooms with human-friendly names. Aliases take the +form ``#friendlyname:server.name``. As room aliases are scoped to a particular home server domain name, it is likely that a home server will reject attempts to maintain aliases on other @@ -812,17 +802,11 @@ appears to have a room alias of ``#alias:example.com``, this SHOULD be checked to make sure that the room's ID matches the ``room_id`` returned from the request. -Room aliases can be checked in the same way they are resolved; by sending a -``GET /directory/room/``:: - - { - "room_id": , - "servers": [ , , ] - } - Home servers can respond to resolve requests for aliases on other domains than their own by using the federation API to ask other domain name home servers. +{{directory_http_api}} + Permissions ~~~~~~~~~~~ From 5e30b5b8d74cdfbd764175fd735b3c39d652453e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 26 Nov 2015 16:46:29 +0000 Subject: [PATCH 094/111] Remove POST version of /send PUT should always be used. --- api/client-server/v1/room_send.yaml | 47 --------------------- specification/modules/instant_messaging.rst | 8 +--- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/v1/room_send.yaml index 9c9273dfe..3ab1ac40d 100644 --- a/api/client-server/v1/room_send.yaml +++ b/api/client-server/v1/room_send.yaml @@ -76,50 +76,3 @@ paths: type: string description: |- A unique identifier for the event. - "/rooms/{roomId}/send/{eventType}": - post: - summary: Send a message event to the given room. - description: |- - This endpoint can be used to send a message event to a room; however - the lack of a transaction ID means that it is possible to cause message - duplication if events are resent on error, so it is preferable to use - `PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room to send the event to. - required: true - x-example: "!636q39766251:example.com" - - in: path - type: string - name: eventType - description: The type of event to send. - required: true - x-example: "m.room.message" - - in: body - name: body - schema: - type: object - example: |- - { - "msgtype": "m.text", - "body": "hello" - } - responses: - 200: - description: "An ID for the sent event." - examples: - application/json: |- - { - "event_id": "YUwRidLecu" - } - schema: - type: object - properties: - event_id: - type: string - description: |- - A unique identifier for the event. diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index a58c762f3..cd3850014 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key. Recommendations when sending messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use -``PUT`` requests with `transaction IDs`_ to make requests idempotent. This -ensures that messages are sent exactly once even under poor network conditions. -Clients SHOULD retry requests using an exponential-backoff algorithm for a +In the event of send failure, clients SHOULD retry requests using an +exponential-backoff algorithm for a certain amount of time T. It is recommended that T is no longer than 5 minutes. After this time, the client should stop retrying and mark the message as "unsent". Users should be able to manually resend unsent messages. @@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room rather than a global queue, as ordering is only relevant within a single room rather than between rooms. -.. _`transaction IDs`: `sect:txn_ids`_ - Local echo ~~~~~~~~~~ From d39494b6dfda679bc4d38a96efb1715428f35279 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 26 Nov 2015 16:55:12 +0000 Subject: [PATCH 095/111] Fix typo in sync example --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c3d120b45..6f2f1412d 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -264,7 +264,7 @@ paths: "account_data": { "events": [ { - "type": "m.tags", + "type": "m.tag", "content": {"tags": {"work": {"order": "1"}}} } ] From 8d415367573bce44a3a9f2f663157fa08e054bf9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 11:37:24 +0000 Subject: [PATCH 096/111] Address kegan's comments Minor fixes to the e2e spec as raiseds by kegan --- specification/41_end_to_end_encryption.rst | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/specification/41_end_to_end_encryption.rst b/specification/41_end_to_end_encryption.rst index 023a56848..c5fa4528e 100644 --- a/specification/41_end_to_end_encryption.rst +++ b/specification/41_end_to_end_encryption.rst @@ -67,32 +67,35 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with "m." are reserved for algorithms +specification. Algorithm names that start with `m.` are reserved for algorithms defined by this specification. Implementations wanting to experiment with new algorithms are encouraged to pick algorithm names that start with their domain to reduce the risk of collisions. -The name "m.olm.v1.curve25519-aes-sha2" corresponds to version 1 of the Olm +Algorithm names should be short and meaningful, and should list the primitives +used by the algorithm so that it is easier to see if the algorithm is using a +broken primitive. + +The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -Algorithm names should be short and meaningful. A name of "m.olm.v1" is too -short. However a name of -"m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256" -is too long despite giving a more precise description of the algorithm. - -Algorithm names should list the primitives used by the algorithm so that it -easier to see if the algorithm is using a broken primitive. +A name of `m.olm.v1` is too short: it gives no information about the primitives +in use, and is difficult to extend for different primitives. However a name of +`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +is too long despite giving a more precise description of the algorithm: it adds +to the data transfer overhead and sacrifices clarity for human readers without +adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name "ed25519" corresponds to the Ed25519 signature algorithm. The key is +The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name "curve25519" corresponds to the Curve25519 ECDH algorithm. The key is +The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour @@ -149,7 +152,7 @@ Downloading Keys ~~~~~~~~~~~~~~~~ Keys are downloaded as a collection of signed JSON objects. There -will be a JSON object per device per user. If one of the user's +will be one JSON object per device per user. If one of the user's devices doesn't support end-to-end encryption then their homeserver must synthesise a JSON object without any device keys for that device. From 2aa4773cc15acfdfa86be396267b1745cbe73ee6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:01:03 +0000 Subject: [PATCH 097/111] Make the speculator serve up errors as plain text ... so that they are legible. --- scripts/speculator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 74731ff63..abe77c74d 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -135,6 +135,7 @@ func generate(dir string) error { } func writeError(w http.ResponseWriter, code int, err error) { + w.Header().Set("Content-Type", "text/plain") w.WriteHeader(code) io.WriteString(w, fmt.Sprintf("%v\n", err)) } From 2dbb8ba56c35bf4e914d0c8105bf8db14e4c09ff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 12:53:03 +0000 Subject: [PATCH 098/111] Fix title levels make the title decoration consistent with the rest of the spec --- specification/modules/end_to_end_encryption.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index cb770680f..3ec9368ad 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -285,7 +285,7 @@ Encrypted messages are sent in the form. Using Olm -######### ++++++++++ Devices that support olm must include "m.olm.v1.curve25519-aes-sha2" in their list of supported chat algorithms, must list a Curve25519 device key, and From 0b1ba70a320a7f4fe90e34edf4e3352d5e770433 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Nov 2015 13:33:58 +0000 Subject: [PATCH 099/111] fix rst markup `` > ` --- .../modules/end_to_end_encryption.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 3ec9368ad..8b38ab938 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -69,24 +69,24 @@ Messaging Algorithm Names ~~~~~~~~~~~~~~~~~~~~~~~~~ Messaging algorithm names use the extensible naming scheme used throughout this -specification. Algorithm names that start with `m.` are reserved for algorithms -defined by this specification. Implementations wanting to experiment with new -algorithms are encouraged to pick algorithm names that start with their -domain to reduce the risk of collisions. +specification. Algorithm names that start with ``m.`` are reserved for +algorithms defined by this specification. Implementations wanting to experiment +with new algorithms are encouraged to pick algorithm names that start with +their domain to reduce the risk of collisions. Algorithm names should be short and meaningful, and should list the primitives used by the algorithm so that it is easier to see if the algorithm is using a broken primitive. -The name `m.olm.v1.curve25519-aes-sha2` corresponds to version 1 of the Olm +The name ``m.olm.v1.curve25519-aes-sha2`` corresponds to version 1 of the Olm ratchet using Curve25519 for the initial key agreement, HKDF-SHA-256 for ratchet key derivation, Curve25519 for the DH ratchet, HMAC-SHA-256 for the hash ratchet, and HKDF-SHA-256, AES-256 in CBC mode, and 8 byte truncated HMAC-SHA-256 for authenticated encryption. -A name of `m.olm.v1` is too short: it gives no information about the primitives +A name of ``m.olm.v1`` is too short: it gives no information about the primitives in use, and is difficult to extend for different primitives. However a name of -`m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256` +``m.olm.v1.ecdh-curve25519-hdkfsha256.hmacsha256.hkdfsha256-aes256-cbc-hmac64sha256`` is too long despite giving a more precise description of the algorithm: it adds to the data transfer overhead and sacrifices clarity for human readers without adding any useful extra information. @@ -94,10 +94,10 @@ adding any useful extra information. Key Algorithms ~~~~~~~~~~~~~~ -The name `ed25519` corresponds to the Ed25519 signature algorithm. The key is +The name ``ed25519`` corresponds to the Ed25519 signature algorithm. The key is a Base64 encoded 32-byte Ed25519 public key. -The name `curve25519` corresponds to the Curve25519 ECDH algorithm. The key is +The name ``curve25519`` corresponds to the Curve25519 ECDH algorithm. The key is a Base64 encoded 32-byte Curve25519 public key. Client Behaviour From ad4d8ae7a611bdd13167c358c0447e8fad089fd8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 13:50:24 +0000 Subject: [PATCH 100/111] Move client-server intro to client-server section --- specification/client_server_api.rst | 109 ++++++++++++++++++++++++++++ specification/intro.rst | 109 ---------------------------- 2 files changed, 109 insertions(+), 109 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index ba5578bcb..402ce7531 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,115 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +API Standards +------------- + +.. TODO + Need to specify any HMAC or access_token lifetime/ratcheting tricks + We need to specify capability negotiation for extensible transports + +The mandatory baseline for communication in Matrix is exchanging JSON objects +over HTTP APIs. HTTPS is mandated as the baseline for server-server +(federation) communication. HTTPS is recommended for client-server +communication, although HTTP may be supported as a fallback to support basic +HTTP clients. More efficient optional transports for client-server +communication will in future be supported as optional extensions - e.g. a +packed binary encoding over stream-cipher encrypted TCP socket for +low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all +API calls use a Content-Type of ``application/json``. In addition, all strings +MUST be encoded as UTF-8. Clients are authenticated using opaque +``access_token`` strings (see `Client Authentication`_ for details), passed as a +query string parameter on all requests. + +Any errors which occur at the Matrix API level MUST return a "standard error +response". This is a JSON object which looks like:: + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error +codes should have their namespace first in ALL CAPS, followed by a single _ to +ease separating the namespace from the error code. For example, if there was a +custom namespace ``com.mydomain.here``, and a +``FORBIDDEN`` code, the error code should look like +``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the +error, but the keys ``error`` and ``errcode`` MUST always be present. + +Some standard error codes are below: + +:``M_FORBIDDEN``: + Forbidden access, e.g. joining a room without permission, failed login. + +:``M_UNKNOWN_TOKEN``: + The access token specified was not recognised. + +:``M_BAD_JSON``: + Request contained valid JSON, but it was malformed in some way, e.g. missing + required keys, invalid values for keys. + +:``M_NOT_JSON``: + Request did not contain valid JSON. + +:``M_NOT_FOUND``: + No resource was found for this request. + +:``M_LIMIT_EXCEEDED``: + Too many requests have been sent in a short period of time. Wait a while then + try again. + +Some requests have unique error codes: + +:``M_USER_IN_USE``: + Encountered when trying to register a user ID which has been taken. + +:``M_ROOM_IN_USE``: + Encountered when trying to create a room which has been taken. + +:``M_BAD_PAGINATION``: + Encountered when specifying bad pagination query parameters. + +.. _sect:txn_ids: + +The Client-Server API typically uses ``HTTP POST`` to submit requests. This +means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to +make requests idempotent. In order to use a ``PUT``, paths should be suffixed +with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which +identifies the request, and is scoped to a given Client (identified by that +client's ``access_token``). Crucially, it **only** serves to identify new +requests from retransmits. After the request has finished, the ``{txnId}`` +value should be changed (how is not specified; a monotonically increasing +integer is recommended). It is preferable to use ``HTTP PUT`` to make sure +requests to send messages do not get sent more than once should clients need to +retransmit requests. + +Valid requests look like:: + + POST /some/path/here?access_token=secret + { + "key": "This is a post." + } + + PUT /some/path/here/11?access_token=secret + { + "key": "This is a put with a txnId of 11." + } + +In contrast, these are invalid requests:: + + POST /some/path/here/11?access_token=secret + { + "key": "This is a post, but it has a txnId." + } + + PUT /some/path/here?access_token=secret + { + "key": "This is a put but it is missing a txnId." + } + Client Authentication --------------------- Most API endpoints require the user to identify themselves by presenting diff --git a/specification/intro.rst b/specification/intro.rst index 64a211083..6ff7946cf 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -358,112 +358,3 @@ dedicated API. The API is symmetrical to managing Profile data. Would it really be overengineered to use the same API for both profile & private user data, but with different ACLs? -API Standards -------------- - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks - We need to specify capability negotiation for extensible transports - -The mandatory baseline for communication in Matrix is exchanging JSON objects -over HTTP APIs. HTTPS is mandated as the baseline for server-server -(federation) communication. HTTPS is recommended for client-server -communication, although HTTP may be supported as a fallback to support basic -HTTP clients. More efficient optional transports for client-server -communication will in future be supported as optional extensions - e.g. a -packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all -API calls use a Content-Type of ``application/json``. In addition, all strings -MUST be encoded as UTF-8. Clients are authenticated using opaque -``access_token`` strings (see `Client Authentication`_ for details), passed as a -query string parameter on all requests. - -Any errors which occur at the Matrix API level MUST return a "standard error -response". This is a JSON object which looks like:: - - { - "errcode": "", - "error": "" - } - -The ``error`` string will be a human-readable error message, usually a sentence -explaining what went wrong. The ``errcode`` string will be a unique string -which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error -codes should have their namespace first in ALL CAPS, followed by a single _ to -ease separating the namespace from the error code. For example, if there was a -custom namespace ``com.mydomain.here``, and a -``FORBIDDEN`` code, the error code should look like -``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the -error, but the keys ``error`` and ``errcode`` MUST always be present. - -Some standard error codes are below: - -:``M_FORBIDDEN``: - Forbidden access, e.g. joining a room without permission, failed login. - -:``M_UNKNOWN_TOKEN``: - The access token specified was not recognised. - -:``M_BAD_JSON``: - Request contained valid JSON, but it was malformed in some way, e.g. missing - required keys, invalid values for keys. - -:``M_NOT_JSON``: - Request did not contain valid JSON. - -:``M_NOT_FOUND``: - No resource was found for this request. - -:``M_LIMIT_EXCEEDED``: - Too many requests have been sent in a short period of time. Wait a while then - try again. - -Some requests have unique error codes: - -:``M_USER_IN_USE``: - Encountered when trying to register a user ID which has been taken. - -:``M_ROOM_IN_USE``: - Encountered when trying to create a room which has been taken. - -:``M_BAD_PAGINATION``: - Encountered when specifying bad pagination query parameters. - -.. _sect:txn_ids: - -The Client-Server API typically uses ``HTTP POST`` to submit requests. This -means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to -make requests idempotent. In order to use a ``PUT``, paths should be suffixed -with ``/{txnId}``. ``{txnId}`` is a unique client-generated transaction ID which -identifies the request, and is scoped to a given Client (identified by that -client's ``access_token``). Crucially, it **only** serves to identify new -requests from retransmits. After the request has finished, the ``{txnId}`` -value should be changed (how is not specified; a monotonically increasing -integer is recommended). It is preferable to use ``HTTP PUT`` to make sure -requests to send messages do not get sent more than once should clients need to -retransmit requests. - -Valid requests look like:: - - POST /some/path/here?access_token=secret - { - "key": "This is a post." - } - - PUT /some/path/here/11?access_token=secret - { - "key": "This is a put with a txnId of 11." - } - -In contrast, these are invalid requests:: - - POST /some/path/here/11?access_token=secret - { - "key": "This is a post, but it has a txnId." - } - - PUT /some/path/here?access_token=secret - { - "key": "This is a put but it is missing a txnId." - } - From f0c99a6925a4d5ff38fcc563a50ec280b6895631 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:03:22 +0000 Subject: [PATCH 101/111] Generate API docs from swagger --- api/client-server/api-docs | 50 - api/client-server/api-docs-content | 119 --- api/client-server/api-docs-directory | 101 -- api/client-server/api-docs-events | 247 ----- api/client-server/api-docs-login | 120 --- api/client-server/api-docs-presence | 164 ---- api/client-server/api-docs-profile | 122 --- api/client-server/api-docs-registration | 120 --- api/client-server/api-docs-rooms | 1128 ----------------------- scripts/add-matrix-org-stylings.sh | 8 +- scripts/generate-http-docs.sh | 27 + 11 files changed, 32 insertions(+), 2174 deletions(-) delete mode 100644 api/client-server/api-docs delete mode 100644 api/client-server/api-docs-content delete mode 100644 api/client-server/api-docs-directory delete mode 100644 api/client-server/api-docs-events delete mode 100644 api/client-server/api-docs-login delete mode 100644 api/client-server/api-docs-presence delete mode 100644 api/client-server/api-docs-profile delete mode 100644 api/client-server/api-docs-registration delete mode 100644 api/client-server/api-docs-rooms create mode 100755 scripts/generate-http-docs.sh diff --git a/api/client-server/api-docs b/api/client-server/api-docs deleted file mode 100644 index 9c8c60665..000000000 --- a/api/client-server/api-docs +++ /dev/null @@ -1,50 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "apis": [ - { - "path": "-login", - "description": "Login operations" - }, - { - "path": "-registration", - "description": "Registration operations" - }, - { - "path": "-rooms", - "description": "Room operations" - }, - { - "path": "-profile", - "description": "Profile operations" - }, - { - "path": "-presence", - "description": "Presence operations" - }, - { - "path": "-events", - "description": "Event operations" - }, - { - "path": "-directory", - "description": "Directory operations" - }, - { - "path": "-content", - "description": "Content repository operations" - } - ], - "authorizations": { - "token": { - "scopes": [] - } - }, - "info": { - "title": "Matrix Client-Server API Reference", - "description": "This contains the client-server API for the reference implementation of the home server", - "termsOfServiceUrl": "http://matrix.org", - "license": "Apache 2.0", - "licenseUrl": "http://www.apache.org/licenses/LICENSE-2.0.html" - } -} diff --git a/api/client-server/api-docs-content b/api/client-server/api-docs-content deleted file mode 100644 index a0a31da07..000000000 --- a/api/client-server/api-docs-content +++ /dev/null @@ -1,119 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix", - "resourcePath": "/media/v1/", - "apis": [ - { - "path": "/media/v1/upload", - "operations": [ - { - "method": "POST", - "summary": "Upload some content to the content repository.", - "type": "ContentUploadResponse", - "nickname": "upload_content", - "parameters": [ - { - "name": "body", - "description": "The file to upload.", - "required": true, - "type": "file", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/media/v1/download/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get the content stored at this address.", - "type": "file", - "nickname": "download_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/media/v1/thumbnail/{serverName}/{mediaId}", - "operations": [ - { - "method": "GET", - "summary": "Get a thumbnail of the content stored at this address.", - "type": "file", - "nickname": "thumbnail_content", - "parameters": [ - { - "name": "serverName", - "description": "The serverName from the mxc:/// URI (the authority component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "mediaId", - "description": "The mediaId from the mxc:/// URI (the path component).", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "width", - "description": "The desired width of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "height", - "description": "The desired height of the thumbnail.", - "required": false, - "type": "integer", - "paramType": "query" - }, - { - "name": "method", - "description": "The desired resizing method.", - "enum": [ - "crop", - "scale" - ], - "required": false, - "type": "string", - "paramType": "query" - } - ] - } - ] - } - ], - "models": { - "ContentUploadResponse": { - "id": "ContentUploadResponse", - "properties": { - "content_uri": { - "type": "string", - "description": "The mxc:// URI where this content is stored. This is of the form 'mxc://{serverName}/{mediaId}'", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-directory b/api/client-server/api-docs-directory deleted file mode 100644 index 5dda58065..000000000 --- a/api/client-server/api-docs-directory +++ /dev/null @@ -1,101 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/directory", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/directory/room/{roomAlias}", - "operations": [ - { - "method": "GET", - "summary": "Get the room ID corresponding to this room alias.", - "notes": "Volatile: This API is likely to change.", - "type": "DirectoryResponse", - "nickname": "get_room_id_for_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "PUT", - "summary": "Create a new mapping from room alias to room ID.", - "notes": "Volatile: This API is likely to change.", - "type": "void", - "nickname": "add_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to set.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The room ID to set.", - "required": true, - "type": "RoomAliasRequest", - "paramType": "body" - } - ] - }, - { - "method": "DELETE", - "summary": "Removes a mapping of room alias to room ID.", - "notes": "Only privileged users can perform this action.", - "type": "void", - "nickname": "remove_room_alias", - "parameters": [ - { - "name": "roomAlias", - "description": "The room alias to remove.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DirectoryResponse": { - "id": "DirectoryResponse", - "properties": { - "room_id": { - "type": "string", - "description": "The fully-qualified room ID.", - "required": true - }, - "servers": { - "type": "array", - "items": { - "$ref": "string" - }, - "description": "A list of servers that know about this room.", - "required": true - } - } - }, - "RoomAliasRequest": { - "id": "RoomAliasRequest", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID to map the alias to.", - "required": true - } - } - } - } -} diff --git a/api/client-server/api-docs-events b/api/client-server/api-docs-events deleted file mode 100644 index 1bdb9b034..000000000 --- a/api/client-server/api-docs-events +++ /dev/null @@ -1,247 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/events", - "produces": [ - "application/json" - ], - "apis": [ - { - "path": "/events", - "operations": [ - { - "method": "GET", - "summary": "Listen on the event stream", - "notes": "This can only be done by the logged in user. This will block until an event is received, or until the timeout is reached.", - "type": "PaginationChunk", - "nickname": "get_event_stream", - "parameters": [ - { - "name": "from", - "description": "The token to stream from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "timeout", - "description": "The maximum time in milliseconds to wait for an event.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ], - - "responseMessages": [ - { - "code": 400, - "message": "Bad pagination token." - } - ] - }, - { - "path": "/events/{eventId}", - "operations": [ - { - "method": "GET", - "summary": "Get information about a single event.", - "notes": "Get information about a single event.", - "type": "Event", - "nickname": "get_event", - "parameters": [ - { - "name": "eventId", - "description": "The event ID to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Event not found." - } - ] - } - ] - }, - { - "path": "/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get this user's current state.", - "notes": "Get this user's current state.", - "type": "InitialSyncResponse", - "nickname": "initial_sync", - "parameters": [ - { - "name": "limit", - "description": "The maximum number of messages to return for each room.", - "type": "integer", - "paramType": "query", - "required": false - } - ] - } - ] - }, - { - "path": "/publicRooms", - "operations": [ - { - "method": "GET", - "summary": "Get a list of publicly visible rooms.", - "type": "PublicRoomsPaginationChunk", - "nickname": "get_public_room_list" - } - ] - } - ], - "models": { - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "An array of events.", - "required": true, - "items": { - "$ref": "Event" - } - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred.", - "required": true - } - } - }, - "PublicRoomInfo": { - "id": "PublicRoomInfo", - "properties": { - "aliases": { - "type": "array", - "description": "A list of room aliases for this room.", - "items": { - "$ref": "string" - } - }, - "name": { - "type": "string", - "description": "The name of the room, as given by the m.room.name state event." - }, - "room_id": { - "type": "string", - "description": "The room ID for this public room.", - "required": true - }, - "topic": { - "type": "string", - "description": "The topic of this room, as given by the m.room.topic state event." - } - } - }, - "PublicRoomsPaginationChunk": { - "id": "PublicRoomsPaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - }, - "chunk": { - "type": "array", - "description": "A list of public room data.", - "required": true, - "items": { - "$ref": "PublicRoomInfo" - } - } - } - }, - "InitialSyncResponse": { - "id": "InitialSyncResponse", - "properties": { - "end": { - "type": "string", - "description": "A streaming token which can be used with /events to continue from this snapshot of data.", - "required": true - }, - "presence": { - "type": "array", - "description": "A list of presence events.", - "items": { - "$ref": "Event" - }, - "required": false - }, - "rooms": { - "type": "array", - "description": "A list of initial sync room data.", - "required": false, - "items": { - "$ref": "InitialSyncRoomData" - } - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "PaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/api/client-server/api-docs-login b/api/client-server/api-docs-login deleted file mode 100644 index d6f8d84f2..000000000 --- a/api/client-server/api-docs-login +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_login_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when logging in.", - "type": "LoginFlows" - }, - { - "method": "POST", - "nickname": "submit_login", - "notes": "If this is part of a multi-stage login, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A login submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "LoginSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a login action.", - "type": "LoginResult" - } - ], - "path": "/login" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "LoginFlows": { - "id": "LoginFlows", - "properties": { - "flows": { - "description": "A list of valid login flows.", - "type": "array", - "items": { - "$ref": "LoginInfo" - } - } - } - }, - "LoginInfo": { - "id": "LoginInfo", - "properties": { - "stages": { - "description": "Multi-stage login only: An array of all the login types required to login.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The login type that must be used when logging in.", - "type": "string" - } - } - }, - "LoginResult": { - "id": "LoginResult", - "properties": { - "access_token": { - "description": "The access token for this user's login if this is the final stage of the login process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage login only: The next login type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token to send when submitting the next login type.", - "type": "string" - } - } - }, - "LoginSubmission": { - "id": "LoginSubmission", - "properties": { - "type": { - "description": "The type of login being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage login only: The session token from an earlier login stage.", - "type": "string" - }, - "_login_type_defined_keys_": { - "description": "Keys as defined by the specified login type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/login", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-presence b/api/client-server/api-docs-presence deleted file mode 100644 index 6b2244602..000000000 --- a/api/client-server/api-docs-presence +++ /dev/null @@ -1,164 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/presence", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/presence/{userId}/status", - "operations": [ - { - "method": "PUT", - "summary": "Update this user's presence state.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "update_presence", - "parameters": [ - { - "name": "body", - "description": "The new presence state", - "required": true, - "type": "PresenceUpdate", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose presence to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get this user's presence state.", - "notes": "Get this user's presence state.", - "type": "PresenceUpdate", - "nickname": "get_presence", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/presence/list/{userId}", - "operations": [ - { - "method": "GET", - "summary": "Retrieve a list of presences for all of this user's friends.", - "notes": "", - "type": "array", - "items": { - "$ref": "Presence" - }, - "nickname": "get_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "POST", - "summary": "Add or remove users from this presence list.", - "notes": "Add or remove users from this presence list.", - "type": "void", - "nickname": "modify_presence_list", - "parameters": [ - { - "name": "userId", - "description": "The user whose presence list is being modified.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The modifications to make to this presence list.", - "required": true, - "type": "PresenceListModifications", - "paramType": "body" - } - ] - } - ] - } - ], - "models": { - "PresenceUpdate": { - "id": "PresenceUpdate", - "properties": { - "presence": { - "type": "string", - "description": "Enum: The presence state.", - "enum": [ - "offline", - "unavailable", - "online", - "free_for_chat" - ] - }, - "status_msg": { - "type": "string", - "description": "The user-defined message associated with this presence state." - } - }, - "subTypes": [ - "Presence" - ] - }, - "Presence": { - "id": "Presence", - "properties": { - "last_active_ago": { - "type": "integer", - "format": "int64", - "description": "The last time this user performed an action on their home server." - }, - "user_id": { - "type": "string", - "description": "The fully qualified user ID" - } - } - }, - "PresenceListModifications": { - "id": "PresenceListModifications", - "properties": { - "invite": { - "type": "array", - "description": "A list of user IDs to add to the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - }, - "drop": { - "type": "array", - "description": "A list of user IDs to remove from the list.", - "items": { - "type": "string", - "description": "A fully qualified user ID." - } - } - } - } - } -} diff --git a/api/client-server/api-docs-profile b/api/client-server/api-docs-profile deleted file mode 100644 index d2fccaa67..000000000 --- a/api/client-server/api-docs-profile +++ /dev/null @@ -1,122 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/profile", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "apis": [ - { - "path": "/profile/{userId}/displayname", - "operations": [ - { - "method": "PUT", - "summary": "Set a display name.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_display_name", - "parameters": [ - { - "name": "body", - "description": "The new display name for this user.", - "required": true, - "type": "DisplayName", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose display name to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get a display name.", - "notes": "This can be done by anyone.", - "type": "DisplayName", - "nickname": "get_display_name", - "parameters": [ - { - "name": "userId", - "description": "The user whose display name to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/profile/{userId}/avatar_url", - "operations": [ - { - "method": "PUT", - "summary": "Set an avatar URL.", - "notes": "This can only be done by the logged in user.", - "type": "void", - "nickname": "set_avatar_url", - "parameters": [ - { - "name": "body", - "description": "The new avatar url for this user.", - "required": true, - "type": "AvatarUrl", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose avatar url to set.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get an avatar url.", - "notes": "This can be done by anyone.", - "type": "AvatarUrl", - "nickname": "get_avatar_url", - "parameters": [ - { - "name": "userId", - "description": "The user whose avatar url to get.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "DisplayName": { - "id": "DisplayName", - "properties": { - "displayname": { - "type": "string", - "description": "The textual display name" - } - } - }, - "AvatarUrl": { - "id": "AvatarUrl", - "properties": { - "avatar_url": { - "type": "string", - "description": "A url to an image representing an avatar." - } - } - } - } -} diff --git a/api/client-server/api-docs-registration b/api/client-server/api-docs-registration deleted file mode 100644 index 11c170c3e..000000000 --- a/api/client-server/api-docs-registration +++ /dev/null @@ -1,120 +0,0 @@ -{ - "apiVersion": "1.0.0", - "apis": [ - { - "operations": [ - { - "method": "GET", - "nickname": "get_registration_info", - "notes": "All login stages MUST be mentioned if there is >1 login type.", - "summary": "Get the login mechanism to use when registering.", - "type": "RegistrationFlows" - }, - { - "method": "POST", - "nickname": "submit_registration", - "notes": "If this is part of a multi-stage registration, there MUST be a 'session' key.", - "parameters": [ - { - "description": "A registration submission", - "name": "body", - "paramType": "body", - "required": true, - "type": "RegistrationSubmission" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad login type" - }, - { - "code": 400, - "message": "Missing JSON keys" - } - ], - "summary": "Submit a registration action.", - "type": "RegistrationResult" - } - ], - "path": "/register" - } - ], - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "consumes": [ - "application/json" - ], - "models": { - "RegistrationFlows": { - "id": "RegistrationFlows", - "properties": { - "flows": { - "description": "A list of valid registration flows.", - "type": "array", - "items": { - "$ref": "RegistrationInfo" - } - } - } - }, - "RegistrationInfo": { - "id": "RegistrationInfo", - "properties": { - "stages": { - "description": "Multi-stage registration only: An array of all the login types required to registration.", - "items": { - "$ref": "string" - }, - "type": "array" - }, - "type": { - "description": "The first login type that must be used when logging in.", - "type": "string" - } - } - }, - "RegistrationResult": { - "id": "RegistrationResult", - "properties": { - "access_token": { - "description": "The access token for this user's registration if this is the final stage of the registration process.", - "type": "string" - }, - "user_id": { - "description": "The user's fully-qualified user ID.", - "type": "string" - }, - "next": { - "description": "Multi-stage registration only: The next registration type to submit.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token to send when submitting the next registration type.", - "type": "string" - } - } - }, - "RegistrationSubmission": { - "id": "RegistrationSubmission", - "properties": { - "type": { - "description": "The type of registration being submitted.", - "type": "string" - }, - "session": { - "description": "Multi-stage registration only: The session token from an earlier registration stage.", - "type": "string" - }, - "_registration_type_defined_keys_": { - "description": "Keys as defined by the specified registration type, e.g. \"user\", \"password\"" - } - } - } - }, - "produces": [ - "application/json" - ], - "resourcePath": "/register", - "swaggerVersion": "1.2" -} - diff --git a/api/client-server/api-docs-rooms b/api/client-server/api-docs-rooms deleted file mode 100644 index 0d3d9fbcb..000000000 --- a/api/client-server/api-docs-rooms +++ /dev/null @@ -1,1128 +0,0 @@ -{ - "apiVersion": "1.0.0", - "swaggerVersion": "1.2", - "basePath": "http://localhost:8008/_matrix/client/api/v1", - "resourcePath": "/rooms", - "produces": [ - "application/json" - ], - "consumes": [ - "application/json" - ], - "authorizations": { - "token": [] - }, - "apis": [ - { - "path": "/rooms/{roomId}/send/{eventType}", - "operations": [ - { - "method": "POST", - "summary": "Send a generic non-state event to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_non_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/{eventType}/{stateKey}", - "operations": [ - { - "method": "PUT", - "summary": "Send a generic state event to this room.", - "notes": "The state key can be omitted, such that you can PUT to /rooms/{roomId}/state/{eventType}. The state key defaults to a 0 length string in this case.", - "type": "void", - "nickname": "send_state_event", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The event contents", - "required": true, - "type": "EventContent", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "eventType", - "description": "The type of event to send.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "stateKey", - "description": "An identifier used to specify clobbering semantics. State events with the same (roomId, eventType, stateKey) will be replaced.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message", - "operations": [ - { - "method": "POST", - "summary": "Send a message in this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_message", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The message contents", - "required": true, - "type": "Message", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the message in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.topic", - "operations": [ - { - "method": "PUT", - "summary": "Set the topic for this room.", - "notes": "Set the topic for this room.", - "type": "void", - "nickname": "set_topic", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The topic contents", - "required": true, - "type": "Topic", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the topic for this room.", - "notes": "Get the topic for this room.", - "type": "Topic", - "nickname": "get_topic", - "parameters": [ - { - "name": "roomId", - "description": "The room to get topic in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Topic not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.name", - "operations": [ - { - "method": "PUT", - "summary": "Set the name of this room.", - "notes": "Set the name of this room.", - "type": "void", - "nickname": "set_room_name", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The name contents", - "required": true, - "type": "RoomName", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - }, - { - "method": "GET", - "summary": "Get the room's name.", - "notes": "", - "type": "RoomName", - "nickname": "get_room_name", - "parameters": [ - { - "name": "roomId", - "description": "The room to get the name of.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Name not found." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.power_levels", - "operations": [ - { - "method": "PUT", - "summary": "Set the power levels for this room.", - "notes": "This has to be set atomically. The levels set will clobber what was previously there.", - "type": "void", - "nickname": "set_room_power_levels", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The power levels", - "required": true, - "type": "RoomPowerLevels", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to set the power levels for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/send/m.room.message.feedback", - "operations": [ - { - "method": "POST", - "summary": "Send feedback to a message.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "EventId", - "nickname": "send_feedback", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The feedback contents", - "required": true, - "type": "Feedback", - "paramType": "body" - }, - { - "name": "roomId", - "description": "The room to send the feedback in.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad feedback type." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/invite", - "operations": [ - { - "method": "POST", - "summary": "Invite a user to this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "invite", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to invite.", - "required": true, - "type": "InviteRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/join", - "operations": [ - { - "method": "POST", - "summary": "Join this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "join_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to join.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "JoinRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/leave", - "operations": [ - { - "method": "POST", - "summary": "Leave this room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "void", - "nickname": "leave", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room to leave.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "required": true, - "type": "LeaveRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/ban", - "operations": [ - { - "method": "POST", - "summary": "Ban a user in the room.", - "notes": "This operation can also be done as a PUT by suffixing /{txnId}. The caller must have the required power level to do this operation.", - "type": "void", - "nickname": "ban", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomId", - "description": "The room which has the user to ban.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The user to ban.", - "required": true, - "type": "BanRequest", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state/m.room.member/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Change the membership state for a user in a room.", - "notes": "Change the membership state for a user in a room.", - "type": "void", - "nickname": "set_membership", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The new membership state", - "required": true, - "type": "Member", - "paramType": "body" - }, - { - "name": "userId", - "description": "The user whose membership is being changed.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "No membership key." - }, - { - "code": 400, - "message": "Bad membership value." - }, - { - "code": 403, - "message": "When inviting: You are not in the room." - }, - { - "code": 403, - "message": "When inviting: is already in the room." - }, - { - "code": 403, - "message": "When joining: Cannot force another user to join." - }, - { - "code": 403, - "message": "When joining: You are not invited to this room." - } - ] - }, - { - "method": "GET", - "summary": "Get the membership state of a user in a room.", - "notes": "Get the membership state of a user in a room.", - "type": "Member", - "nickname": "get_membership", - "parameters": [ - { - "name": "userId", - "description": "The user whose membership state you want to get.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "roomId", - "description": "The room which has this user.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 404, - "message": "Member not found." - } - ] - } - ] - }, - { - "path": "/join/{roomAliasOrId}", - "operations": [ - { - "method": "POST", - "summary": "Join a room via a room alias or room ID.", - "notes": "This endpoint, unlike /rooms/{roomId}/join allows the client to join a room by it's alias. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "JoinRoomInfo", - "nickname": "join", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "roomAliasOrId", - "description": "The room alias or room ID to join.", - "required": true, - "type": "string", - "paramType": "path" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Bad room alias." - } - ] - } - ] - }, - { - "path": "/createRoom", - "operations": [ - { - "method": "POST", - "summary": "Create a room.", - "notes": "Create a room. This operation can also be done as a PUT by suffixing /{txnId}.", - "type": "RoomInfo", - "nickname": "create_room", - "consumes": [ - "application/json" - ], - "parameters": [ - { - "name": "body", - "description": "The desired configuration for the room.", - "required": true, - "type": "RoomConfig", - "paramType": "body" - } - ], - "responseMessages": [ - { - "code": 400, - "message": "Body must be JSON." - }, - { - "code": 400, - "message": "Room alias already taken." - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/messages", - "operations": [ - { - "method": "GET", - "summary": "Get a list of messages for this room.", - "notes": "Get a list of messages for this room.", - "type": "MessagePaginationChunk", - "nickname": "get_messages", - "parameters": [ - { - "name": "roomId", - "description": "The room to get messages in.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of messages to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/members", - "operations": [ - { - "method": "GET", - "summary": "Get a list of members for this room.", - "notes": "Get a list of members for this room.", - "type": "MemberPaginationChunk", - "nickname": "get_members", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of members from.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "from", - "description": "The token to start getting results from.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "to", - "description": "The token to stop getting results at.", - "required": false, - "type": "string", - "paramType": "query" - }, - { - "name": "limit", - "description": "The maximum number of members to return.", - "required": false, - "type": "integer", - "paramType": "query" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/state", - "operations": [ - { - "method": "GET", - "summary": "Get a list of all the current state events for this room.", - "notes": "This is equivalent to the events returned under the 'state' key for this room in /initialSync.", - "type": "array", - "items": { - "$ref": "Event" - }, - "nickname": "get_state_events", - "parameters": [ - { - "name": "roomId", - "description": "The room to get a list of current state events from.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/typing/{userId}", - "operations": [ - { - "method": "PUT", - "summary": "Inform the server that this user is typing.", - "notes": "Only the authorised user can send typing notifications.", - "type": "void", - "nickname": "put_typing", - "parameters": [ - { - "name": "roomId", - "description": "The room to send the m.typing event into.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "userId", - "description": "The user ID of the typer.", - "required": true, - "type": "string", - "paramType": "path" - }, - { - "name": "body", - "description": "The typing information.", - "required": true, - "type": "Typing", - "paramType": "body" - } - ] - } - ] - }, - { - "path": "/rooms/{roomId}/initialSync", - "operations": [ - { - "method": "GET", - "summary": "Get all the current information for this room, including messages and state events.", - "type": "InitialSyncRoomData", - "nickname": "get_room_sync_data", - "parameters": [ - { - "name": "roomId", - "description": "The room to get information for.", - "required": true, - "type": "string", - "paramType": "path" - } - ] - } - ] - } - ], - "models": { - "Topic": { - "id": "Topic", - "properties": { - "topic": { - "type": "string", - "description": "The topic text" - } - } - }, - "RoomName": { - "id": "RoomName", - "properties": { - "name": { - "type": "string", - "description": "The human-readable name for the room. Can contain spaces." - } - } - }, - "UserPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__user_id__": { - "type": "integer", - "description": "The power level for __user_id__" - } - } - }, - "EventPowerLevels": { - "id": "UserPowerLevels", - "properties": { - "__event_type__": { - "type": "integer", - "description": "The power level required in order to send __event_type__ events." - } - } - }, - "RoomPowerLevels": { - "id": "RoomPowerLevels", - "properties": { - "ban": { - "type": "integer", - "description": "The minimum level required in order to ban someone." - }, - "kick": { - "type": "integer", - "description": "The minimum level required in order to kick someone." - }, - "redact": { - "type": "integer", - "description": "The minimum level required in order to redact a message." - }, - "users_default": { - "type": "integer", - "description": "The default power level of a user who is not in the 'users' list." - }, - "state_default": { - "type": "integer", - "description": "The default power level required in order to send state events." - }, - "events_default": { - "type": "integer", - "description": "The default power level required in order to send non-state events." - }, - "users": { - "type": "UserPowerLevels", - "description": "Mappings of user ID to power level." - }, - "events": { - "type": "EventPowerLevels", - "description": "Mappings of event type to power level." - } - } - }, - "Message": { - "id": "Message", - "properties": { - "msgtype": { - "type": "string", - "description": "The type of message being sent, e.g. \"m.text\"", - "required": true - }, - "_msgtype_defined_keys_": { - "description": "Additional keys as defined by the msgtype, e.g. \"body\"" - } - } - }, - "Feedback": { - "id": "Feedback", - "properties": { - "target_event_id": { - "type": "string", - "description": "The event ID being acknowledged.", - "required": true - }, - "type": { - "type": "string", - "description": "The type of feedback. Either 'delivered' or 'read'.", - "required": true - } - } - }, - "Member": { - "id": "Member", - "properties": { - "membership": { - "type": "string", - "description": "Enum: The membership state of this member.", - "enum": [ - "invite", - "join", - "leave", - "ban" - ] - } - } - }, - "RoomInfo": { - "id": "RoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The allocated room ID.", - "required": true - }, - "room_alias": { - "type": "string", - "description": "The alias for the room.", - "required": false - } - } - }, - "JoinRoomInfo": { - "id": "JoinRoomInfo", - "properties": { - "room_id": { - "type": "string", - "description": "The room ID joined, if joined via a room alias only.", - "required": true - } - } - }, - "Typing": { - "id": "Typing", - "properties": { - "typing": { - "type": "boolean", - "description": "True if the user is currently typing.", - "required": true - }, - "timeout": { - "type": "integer", - "description": "The length of time until the user should be treated as no longer typing, in milliseconds. Can be omitted if they are no longer typing.", - "required": true - } - } - }, - "RoomConfig": { - "id": "RoomConfig", - "properties": { - "visibility": { - "type": "string", - "description": "Enum: The room visibility. The room_alias_name is required if the visibility is public; without it the room will remain private.", - "required": false, - "enum": [ - "public", - "private" - ] - }, - "room_alias_name": { - "type": "string", - "description": "Localpart of the alias to give the new room. The home server will attach its domain name after this.", - "required": false - }, - "name": { - "type": "string", - "description": "Sets the name of the room. Send a m.room.name event after creating the room with the 'name' key specified.", - "required": false - }, - "topic": { - "type": "string", - "description": "Sets the topic for the room. Send a m.room.topic event after creating the room with the 'topic' key specified.", - "required": false - }, - "invite": { - "type": "array", - "description": "The list of user IDs to invite. Sends m.room.member events after creating the room.", - "items": { - "$ref": "string" - }, - "required": false - } - } - }, - "PaginationRequest": { - "id": "PaginationRequest", - "properties": { - "from": { - "type": "string", - "description": "The token to start getting results from." - }, - "to": { - "type": "string", - "description": "The token to stop getting results at." - }, - "limit": { - "type": "integer", - "description": "The maximum number of entries to return." - } - } - }, - "PaginationChunk": { - "id": "PaginationChunk", - "properties": { - "start": { - "type": "string", - "description": "A token which correlates to the first value in \"chunk\" for paginating.", - "required": true - }, - "end": { - "type": "string", - "description": "A token which correlates to the last value in \"chunk\" for paginating.", - "required": true - } - }, - "subTypes": [ - "MessagePaginationChunk" - ] - }, - "MessagePaginationChunk": { - "id": "MessagePaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of message events.", - "items": { - "$ref": "MessageEvent" - }, - "required": true - } - } - }, - "MemberPaginationChunk": { - "id": "MemberPaginationChunk", - "properties": { - "chunk": { - "type": "array", - "description": "A list of member events.", - "items": { - "$ref": "MemberEvent" - }, - "required": true - } - } - }, - "Event": { - "id": "Event", - "properties": { - "event_id": { - "type": "string", - "description": "An ID which uniquely identifies this event. This is automatically set by the server.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The room in which this event occurred. This is automatically set by the server.", - "required": true - }, - "type": { - "type": "string", - "description": "The event type.", - "required": true - } - }, - "subTypes": [ - "MessageEvent" - ] - }, - "EventId": { - "id": "EventId", - "properties": { - "event_id": { - "type": "string", - "description": "The allocated event ID for this event.", - "required": true - } - } - }, - "EventContent": { - "id": "EventContent", - "properties": { - "__event_content_keys__": { - "type": "string", - "description": "Event-specific content keys and values.", - "required": false - } - } - }, - "MessageEvent": { - "id": "MessageEvent", - "properties": { - "content": { - "type": "Message" - } - } - }, - "MemberEvent": { - "id": "MemberEvent", - "properties": { - "content": { - "type": "Member" - } - } - }, - "InviteRequest": { - "id": "InviteRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - } - } - }, - "JoinRequest": { - "id": "JoinRequest", - "properties": {} - }, - "LeaveRequest": { - "id": "LeaveRequest", - "properties": {} - }, - "BanRequest": { - "id": "BanRequest", - "properties": { - "user_id": { - "type": "string", - "description": "The fully-qualified user ID." - }, - "reason": { - "type": "string", - "description": "The reason for the ban." - } - } - }, - "InitialSyncRoomData": { - "id": "InitialSyncRoomData", - "properties": { - "membership": { - "type": "string", - "description": "This user's membership state in this room.", - "required": true - }, - "room_id": { - "type": "string", - "description": "The ID of this room.", - "required": true - }, - "messages": { - "type": "MessagePaginationChunk", - "description": "The most recent messages for this room, governed by the limit parameter.", - "required": false - }, - "state": { - "type": "array", - "description": "A list of state events representing the current state of the room.", - "required": false, - "items": { - "$ref": "Event" - } - }, - "presence": { - "type": "array", - "description": "A list of m.presence events representing the current presence state of the room members.", - "required": false, - "items": { - "$ref": "Event" - } - } - } - } - } -} diff --git a/scripts/add-matrix-org-stylings.sh b/scripts/add-matrix-org-stylings.sh index ef4e1014a..9a9256b2f 100755 --- a/scripts/add-matrix-org-stylings.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -16,9 +16,11 @@ for f in "$1"/{head,nav,footer}.html; do fi done +files=gen/*.html + perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header -#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s##
@@ -27,7 +29,7 @@ perl -MFile::Slurp -pi -e 'BEGIN { $nav = read_file("'$NAV_BAR'") } s## <
-#' gen/specification.html gen/howtos.html +#' ${files} perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s##
@@ -39,4 +41,4 @@ perl -MFile::Slurp -pi -e 'BEGIN { $footer = read_file("'$FOOTER'") } s## $footer
- #' gen/specification.html gen/howtos.html + #' ${files} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh new file mode 100755 index 000000000..0a2bc5f69 --- /dev/null +++ b/scripts/generate-http-docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu + +# This script generates an HTML page containing all of the client-server API docs. +# It takes all of the swagger YAML files for the client-server API, and turns +# them into API docs, with none of the narrative found in the rst files which +# normally wrap these API docs. + +cd "$(dirname $0)" + +mkdir -p tmp gen + +cat >tmp/http_apis <> tmp/http_apis +done + +(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis) +rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html From 7f2813354daecf30bf1a14fa715f88a3511ed5b6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 15:42:21 +0000 Subject: [PATCH 102/111] Swaggerify /ban --- api/client-server/v1/banning.yaml | 76 +++++++++++++++++++++++++++++ specification/client_server_api.rst | 2 + 2 files changed, 78 insertions(+) create mode 100644 api/client-server/v1/banning.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/v1/banning.yaml new file mode 100644 index 000000000..e841050f9 --- /dev/null +++ b/api/client-server/v1/banning.yaml @@ -0,0 +1,76 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Banning API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/ban": + post: + summary: Ban a user in the room. + description: |- + Ban a user in the room. If the user is currently in the room, also kick them. + + When a user is banned from a room, they may not join it until they are unbanned. + + The caller must have the required power level in order to perform this operation. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) from which the user should be banned. + required: true + x-example: "!e42d8c:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "reason": "Telling unfunny jokes", + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the user being banned. + reason: + type: string + description: The reason the user has been banned. + required: ["user_id"] + responses: + 200: + description: The user has been kicked and banned from the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to ban the user from the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The banner is not currently in the room. + - The banner's power level is insufficient to ban users from the room. + examples: + application/json: |- + { + "errcode": "M_FORBIDDEN", + "error": "You do not have a high enough power level to ban from this room." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 402ce7531..58d0f1b20 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -983,6 +983,8 @@ following values: {{joining_http_api}} +{{banning_http_api}} + Leaving rooms ~~~~~~~~~~~~~ A user can leave a room to stop receiving events for that room. A user must From 3951785f19b2b6c38980bcf2748479ce4ba5ef86 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 16:23:31 +0000 Subject: [PATCH 103/111] Fix alias path --- api/client-server/v1/joining.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml index 20fdbd654..b6d2df18e 100644 --- a/api/client-server/v1/joining.yaml +++ b/api/client-server/v1/joining.yaml @@ -65,4 +65,4 @@ paths: x-alias: canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" aliases: - - /join/{roomId} + - /_matrix/client/api/v1/join/{roomId} From e171acf01ffecc4a931712928ffca024cbdade8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:21:32 +0000 Subject: [PATCH 104/111] Split spec into page-per-section --- scripts/gendoc.py | 52 +++++++++++++------ specification/intro.rst | 10 ++++ .../modules/typing_notifications.rst | 6 +-- specification/targets.yaml | 11 +++- 4 files changed, 57 insertions(+), 22 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 28a115284..9b87cb30e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -276,6 +276,12 @@ def run_through_template(input, set_verbose): raise +def get_build_targets(targets_listing): + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + return all_targets["targets"].keys() + + """ Extract and resolve groups for the given target in the given targets listing. Args: @@ -386,21 +392,34 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates): prepare_env() - log("Building spec [target=%s]" % target_name) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst", VERBOSE) - fix_relative_titles( - target=target, filename="tmp/templated_spec.rst", - out_filename="tmp/full_spec.rst" - ) - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this - rst2html("tmp/full_spec.rst", "gen/specification.html") - addAnchors("gen/specification.html") - rst2html("tmp/howto.rst", "gen/howtos.html") + log("Building spec [target=%s]" % requested_target_name) + + targets = [requested_target_name] + if requested_target_name == "all": + targets = get_build_targets("../specification/targets.yaml") + + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + rst_file = "tmp/spec_%s.rst" % (target_name,) + html_file = "gen/%s.html" % (target_name,) + + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + run_through_template(templated_file, VERBOSE) + fix_relative_titles( + target=target, filename=templated_file, + out_filename=rst_file, + ) + rst2html(rst_file, html_file) + addAnchors(html_file) + + if requested_target_name == "all": + shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + rst2html("tmp/howto.rst", "gen/howtos.html") + if not keep_intermediates: cleanup_env() @@ -414,8 +433,9 @@ if __name__ == '__main__': help="Do not delete intermediate files. They will be found in tmp/" ) parser.add_argument( - "--target", "-t", default="main", - help="Specify the build target to build from specification/targets.yaml" + "--target", "-t", default="all", + help="Specify the build target to build from specification/targets.yaml. " + + "The value 'all' will build all of the targets therein." ) parser.add_argument( "--verbose", "-v", action="store_true", diff --git a/specification/intro.rst b/specification/intro.rst index 6ff7946cf..8c08bf249 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -8,6 +8,16 @@ https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +APIs +~~~~ +The following APIs are documented in this specification: + +- `Client-Server API `_ for writing Matrix clients. +- `Server-Server API `_ for writing servers which can federate with Matrix. +- `Application Service API `_ for writing privileged plugins to servers. + +There are also some `appendices `_. + Changelog ~~~~~~~~~ {{spec_changelog}} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index d22536327..da383e731 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -5,10 +5,8 @@ Typing Notifications Users may wish to be informed when another user is typing in a room. This can be achieved using typing notifications. These are ephemeral events scoped to a -``room_id``. This means they do not form part of the `Event Graph`_ but still -have a ``room_id`` key. - -.. _Event Graph: `sect:event-graph`_ +``room_id``. This means they do not form part of the +`Event Graph `_ but still have a ``room_id`` key. Events ------ diff --git a/specification/targets.yaml b/specification/targets.yaml index 8e6a2ce00..cd0f6f413 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,16 +1,23 @@ targets: - main: # arbitrary name to identify this build target + index: files: # the sort order of files to cat - intro.rst + client_server: + files: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - modules.rst - { 1: feature_profiles.rst } - { 1: "group:modules" } # reference a group of files + application_service: + files: - application_service_api.rst + server_server: + files: - server_server_api.rst - - identity_servers.rst + appendices: + files: - appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: From 30ed918633cf47ceb2d5dac793b1e3a3e9a0d80f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:53:58 +0000 Subject: [PATCH 105/111] speculator: Allow spec viewing for multi-page spec --- scripts/speculator/main.go | 94 +++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index abe77c74d..3736672f4 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -57,8 +57,8 @@ var ( port = flag.Int("port", 9000, "Port on which to listen for HTTP") includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte - styledSpecCache *lru.Cache // string -> []byte + specCache *lru.Cache // string -> map[string][]byte filename -> contents + styledSpecCache *lru.Cache // string -> map[string][]byte filename -> contents ) func (u *User) IsTrusted() bool { @@ -105,10 +105,7 @@ func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) { if !strings.HasPrefix(url.Path, pathPrefix+"/") { return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) } - prNumber := url.Path[len(pathPrefix)+1:] - if strings.Contains(prNumber, "/") { - return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) - } + prNumber := strings.Split(url.Path[len(pathPrefix)+1:], "/")[0] resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) defer resp.Body.Close() @@ -196,6 +193,17 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } +func extractPath(path, base string) string { + // Assume exactly one flat directory + max := strings.Count(base, "/") + 2 + parts := strings.SplitN(path, "/", max) + + if len(parts) < max || parts[max-1] == "" { + return "index.html" + } + return parts[max-1] +} + func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string @@ -206,7 +214,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } - if strings.ToLower(req.URL.Path) == "/spec/head" { + if strings.HasPrefix(strings.ToLower(req.URL.Path), "/spec/head") { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { @@ -236,36 +244,59 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache = styledSpecCache } + var pathToContent map[string][]byte + if cached, ok := cache.Get(sha); ok { - w.Write(cached.([]byte)) - return - } + pathToContent = cached.(map[string][]byte) + } else { + dst, err := s.generateAt(sha) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, 500, err) + return + } - dst, err := s.generateAt(sha) - defer os.RemoveAll(dst) - if err != nil { - writeError(w, 500, err) - return - } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } - if styleLikeMatrixDotOrg { - cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) - cmd.Dir = path.Join(dst, "scripts") - var b bytes.Buffer - cmd.Stderr = &b - if err := cmd.Run(); err != nil { - writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) - return + fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen")) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading directory: %v", err)) + } + pathToContent = make(map[string][]byte) + for _, fi := range fis { + b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name())) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + return + } + pathToContent[fi.Name()] = b } + cache.Add(sha, pathToContent) } - b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) - if err != nil { - writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + requestedPath := extractPath(req.URL.Path, "/spec/pr") + if b, ok := pathToContent[requestedPath]; ok { + w.Write(b) return } - w.Write(b) - cache.Add(sha, b) + if requestedPath == "index.html" { + // Fall back to single-page spec for old PRs + if b, ok := pathToContent["specification.html"]; ok { + w.Write(b) + return + } + } + w.WriteHeader(404) + w.Write([]byte("Not found")) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -322,7 +353,7 @@ func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) { return } - diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + diffCmd := exec.Command("diff", "-r", "-u", path.Join(base, "scripts", "tmp"), path.Join(head, "scripts", "tmp")) var diff bytes.Buffer diffCmd.Stdout = &diff if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { @@ -366,7 +397,8 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html")) + requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b if err := cmd.Run(); err != nil { From b479b54cd887d0fd49a33458a77a51adaffc45bf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 18:52:58 +0000 Subject: [PATCH 106/111] Add tables of contents to individual specs --- specification/application_service_api.rst | 3 +++ specification/client_server_api.rst | 3 +++ specification/server_server_api.rst | 3 +++ 3 files changed, 9 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index cf2f9d57f..bdf8a52ea 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -12,6 +12,9 @@ irrespective of the underlying homeserver implementation. Add in Client-Server services? Overview of bots? Seems weird to be in the spec given it is VERY implementation specific. +.. contents:: Table of Contents +.. sectnum:: + Application Services -------------------- Application services are passive and can only observe events from a given diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 36ed2290c..134c12c35 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -17,6 +17,9 @@ shortly. Documentation for the old `V1 authentication <../attic/v1_registration_login.rst>`_ is still available separately. +.. contents:: Table of Contents +.. sectnum:: + API Standards ------------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index cc9426e2b..012c2ac96 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -41,6 +41,9 @@ EDUs and PDUs are further wrapped in an envelope called a Transaction, which is transferred from the origin to the destination home server using an HTTPS PUT request. +.. contents:: Table of Contents +.. sectnum:: + Server Discovery ---------------- From 6c66bfc75589392b61d67e1a37c3f11b66a5cf7e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 11:22:57 +0000 Subject: [PATCH 107/111] Flatten out v1 and v2_alpha directories As a side effect, I got rid of all of the horrible symlinks and just put in all of the proper relative paths. Because the horrible symlinks were horrible. --- .../{v2_alpha => }/account-data.yaml | 0 .../{v1 => }/application_service.yaml | 0 api/client-server/{v1 => }/banning.yaml | 0 api/client-server/{v1 => }/content-repo.yaml | 0 api/client-server/{v1 => }/create_room.yaml | 0 .../{v1 => }/definitions/error.yaml | 0 .../{v2_alpha => }/definitions/event.json | 0 .../definitions/event_batch.json | 0 .../definitions/event_filter.json | 0 .../{v1 => }/definitions/push_condition.json | 0 .../{v1 => }/definitions/push_rule.json | 0 .../{v1 => }/definitions/push_ruleset.json | 0 .../definitions/room_event_filter.json | 2 +- .../definitions/sync_filter.json | 8 ++-- .../definitions/timeline_batch.json | 2 +- api/client-server/{v1 => }/directory.yaml | 0 api/client-server/{v2_alpha => }/filter.yaml | 0 api/client-server/{v1 => }/guest_events.yaml | 2 +- api/client-server/{v1 => }/inviting.yaml | 0 api/client-server/{v1 => }/joining.yaml | 0 api/client-server/{v1 => }/leaving.yaml | 0 .../{v1 => }/list_public_rooms.yaml | 0 api/client-server/{v1 => }/login.yaml | 0 .../{v1 => }/message_pagination.yaml | 0 .../{v1/sync.yaml => old_sync.yaml} | 14 +++--- api/client-server/{v1 => }/presence.yaml | 2 +- api/client-server/{v1 => }/profile.yaml | 0 api/client-server/{v1 => }/push_notifier.yaml | 0 api/client-server/{v1 => }/pusher.yaml | 0 api/client-server/{v1 => }/pushrules.yaml | 0 .../{v2_alpha => }/receipts.yaml | 0 .../{v2_alpha => }/registration.yaml | 0 api/client-server/{v1 => }/room_send.yaml | 0 api/client-server/{v1 => }/room_state.yaml | 0 api/client-server/{v1 => }/rooms.yaml | 10 ++--- api/client-server/{v1 => }/search.yaml | 2 +- api/client-server/{v2_alpha => }/sync.yaml | 0 api/client-server/{v2_alpha => }/tags.yaml | 0 .../{v1 => }/third_party_membership.yaml | 0 api/client-server/{v1 => }/typing.yaml | 0 api/client-server/v1/core-event-schema | 1 - api/client-server/v1/v1-event-schema | 1 - .../v2_alpha/definitions/definitions | 1 - .../v2_alpha/definitions/error.yaml | 10 ----- api/client-server/{v1 => }/voip.yaml | 0 event-schemas/README.md | 2 +- event-schemas/check.sh | 6 +-- event-schemas/examples/{v1 => }/m.call.answer | 0 .../examples/{v1 => }/m.call.candidates | 0 event-schemas/examples/{v1 => }/m.call.hangup | 0 event-schemas/examples/{v1 => }/m.call.invite | 0 event-schemas/examples/{v1 => }/m.presence | 0 event-schemas/examples/{v1 => }/m.receipt | 0 .../examples/{v1 => }/m.room.aliases | 0 event-schemas/examples/{v1 => }/m.room.avatar | 0 .../examples/{v1 => }/m.room.canonical_alias | 0 event-schemas/examples/{v1 => }/m.room.create | 0 .../examples/{v1 => }/m.room.guest_access | 0 .../{v1 => }/m.room.history_visibility | 0 .../examples/{v1 => }/m.room.join_rules | 0 event-schemas/examples/{v1 => }/m.room.member | 0 .../{v1 => }/m.room.member#invite_room_state | 0 .../{v1 => }/m.room.member#third_party_invite | 0 .../examples/{v1 => }/m.room.message#m.audio | 0 .../examples/{v1 => }/m.room.message#m.emote | 0 .../examples/{v1 => }/m.room.message#m.file | 0 .../examples/{v1 => }/m.room.message#m.image | 0 .../{v1 => }/m.room.message#m.location | 0 .../examples/{v1 => }/m.room.message#m.notice | 0 .../examples/{v1 => }/m.room.message#m.text | 0 .../examples/{v1 => }/m.room.message#m.video | 0 .../examples/{v1 => }/m.room.message.feedback | 0 event-schemas/examples/{v1 => }/m.room.name | 0 .../examples/{v1 => }/m.room.power_levels | 0 .../examples/{v1 => }/m.room.redaction | 0 .../{v1 => }/m.room.third_party_invite | 0 event-schemas/examples/{v1 => }/m.room.topic | 0 event-schemas/examples/{v1 => }/m.tag | 0 event-schemas/examples/{v1 => }/m.typing | 0 .../{v1 => }/core-event-schema/event.json | 0 .../msgtype_infos/image_info.json | 0 .../core-event-schema/room_event.json | 2 +- .../core-event-schema/state_event.json | 2 +- event-schemas/schema/{v1 => }/m.call.answer | 0 .../schema/{v1 => }/m.call.candidates | 0 event-schemas/schema/{v1 => }/m.call.hangup | 0 event-schemas/schema/{v1 => }/m.call.invite | 0 event-schemas/schema/{v1 => }/m.presence | 0 event-schemas/schema/{v1 => }/m.receipt | 0 event-schemas/schema/{v1 => }/m.room.aliases | 0 event-schemas/schema/{v1 => }/m.room.avatar | 0 .../schema/{v1 => }/m.room.canonical_alias | 0 event-schemas/schema/{v1 => }/m.room.create | 0 .../schema/{v1 => }/m.room.guest_access | 0 .../schema/{v1 => }/m.room.history_visibility | 0 .../schema/{v1 => }/m.room.join_rules | 0 event-schemas/schema/{v1 => }/m.room.member | 0 event-schemas/schema/{v1 => }/m.room.message | 0 .../schema/{v1 => }/m.room.message#m.audio | 0 .../schema/{v1 => }/m.room.message#m.emote | 0 .../schema/{v1 => }/m.room.message#m.file | 0 .../schema/{v1 => }/m.room.message#m.image | 0 .../schema/{v1 => }/m.room.message#m.location | 0 .../schema/{v1 => }/m.room.message#m.notice | 0 .../schema/{v1 => }/m.room.message#m.text | 0 .../schema/{v1 => }/m.room.message#m.video | 0 .../schema/{v1 => }/m.room.message.feedback | 0 event-schemas/schema/{v1 => }/m.room.name | 0 .../schema/{v1 => }/m.room.power_levels | 0 .../schema/{v1 => }/m.room.redaction | 0 .../schema/{v1 => }/m.room.third_party_invite | 0 event-schemas/schema/{v1 => }/m.room.topic | 0 event-schemas/schema/{v1 => }/m.tag | 0 event-schemas/schema/{v1 => }/m.typing | 0 .../v1/core-event-schema/core-event-schema | 1 - event-schemas/schema/v1/v1-event-schema | 1 - jenkins.sh | 2 +- scripts/generate-http-docs.sh | 2 +- specification/client_server_api.rst | 6 +-- specification/modules/account_data.rst | 2 +- specification/modules/receipts.rst | 2 +- specification/modules/tags.rst | 2 +- templating/matrix_templates/units.py | 44 +++++++------------ 123 files changed, 52 insertions(+), 77 deletions(-) rename api/client-server/{v2_alpha => }/account-data.yaml (100%) rename api/client-server/{v1 => }/application_service.yaml (100%) rename api/client-server/{v1 => }/banning.yaml (100%) rename api/client-server/{v1 => }/content-repo.yaml (100%) rename api/client-server/{v1 => }/create_room.yaml (100%) rename api/client-server/{v1 => }/definitions/error.yaml (100%) rename api/client-server/{v2_alpha => }/definitions/event.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_batch.json (100%) rename api/client-server/{v2_alpha => }/definitions/event_filter.json (100%) rename api/client-server/{v1 => }/definitions/push_condition.json (100%) rename api/client-server/{v1 => }/definitions/push_rule.json (100%) rename api/client-server/{v1 => }/definitions/push_ruleset.json (100%) rename api/client-server/{v2_alpha => }/definitions/room_event_filter.json (92%) rename api/client-server/{v2_alpha => }/definitions/sync_filter.json (84%) rename api/client-server/{v2_alpha => }/definitions/timeline_batch.json (87%) rename api/client-server/{v1 => }/directory.yaml (100%) rename api/client-server/{v2_alpha => }/filter.yaml (100%) rename api/client-server/{v1 => }/guest_events.yaml (97%) rename api/client-server/{v1 => }/inviting.yaml (100%) rename api/client-server/{v1 => }/joining.yaml (100%) rename api/client-server/{v1 => }/leaving.yaml (100%) rename api/client-server/{v1 => }/list_public_rooms.yaml (100%) rename api/client-server/{v1 => }/login.yaml (100%) rename api/client-server/{v1 => }/message_pagination.yaml (100%) rename api/client-server/{v1/sync.yaml => old_sync.yaml} (96%) rename api/client-server/{v1 => }/presence.yaml (98%) rename api/client-server/{v1 => }/profile.yaml (100%) rename api/client-server/{v1 => }/push_notifier.yaml (100%) rename api/client-server/{v1 => }/pusher.yaml (100%) rename api/client-server/{v1 => }/pushrules.yaml (100%) rename api/client-server/{v2_alpha => }/receipts.yaml (100%) rename api/client-server/{v2_alpha => }/registration.yaml (100%) rename api/client-server/{v1 => }/room_send.yaml (100%) rename api/client-server/{v1 => }/room_state.yaml (100%) rename api/client-server/{v1 => }/rooms.yaml (97%) rename api/client-server/{v1 => }/search.yaml (97%) rename api/client-server/{v2_alpha => }/sync.yaml (100%) rename api/client-server/{v2_alpha => }/tags.yaml (100%) rename api/client-server/{v1 => }/third_party_membership.yaml (100%) rename api/client-server/{v1 => }/typing.yaml (100%) delete mode 120000 api/client-server/v1/core-event-schema delete mode 120000 api/client-server/v1/v1-event-schema delete mode 120000 api/client-server/v2_alpha/definitions/definitions delete mode 100644 api/client-server/v2_alpha/definitions/error.yaml rename api/client-server/{v1 => }/voip.yaml (100%) rename event-schemas/examples/{v1 => }/m.call.answer (100%) rename event-schemas/examples/{v1 => }/m.call.candidates (100%) rename event-schemas/examples/{v1 => }/m.call.hangup (100%) rename event-schemas/examples/{v1 => }/m.call.invite (100%) rename event-schemas/examples/{v1 => }/m.presence (100%) rename event-schemas/examples/{v1 => }/m.receipt (100%) rename event-schemas/examples/{v1 => }/m.room.aliases (100%) rename event-schemas/examples/{v1 => }/m.room.avatar (100%) rename event-schemas/examples/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/examples/{v1 => }/m.room.create (100%) rename event-schemas/examples/{v1 => }/m.room.guest_access (100%) rename event-schemas/examples/{v1 => }/m.room.history_visibility (100%) rename event-schemas/examples/{v1 => }/m.room.join_rules (100%) rename event-schemas/examples/{v1 => }/m.room.member (100%) rename event-schemas/examples/{v1 => }/m.room.member#invite_room_state (100%) rename event-schemas/examples/{v1 => }/m.room.member#third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.file (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.image (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.location (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.text (100%) rename event-schemas/examples/{v1 => }/m.room.message#m.video (100%) rename event-schemas/examples/{v1 => }/m.room.message.feedback (100%) rename event-schemas/examples/{v1 => }/m.room.name (100%) rename event-schemas/examples/{v1 => }/m.room.power_levels (100%) rename event-schemas/examples/{v1 => }/m.room.redaction (100%) rename event-schemas/examples/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/examples/{v1 => }/m.room.topic (100%) rename event-schemas/examples/{v1 => }/m.tag (100%) rename event-schemas/examples/{v1 => }/m.typing (100%) rename event-schemas/schema/{v1 => }/core-event-schema/event.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/msgtype_infos/image_info.json (100%) rename event-schemas/schema/{v1 => }/core-event-schema/room_event.json (93%) rename event-schemas/schema/{v1 => }/core-event-schema/state_event.json (93%) rename event-schemas/schema/{v1 => }/m.call.answer (100%) rename event-schemas/schema/{v1 => }/m.call.candidates (100%) rename event-schemas/schema/{v1 => }/m.call.hangup (100%) rename event-schemas/schema/{v1 => }/m.call.invite (100%) rename event-schemas/schema/{v1 => }/m.presence (100%) rename event-schemas/schema/{v1 => }/m.receipt (100%) rename event-schemas/schema/{v1 => }/m.room.aliases (100%) rename event-schemas/schema/{v1 => }/m.room.avatar (100%) rename event-schemas/schema/{v1 => }/m.room.canonical_alias (100%) rename event-schemas/schema/{v1 => }/m.room.create (100%) rename event-schemas/schema/{v1 => }/m.room.guest_access (100%) rename event-schemas/schema/{v1 => }/m.room.history_visibility (100%) rename event-schemas/schema/{v1 => }/m.room.join_rules (100%) rename event-schemas/schema/{v1 => }/m.room.member (100%) rename event-schemas/schema/{v1 => }/m.room.message (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.audio (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.emote (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.file (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.image (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.location (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.notice (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.text (100%) rename event-schemas/schema/{v1 => }/m.room.message#m.video (100%) rename event-schemas/schema/{v1 => }/m.room.message.feedback (100%) rename event-schemas/schema/{v1 => }/m.room.name (100%) rename event-schemas/schema/{v1 => }/m.room.power_levels (100%) rename event-schemas/schema/{v1 => }/m.room.redaction (100%) rename event-schemas/schema/{v1 => }/m.room.third_party_invite (100%) rename event-schemas/schema/{v1 => }/m.room.topic (100%) rename event-schemas/schema/{v1 => }/m.tag (100%) rename event-schemas/schema/{v1 => }/m.typing (100%) delete mode 120000 event-schemas/schema/v1/core-event-schema/core-event-schema delete mode 120000 event-schemas/schema/v1/v1-event-schema diff --git a/api/client-server/v2_alpha/account-data.yaml b/api/client-server/account-data.yaml similarity index 100% rename from api/client-server/v2_alpha/account-data.yaml rename to api/client-server/account-data.yaml diff --git a/api/client-server/v1/application_service.yaml b/api/client-server/application_service.yaml similarity index 100% rename from api/client-server/v1/application_service.yaml rename to api/client-server/application_service.yaml diff --git a/api/client-server/v1/banning.yaml b/api/client-server/banning.yaml similarity index 100% rename from api/client-server/v1/banning.yaml rename to api/client-server/banning.yaml diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/content-repo.yaml similarity index 100% rename from api/client-server/v1/content-repo.yaml rename to api/client-server/content-repo.yaml diff --git a/api/client-server/v1/create_room.yaml b/api/client-server/create_room.yaml similarity index 100% rename from api/client-server/v1/create_room.yaml rename to api/client-server/create_room.yaml diff --git a/api/client-server/v1/definitions/error.yaml b/api/client-server/definitions/error.yaml similarity index 100% rename from api/client-server/v1/definitions/error.yaml rename to api/client-server/definitions/error.yaml diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/definitions/event.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event.json rename to api/client-server/definitions/event.json diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/definitions/event_batch.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_batch.json rename to api/client-server/definitions/event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/definitions/event_filter.json similarity index 100% rename from api/client-server/v2_alpha/definitions/event_filter.json rename to api/client-server/definitions/event_filter.json diff --git a/api/client-server/v1/definitions/push_condition.json b/api/client-server/definitions/push_condition.json similarity index 100% rename from api/client-server/v1/definitions/push_condition.json rename to api/client-server/definitions/push_condition.json diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/definitions/push_rule.json similarity index 100% rename from api/client-server/v1/definitions/push_rule.json rename to api/client-server/definitions/push_rule.json diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/definitions/push_ruleset.json similarity index 100% rename from api/client-server/v1/definitions/push_ruleset.json rename to api/client-server/definitions/push_ruleset.json diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/definitions/room_event_filter.json similarity index 92% rename from api/client-server/v2_alpha/definitions/room_event_filter.json rename to api/client-server/definitions/room_event_filter.json index 86375781c..02e5d0e01 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/definitions/room_event_filter.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref": "definitions/event_filter.json"}], + "allOf": [{"$ref": "event_filter.json"}], "properties": { "rooms": { "type": "array", diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/definitions/sync_filter.json similarity index 84% rename from api/client-server/v2_alpha/definitions/sync_filter.json rename to api/client-server/definitions/sync_filter.json index 0cd6a7986..defc318a6 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/definitions/sync_filter.json @@ -7,24 +7,24 @@ "state": { "description": "The state events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "timeline": { "description": "The message and state update events to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] }, "ephemeral": { "description": "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", - "allOf": [{"$ref": "definitions/room_event_filter.json"}] + "allOf": [{"$ref": "room_event_filter.json"}] } } }, "presence": { "description": "The presence updates to include.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "event_filter.json"}] }, "event_format": { "description": diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/definitions/timeline_batch.json similarity index 87% rename from api/client-server/v2_alpha/definitions/timeline_batch.json rename to api/client-server/definitions/timeline_batch.json index f27a5746c..6f7e714a6 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/event_batch.json"}], + "allOf": [{"$ref":"event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v1/directory.yaml b/api/client-server/directory.yaml similarity index 100% rename from api/client-server/v1/directory.yaml rename to api/client-server/directory.yaml diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/filter.yaml similarity index 100% rename from api/client-server/v2_alpha/filter.yaml rename to api/client-server/filter.yaml diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/guest_events.yaml similarity index 97% rename from api/client-server/v1/guest_events.yaml rename to api/client-server/guest_events.yaml index bbb5799af..671b355af 100644 --- a/api/client-server/v1/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -98,6 +98,6 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." diff --git a/api/client-server/v1/inviting.yaml b/api/client-server/inviting.yaml similarity index 100% rename from api/client-server/v1/inviting.yaml rename to api/client-server/inviting.yaml diff --git a/api/client-server/v1/joining.yaml b/api/client-server/joining.yaml similarity index 100% rename from api/client-server/v1/joining.yaml rename to api/client-server/joining.yaml diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/leaving.yaml similarity index 100% rename from api/client-server/v1/leaving.yaml rename to api/client-server/leaving.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml similarity index 100% rename from api/client-server/v1/list_public_rooms.yaml rename to api/client-server/list_public_rooms.yaml diff --git a/api/client-server/v1/login.yaml b/api/client-server/login.yaml similarity index 100% rename from api/client-server/v1/login.yaml rename to api/client-server/login.yaml diff --git a/api/client-server/v1/message_pagination.yaml b/api/client-server/message_pagination.yaml similarity index 100% rename from api/client-server/v1/message_pagination.yaml rename to api/client-server/message_pagination.yaml diff --git a/api/client-server/v1/sync.yaml b/api/client-server/old_sync.yaml similarity index 96% rename from api/client-server/v1/sync.yaml rename to api/client-server/old_sync.yaml index 58cd55443..6dbe7e5ea 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/old_sync.yaml @@ -84,7 +84,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -285,7 +285,7 @@ paths: type: object title: Event allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" rooms: type: array items: @@ -304,7 +304,7 @@ paths: title: "InviteEvent" description: "The invite event if ``membership`` is ``invite``" allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" messages: type: object title: PaginationChunk @@ -332,7 +332,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -345,7 +345,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -361,7 +361,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id", "membership"] required: ["end", "rooms", "presence"] 404: @@ -398,6 +398,6 @@ paths: } schema: allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" 404: description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/presence.yaml b/api/client-server/presence.yaml similarity index 98% rename from api/client-server/v1/presence.yaml rename to api/client-server/presence.yaml index 5684398b0..33df17d23 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/presence.yaml @@ -205,4 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" diff --git a/api/client-server/v1/profile.yaml b/api/client-server/profile.yaml similarity index 100% rename from api/client-server/v1/profile.yaml rename to api/client-server/profile.yaml diff --git a/api/client-server/v1/push_notifier.yaml b/api/client-server/push_notifier.yaml similarity index 100% rename from api/client-server/v1/push_notifier.yaml rename to api/client-server/push_notifier.yaml diff --git a/api/client-server/v1/pusher.yaml b/api/client-server/pusher.yaml similarity index 100% rename from api/client-server/v1/pusher.yaml rename to api/client-server/pusher.yaml diff --git a/api/client-server/v1/pushrules.yaml b/api/client-server/pushrules.yaml similarity index 100% rename from api/client-server/v1/pushrules.yaml rename to api/client-server/pushrules.yaml diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/receipts.yaml similarity index 100% rename from api/client-server/v2_alpha/receipts.yaml rename to api/client-server/receipts.yaml diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/registration.yaml similarity index 100% rename from api/client-server/v2_alpha/registration.yaml rename to api/client-server/registration.yaml diff --git a/api/client-server/v1/room_send.yaml b/api/client-server/room_send.yaml similarity index 100% rename from api/client-server/v1/room_send.yaml rename to api/client-server/room_send.yaml diff --git a/api/client-server/v1/room_state.yaml b/api/client-server/room_state.yaml similarity index 100% rename from api/client-server/v1/room_state.yaml rename to api/client-server/room_state.yaml diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/rooms.yaml similarity index 97% rename from api/client-server/v1/rooms.yaml rename to api/client-server/rooms.yaml index ff5587a90..f7654f145 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -173,7 +173,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" 403: description: > You aren't a member of the room and weren't previously a @@ -355,7 +355,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -368,7 +368,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "core-event-schema/state_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -383,7 +383,7 @@ paths: title: Event type: object allOf: - - "$ref": "core-event-schema/event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/event.json" required: ["room_id"] 403: description: > @@ -453,7 +453,7 @@ paths: title: MemberEvent type: object allOf: - - "$ref": "v1-event-schema/m.room.member" + - "$ref": "../../event-schemas/schema/m.room.member" 403: description: > You aren't a member of the room and weren't previously a diff --git a/api/client-server/v1/search.yaml b/api/client-server/search.yaml similarity index 97% rename from api/client-server/v1/search.yaml rename to api/client-server/search.yaml index cc2990564..3da9486f3 100644 --- a/api/client-server/v1/search.yaml +++ b/api/client-server/search.yaml @@ -111,7 +111,7 @@ paths: title: Event description: The event that matched. allOf: - - "$ref": "core-event-schema/room_event.json" + - "$ref": "../../event-schemas/schema/core-event-schema/room_event.json" examples: application/json: |- { diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/sync.yaml similarity index 100% rename from api/client-server/v2_alpha/sync.yaml rename to api/client-server/sync.yaml diff --git a/api/client-server/v2_alpha/tags.yaml b/api/client-server/tags.yaml similarity index 100% rename from api/client-server/v2_alpha/tags.yaml rename to api/client-server/tags.yaml diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/third_party_membership.yaml similarity index 100% rename from api/client-server/v1/third_party_membership.yaml rename to api/client-server/third_party_membership.yaml diff --git a/api/client-server/v1/typing.yaml b/api/client-server/typing.yaml similarity index 100% rename from api/client-server/v1/typing.yaml rename to api/client-server/typing.yaml diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema deleted file mode 120000 index 045aecb02..000000000 --- a/api/client-server/v1/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/v1-event-schema b/api/client-server/v1/v1-event-schema deleted file mode 120000 index 7a0d0326b..000000000 --- a/api/client-server/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1 \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions deleted file mode 120000 index 945c9b46d..000000000 --- a/api/client-server/v2_alpha/definitions/definitions +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/error.yaml b/api/client-server/v2_alpha/definitions/error.yaml deleted file mode 100644 index 20312ae4d..000000000 --- a/api/client-server/v2_alpha/definitions/error.yaml +++ /dev/null @@ -1,10 +0,0 @@ -type: object -description: A Matrix-level Error -properties: - errcode: - type: string - description: An error code. - error: - type: string - description: A human-readable error message. -required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/v1/voip.yaml b/api/client-server/voip.yaml similarity index 100% rename from api/client-server/v1/voip.yaml rename to api/client-server/voip.yaml diff --git a/event-schemas/README.md b/event-schemas/README.md index 1030a04c5..950c30181 100644 --- a/event-schemas/README.md +++ b/event-schemas/README.md @@ -7,7 +7,7 @@ resolved correctly. For basic CLI testing, we recommend and have verified they work with the Node.js package [z-schema](https://github.com/zaggino/z-schema): ``` $ npm install -g z-schema - $ z-schema schema/v1/m.room.message examples/v1/m.room.message_m.text + $ z-schema schema/m.room.message examples/m.room.message_m.text schema validation passed json #1 validation passed ``` diff --git a/event-schemas/check.sh b/event-schemas/check.sh index a6d03b5a4..3d411768c 100755 --- a/event-schemas/check.sh +++ b/event-schemas/check.sh @@ -6,17 +6,17 @@ if ! which z-schema; then exit 1 fi -find schema/v1/m.* | while read line +find schema/m.* | while read line do split_path=(${line///// }) event_type=(${split_path[2]}) echo "Checking $event_type" echo "--------------------" # match exact name or exact name with a # - find examples/v1 -name $event_type -o -name "$event_type#*" | while read exline + find examples -name $event_type -o -name "$event_type#*" | while read exline do echo " against $exline" # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/v1/$event_type $exline + z-schema schema/$event_type $exline done done diff --git a/event-schemas/examples/v1/m.call.answer b/event-schemas/examples/m.call.answer similarity index 100% rename from event-schemas/examples/v1/m.call.answer rename to event-schemas/examples/m.call.answer diff --git a/event-schemas/examples/v1/m.call.candidates b/event-schemas/examples/m.call.candidates similarity index 100% rename from event-schemas/examples/v1/m.call.candidates rename to event-schemas/examples/m.call.candidates diff --git a/event-schemas/examples/v1/m.call.hangup b/event-schemas/examples/m.call.hangup similarity index 100% rename from event-schemas/examples/v1/m.call.hangup rename to event-schemas/examples/m.call.hangup diff --git a/event-schemas/examples/v1/m.call.invite b/event-schemas/examples/m.call.invite similarity index 100% rename from event-schemas/examples/v1/m.call.invite rename to event-schemas/examples/m.call.invite diff --git a/event-schemas/examples/v1/m.presence b/event-schemas/examples/m.presence similarity index 100% rename from event-schemas/examples/v1/m.presence rename to event-schemas/examples/m.presence diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/m.receipt similarity index 100% rename from event-schemas/examples/v1/m.receipt rename to event-schemas/examples/m.receipt diff --git a/event-schemas/examples/v1/m.room.aliases b/event-schemas/examples/m.room.aliases similarity index 100% rename from event-schemas/examples/v1/m.room.aliases rename to event-schemas/examples/m.room.aliases diff --git a/event-schemas/examples/v1/m.room.avatar b/event-schemas/examples/m.room.avatar similarity index 100% rename from event-schemas/examples/v1/m.room.avatar rename to event-schemas/examples/m.room.avatar diff --git a/event-schemas/examples/v1/m.room.canonical_alias b/event-schemas/examples/m.room.canonical_alias similarity index 100% rename from event-schemas/examples/v1/m.room.canonical_alias rename to event-schemas/examples/m.room.canonical_alias diff --git a/event-schemas/examples/v1/m.room.create b/event-schemas/examples/m.room.create similarity index 100% rename from event-schemas/examples/v1/m.room.create rename to event-schemas/examples/m.room.create diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/m.room.guest_access similarity index 100% rename from event-schemas/examples/v1/m.room.guest_access rename to event-schemas/examples/m.room.guest_access diff --git a/event-schemas/examples/v1/m.room.history_visibility b/event-schemas/examples/m.room.history_visibility similarity index 100% rename from event-schemas/examples/v1/m.room.history_visibility rename to event-schemas/examples/m.room.history_visibility diff --git a/event-schemas/examples/v1/m.room.join_rules b/event-schemas/examples/m.room.join_rules similarity index 100% rename from event-schemas/examples/v1/m.room.join_rules rename to event-schemas/examples/m.room.join_rules diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/m.room.member similarity index 100% rename from event-schemas/examples/v1/m.room.member rename to event-schemas/examples/m.room.member diff --git a/event-schemas/examples/v1/m.room.member#invite_room_state b/event-schemas/examples/m.room.member#invite_room_state similarity index 100% rename from event-schemas/examples/v1/m.room.member#invite_room_state rename to event-schemas/examples/m.room.member#invite_room_state diff --git a/event-schemas/examples/v1/m.room.member#third_party_invite b/event-schemas/examples/m.room.member#third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.member#third_party_invite rename to event-schemas/examples/m.room.member#third_party_invite diff --git a/event-schemas/examples/v1/m.room.message#m.audio b/event-schemas/examples/m.room.message#m.audio similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.audio rename to event-schemas/examples/m.room.message#m.audio diff --git a/event-schemas/examples/v1/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.emote rename to event-schemas/examples/m.room.message#m.emote diff --git a/event-schemas/examples/v1/m.room.message#m.file b/event-schemas/examples/m.room.message#m.file similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.file rename to event-schemas/examples/m.room.message#m.file diff --git a/event-schemas/examples/v1/m.room.message#m.image b/event-schemas/examples/m.room.message#m.image similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.image rename to event-schemas/examples/m.room.message#m.image diff --git a/event-schemas/examples/v1/m.room.message#m.location b/event-schemas/examples/m.room.message#m.location similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.location rename to event-schemas/examples/m.room.message#m.location diff --git a/event-schemas/examples/v1/m.room.message#m.notice b/event-schemas/examples/m.room.message#m.notice similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.notice rename to event-schemas/examples/m.room.message#m.notice diff --git a/event-schemas/examples/v1/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.text rename to event-schemas/examples/m.room.message#m.text diff --git a/event-schemas/examples/v1/m.room.message#m.video b/event-schemas/examples/m.room.message#m.video similarity index 100% rename from event-schemas/examples/v1/m.room.message#m.video rename to event-schemas/examples/m.room.message#m.video diff --git a/event-schemas/examples/v1/m.room.message.feedback b/event-schemas/examples/m.room.message.feedback similarity index 100% rename from event-schemas/examples/v1/m.room.message.feedback rename to event-schemas/examples/m.room.message.feedback diff --git a/event-schemas/examples/v1/m.room.name b/event-schemas/examples/m.room.name similarity index 100% rename from event-schemas/examples/v1/m.room.name rename to event-schemas/examples/m.room.name diff --git a/event-schemas/examples/v1/m.room.power_levels b/event-schemas/examples/m.room.power_levels similarity index 100% rename from event-schemas/examples/v1/m.room.power_levels rename to event-schemas/examples/m.room.power_levels diff --git a/event-schemas/examples/v1/m.room.redaction b/event-schemas/examples/m.room.redaction similarity index 100% rename from event-schemas/examples/v1/m.room.redaction rename to event-schemas/examples/m.room.redaction diff --git a/event-schemas/examples/v1/m.room.third_party_invite b/event-schemas/examples/m.room.third_party_invite similarity index 100% rename from event-schemas/examples/v1/m.room.third_party_invite rename to event-schemas/examples/m.room.third_party_invite diff --git a/event-schemas/examples/v1/m.room.topic b/event-schemas/examples/m.room.topic similarity index 100% rename from event-schemas/examples/v1/m.room.topic rename to event-schemas/examples/m.room.topic diff --git a/event-schemas/examples/v1/m.tag b/event-schemas/examples/m.tag similarity index 100% rename from event-schemas/examples/v1/m.tag rename to event-schemas/examples/m.tag diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/m.typing similarity index 100% rename from event-schemas/examples/v1/m.typing rename to event-schemas/examples/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/core-event-schema/event.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/event.json rename to event-schemas/schema/core-event-schema/event.json diff --git a/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json b/event-schemas/schema/core-event-schema/msgtype_infos/image_info.json similarity index 100% rename from event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json rename to event-schemas/schema/core-event-schema/msgtype_infos/image_info.json diff --git a/event-schemas/schema/v1/core-event-schema/room_event.json b/event-schemas/schema/core-event-schema/room_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/room_event.json rename to event-schemas/schema/core-event-schema/room_event.json index d5413f8a0..80f7d2651 100644 --- a/event-schemas/schema/v1/core-event-schema/room_event.json +++ b/event-schemas/schema/core-event-schema/room_event.json @@ -3,7 +3,7 @@ "title": "Room Event", "description": "In addition to the Event fields, Room Events MUST have the following additional field.", "allOf":[{ - "$ref": "core-event-schema/event.json" + "$ref": "event.json" }], "properties": { "room_id": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/core-event-schema/state_event.json similarity index 93% rename from event-schemas/schema/v1/core-event-schema/state_event.json rename to event-schemas/schema/core-event-schema/state_event.json index 5809cf7f6..cfcb212c5 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/core-event-schema/state_event.json @@ -3,7 +3,7 @@ "title": "State Event", "description": "In addition to the Room Event fields, State Events have the following additional fields.", "allOf":[{ - "$ref": "core-event-schema/room_event.json" + "$ref": "room_event.json" }], "properties": { "state_key": { diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/m.call.answer similarity index 100% rename from event-schemas/schema/v1/m.call.answer rename to event-schemas/schema/m.call.answer diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/m.call.candidates similarity index 100% rename from event-schemas/schema/v1/m.call.candidates rename to event-schemas/schema/m.call.candidates diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/m.call.hangup similarity index 100% rename from event-schemas/schema/v1/m.call.hangup rename to event-schemas/schema/m.call.hangup diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/m.call.invite similarity index 100% rename from event-schemas/schema/v1/m.call.invite rename to event-schemas/schema/m.call.invite diff --git a/event-schemas/schema/v1/m.presence b/event-schemas/schema/m.presence similarity index 100% rename from event-schemas/schema/v1/m.presence rename to event-schemas/schema/m.presence diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/m.receipt similarity index 100% rename from event-schemas/schema/v1/m.receipt rename to event-schemas/schema/m.receipt diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/m.room.aliases similarity index 100% rename from event-schemas/schema/v1/m.room.aliases rename to event-schemas/schema/m.room.aliases diff --git a/event-schemas/schema/v1/m.room.avatar b/event-schemas/schema/m.room.avatar similarity index 100% rename from event-schemas/schema/v1/m.room.avatar rename to event-schemas/schema/m.room.avatar diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/m.room.canonical_alias similarity index 100% rename from event-schemas/schema/v1/m.room.canonical_alias rename to event-schemas/schema/m.room.canonical_alias diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/m.room.create similarity index 100% rename from event-schemas/schema/v1/m.room.create rename to event-schemas/schema/m.room.create diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/m.room.guest_access similarity index 100% rename from event-schemas/schema/v1/m.room.guest_access rename to event-schemas/schema/m.room.guest_access diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/m.room.history_visibility similarity index 100% rename from event-schemas/schema/v1/m.room.history_visibility rename to event-schemas/schema/m.room.history_visibility diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/m.room.join_rules similarity index 100% rename from event-schemas/schema/v1/m.room.join_rules rename to event-schemas/schema/m.room.join_rules diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/m.room.member similarity index 100% rename from event-schemas/schema/v1/m.room.member rename to event-schemas/schema/m.room.member diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/m.room.message similarity index 100% rename from event-schemas/schema/v1/m.room.message rename to event-schemas/schema/m.room.message diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/m.room.message#m.audio similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.audio rename to event-schemas/schema/m.room.message#m.audio diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/m.room.message#m.emote similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.emote rename to event-schemas/schema/m.room.message#m.emote diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/m.room.message#m.file similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.file rename to event-schemas/schema/m.room.message#m.file diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/m.room.message#m.image similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.image rename to event-schemas/schema/m.room.message#m.image diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/m.room.message#m.location similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.location rename to event-schemas/schema/m.room.message#m.location diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/m.room.message#m.notice similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.notice rename to event-schemas/schema/m.room.message#m.notice diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/m.room.message#m.text similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.text rename to event-schemas/schema/m.room.message#m.text diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/m.room.message#m.video similarity index 100% rename from event-schemas/schema/v1/m.room.message#m.video rename to event-schemas/schema/m.room.message#m.video diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/m.room.message.feedback similarity index 100% rename from event-schemas/schema/v1/m.room.message.feedback rename to event-schemas/schema/m.room.message.feedback diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/m.room.name similarity index 100% rename from event-schemas/schema/v1/m.room.name rename to event-schemas/schema/m.room.name diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/m.room.power_levels similarity index 100% rename from event-schemas/schema/v1/m.room.power_levels rename to event-schemas/schema/m.room.power_levels diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/m.room.redaction similarity index 100% rename from event-schemas/schema/v1/m.room.redaction rename to event-schemas/schema/m.room.redaction diff --git a/event-schemas/schema/v1/m.room.third_party_invite b/event-schemas/schema/m.room.third_party_invite similarity index 100% rename from event-schemas/schema/v1/m.room.third_party_invite rename to event-schemas/schema/m.room.third_party_invite diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/m.room.topic similarity index 100% rename from event-schemas/schema/v1/m.room.topic rename to event-schemas/schema/m.room.topic diff --git a/event-schemas/schema/v1/m.tag b/event-schemas/schema/m.tag similarity index 100% rename from event-schemas/schema/v1/m.tag rename to event-schemas/schema/m.tag diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/m.typing similarity index 100% rename from event-schemas/schema/v1/m.typing rename to event-schemas/schema/m.typing diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema deleted file mode 120000 index 945c9b46d..000000000 --- a/event-schemas/schema/v1/core-event-schema/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema deleted file mode 120000 index 945c9b46d..000000000 --- a/event-schemas/schema/v1/v1-event-schema +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/jenkins.sh b/jenkins.sh index f5ed3b15a..abafefa8e 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,7 +5,7 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) -(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") +(cd api && npm install && node validator.js -s "client-server") (cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} diff --git a/scripts/generate-http-docs.sh b/scripts/generate-http-docs.sh index 0a2bc5f69..b3894a2c2 100755 --- a/scripts/generate-http-docs.sh +++ b/scripts/generate-http-docs.sh @@ -18,7 +18,7 @@ This contains the client-server API for the reference implementation of the home EOF -for f in ../api/client-server/*/*.yaml; do +for f in ../api/client-server/*.yaml; do f="$(basename "${f/.yaml/_http_api}")" echo "{{${f/-/_}}}" >> tmp/http_apis done diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 134c12c35..f43daa7bd 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -468,7 +468,7 @@ This section refers to API Version 2. These API calls currently use the prefix .. _User-Interactive Authentication: `sect:auth-api`_ -{{v2_registration_http_api}} +{{registration_http_api}} Old V1 API docs: |register|_ @@ -769,9 +769,9 @@ When the client first logs in, they will need to initially synchronise with their home server. This is achieved via the initial sync API described below. This API also returns an ``end`` token which can be used with the event stream. -{{sync_http_api}} +{{old_sync_http_api}} -{{v2_sync_http_api}} +{{sync_http_api}} Getting events for a room diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 784bc874f..f3fc72b62 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -24,4 +24,4 @@ the tags are for. Client Behaviour ---------------- -{{v2_account_data_http_api}} +{{account_data_http_api}} diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a8ad3cd34..702a72753 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -52,7 +52,7 @@ dismissing a notification in order for the event to count as "read". A client can update the markers for its user by interacting with the following HTTP APIs. -{{v2_receipts_http_api}} +{{receipts_http_api}} Server behaviour ---------------- diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index bb467bc05..f8c28c554 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -45,4 +45,4 @@ Two special names are listed in the specification: Client Behaviour ---------------- -{{v2_tags_http_api}} +{{tags_http_api}} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 75a12a5ba..069940820 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -17,11 +17,10 @@ import subprocess import urllib import yaml -V1_CLIENT_API = "../api/client-server/v1" -V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" -V1_EVENT_SCHEMA = "../event-schemas/schema/v1" -V2_CLIENT_API = "../api/client-server/v2_alpha" -CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +HTTP_APIS = "../api/client-server" +V1_EVENT_EXAMPLES = "../event-schemas/examples" +V1_EVENT_SCHEMA = "../event-schemas/schema" +CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -549,30 +548,21 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - paths = [ - V1_CLIENT_API, V2_CLIENT_API - ] apis = {} - for path in paths: - is_v2 = (path == V2_CLIENT_API) - if not os.path.exists(V2_CLIENT_API): - self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) + path = HTTP_APIS + for filename in os.listdir(path): + if not filename.endswith(".yaml"): continue - for filename in os.listdir(path): - if not filename.endswith(".yaml"): - continue - self.log("Reading swagger API: %s" % filename) - filepath = os.path.join(path, filename) - with open(filepath, "r") as f: - # strip .yaml - group_name = filename[:-5].replace("-", "_") - if is_v2: - group_name = "v2_" + group_name - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta( - filepath, api, group_name - ) - apis[group_name] = api + self.log("Reading swagger API: %s" % filename) + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) + apis[group_name] = api return apis def load_common_event_fields(self): From 79244e8065e989e409177180a43e8ea312d32739 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 30 Nov 2015 16:40:48 +0000 Subject: [PATCH 108/111] Remove redundant node event schema checker The python one does the same --- event-schemas/check.sh | 22 ---------------------- event-schemas/check_examples.py | 2 +- jenkins.sh | 1 - 3 files changed, 1 insertion(+), 24 deletions(-) delete mode 100755 event-schemas/check.sh diff --git a/event-schemas/check.sh b/event-schemas/check.sh deleted file mode 100755 index 3d411768c..000000000 --- a/event-schemas/check.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# Runs z-schema over all of the schema files (looking for matching examples) - -if ! which z-schema; then - echo >&2 "Need to install z-schema; run: sudo npm install -g z-schema" - exit 1 -fi - -find schema/m.* | while read line -do - split_path=(${line///// }) - event_type=(${split_path[2]}) - echo "Checking $event_type" - echo "--------------------" - # match exact name or exact name with a # - find examples -name $event_type -o -name "$event_type#*" | while read exline - do - echo " against $exline" - # run z-schema: because of bash -e if this fails we bail with exit code 1 - z-schema schema/$event_type $exline - done -done diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index b10b2d0ef..5a4094071 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/env python import sys import json diff --git a/jenkins.sh b/jenkins.sh index abafefa8e..c58ad4734 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -6,7 +6,6 @@ set -ex (cd api && ./check_examples.py) (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server") -(cd event-schemas/ && ./check.sh) : ${GOPATH:=${WORKSPACE}/.gopath} mkdir -p "${GOPATH}" From c6e0322a9e557c8fafa7d6f1fc894f253e5a2cbc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 13:53:03 +0000 Subject: [PATCH 109/111] Swaggerify /account --- api/client-server/administrative_contact.yaml | 157 ++++++++++++++++++ specification/client_server_api.rst | 53 +----- 2 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 api/client-server/administrative_contact.yaml diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml new file mode 100644 index 000000000..68fd0fbae --- /dev/null +++ b/api/client-server/administrative_contact.yaml @@ -0,0 +1,157 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Account Administrative Contact API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/account/password": + post: + summary: Changes a user's password. + description: |- + This API endpoint uses the User-Interactive Authentication API. + An access token should be submitted to this endpoint if the client has + an active session. + The Home Server may change the flows available depending on whether a + valid access token is provided. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + example: |- + { + "new_password": "ihatebananas" + } + properties: + new_password: + type: string + description: The new password for the account. + required: ["new_password"] + responses: + 200: + description: The password has been changed. + examples: + application/json: "{}" + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/account/3pid": + get: + summary: Gets a list of a user's third party identifiers. + description: |- + Gets a list of the third party identifiers that the homeserver has + associated with the user's account. + + This is *not* the same as the list of third party identifiers bound to + the user's Matrix ID in Identity Servers. + + Identifiers in this list may be used by the Home Server as, for example, + identifiers that it will accept to reset the user's account password. + security: + - accessToken: [] + responses: + 200: + description: The lookup was successful. + examples: + application/json: |- + { + "threepids": [ + { + "medium": "email", + "address": "monkey@banana.island" + } + ] + } + schema: + type: object + properties: + threepids: + type: array + items: + type: object + title: Third party identifier + properties: + medium: + type: string + description: The medium of the third party identifier. + enum: ["email"] + address: + type: string + description: The third party identifier address. + post: + summary: Adds contact information to the user's account. + description: Adds contact information to the user's account. + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + properties: + threePidCreds: + title: "ThreePidCredentials" + type: object + description: The third party credentials to associate with the account. + properties: + client_secret: + type: string + description: The client secret used in the session with the Identity Server. + id_server: + type: string + description: The Identity Server to use. + sid: + type: string + description: The session identifier given by the Identity Server. + required: ["client_secret", "id_server", "sid"] + bind: + type: boolean + description: |- + Whether the home server should also bind this third party + identifier to the account's Matrix ID with the passed identity + server. Default: ``false``. + x-example: true + required: ["threePidCreds"] + example: |- + { + "threePidCreds": { + "id_server": "matrix.org", + "sid": "abc123987", + "client_secret": "d0n'tT3ll" + }, + "bind": false + } + responses: + 200: + description: The addition was successful. + examples: + application/json: "{}" + schema: + type: object + 403: + description: The credentials could not be verified with the identity server. + examples: + application/json: |- + { + "errcode": "M_THREEPID_AUTH_FAILED", + "error": "The third party credentials could not be verified by the identity server." + } diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index f43daa7bd..67973c6b3 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -510,58 +510,11 @@ database. Adding Account Administrative Contact Information +++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - POST $V2PREFIX/account/3pid - -Used to add contact information to the user's account. - -The body of the POST request is a JSON object containing: - -threePidCreds - An object containing contact information. -bind - Optional. A boolean indicating whether the Home Server should also bind this - third party identifier to the account's matrix ID with the Identity Server. If - supplied and true, the Home Server must bind the 3pid accordingly. - -The contact information object comprises: - -id_server - The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifier. If the port is the default, it and the - colon should be omitted. -sid - The session ID given by the Identity Server -client_secret - The client secret used in the session with the Identity Server. - -On success, the empty JSON object is returned. - -May also return error codes: - -M_THREEPID_AUTH_FAILED - If the credentials provided could not be verified with the ID Server. - -Fetching Currently Associated Contact Information -+++++++++++++++++++++++++++++++++++++++++++++++++ -Request:: - - GET $V2PREFIX/account/3pid - -This returns a list of third party identifiers that the Home Server has -associated with the user's account. This is *not* the same as the list of third -party identifiers bound to the user's Matrix ID in Identity Servers. Identifiers -in this list may be used by the Home Server as, for example, identifiers that it -will accept to reset the user's account password. -Returns a JSON object with the key ``threepids`` whose contents is an array of -objects with the following keys: +A homeserver may keep some contact information for administrative use. +This is independent of any information kept by any Identity Servers. -medium - The medium of the 3pid (eg, ``email``) -address - The textual address of the 3pid, eg. the email address +{{administrative_contact_http_api}} Pagination ---------- From c4eaf7458ff8489697aae8683256c2d8f4bcee80 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:19:31 +0000 Subject: [PATCH 110/111] 302 to spec/head/index.html rather than serving it on /spec/head Otherwise relative links are broken --- scripts/speculator/main.go | 46 +++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3736672f4..c32eec941 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -193,15 +193,31 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } -func extractPath(path, base string) string { - // Assume exactly one flat directory - max := strings.Count(base, "/") + 2 +// extractPath extracts the file path within the gen directory which should be served for the request. +// Returns one of (file to serve, path to redirect to). +// path is the actual path being requested, e.g. "/spec/head/client_server.html". +// base is the base path of the handler, including a trailing slash, before the PR number, e.g. "/spec/". +func extractPath(path, base string) (string, string) { + // Assumes exactly one flat directory + + // Count slashes in /spec/head/client_server.html + // base is /spec/ + // +1 for the PR number - /spec/head + // +1 for the path-part after the slash after the PR number + max := strings.Count(base, "/") + 1 parts := strings.SplitN(path, "/", max) - if len(parts) < max || parts[max-1] == "" { - return "index.html" + if len(parts) < max { + // Path is base/pr - redirect to base/pr/index.html + return "", path + "/index.html" } - return parts[max-1] + if parts[max-1] == "" { + // Path is base/pr/ - serve index.html + return "index.html", "" + } + + // Path is base/pr/file.html - serve file + return parts[max-1], "" } func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { @@ -283,7 +299,11 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache.Add(sha, pathToContent) } - requestedPath := extractPath(req.URL.Path, "/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } if b, ok := pathToContent[requestedPath]; ok { w.Write(b) return @@ -299,6 +319,12 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { w.Write([]byte("Not found")) } +func (s *server) redirectTo(w http.ResponseWriter, req *http.Request, path string) { + req.URL.Path = path + w.Header().Set("Location", req.URL.String()) + w.WriteHeader(302) +} + // lookupHeadSHA looks up what origin/master's HEAD SHA is. // It attempts to `git fetch` before doing so. // If this fails, it may still return a stale sha, but will also return an error. @@ -397,7 +423,11 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + requestedPath, redirect := extractPath(req.URL.Path, "/diff/spec/") + if redirect != "" { + s.redirectTo(w, req, redirect) + return + } cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b From a4668c1d8c69124e029b456bc6b7cf13dfac0385 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Dec 2015 14:21:20 +0000 Subject: [PATCH 111/111] Fix typo --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index c32eec941..602cdb577 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -204,7 +204,7 @@ func extractPath(path, base string) (string, string) { // base is /spec/ // +1 for the PR number - /spec/head // +1 for the path-part after the slash after the PR number - max := strings.Count(base, "/") + 1 + max := strings.Count(base, "/") + 2 parts := strings.SplitN(path, "/", max) if len(parts) < max {