diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/inviting.yaml similarity index 61% rename from api/client-server/v1/membership.yaml rename to api/client-server/v1/inviting.yaml index c089e154a..0a0287103 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/inviting.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - title: "Matrix Client-Server v1 Room Membership API" + title: "Matrix Client-Server v1 Room Joining API" version: "1.0.0" host: localhost:8008 schemes: @@ -18,55 +18,6 @@ securityDefinitions: name: access_token in: query paths: - "/rooms/{roomId}/join": - post: - summary: Start the requesting user participating in a particular room. - description: |- - This API starts a user participating in a particular room, if that user - is allowed to participate in that room. After this call, the client is - allowed to see all current state events in the room, and all subsequent - events associated with the room until the user leaves the room. - - After a user has joined a room, the room will appear as an entry in the - response of the |initialSync| API. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier or room alias to join. - required: true - x-example: "#monkeys:matrix.org" - responses: - 200: - description: |- - The room has been joined. - - The joined room ID must be returned in the ``room_id`` field. - examples: - application/json: |- - {"room_id": "!d41d8cd:matrix.org"} - schema: - type: object - 403: - description: |- - You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - - The room is invite-only and the user was not invited. - - The user has been banned from the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" - x-alias: - canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" - aliases: - - /join/{roomId} - # With an extra " " to disambiguate from the 3pid invite endpoint # The extra space makes it sort first for what I'm sure is a good reason. "/rooms/{roomId}/invite ": diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml new file mode 100644 index 000000000..20fdbd654 --- /dev/null +++ b/api/client-server/v1/joining.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Inviting API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml new file mode 100644 index 000000000..e81d812b5 --- /dev/null +++ b/api/client-server/v1/leaving.yaml @@ -0,0 +1,92 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Leaving API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/leave": + post: + summary: Stop the requesting user participating in a particular room. + description: |- + This API stops a user participating in a particular room. + + If the user was already in the room, they will no longer be able to see + new events in the room. If the room requires an invite to join, they + will need to be re-invited before they can re-join. + + If the user was invited to the room, but had not joined, this call + serves to reject the invite. + + The user will still be allowed to retrieve history from the room which + they were previously allowed to see. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to leave. + required: true + x-example: "!nkl290a:matrix.org" + responses: + 200: + description: |- + The room has been left. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/rooms/{roomId}/forget": + post: + summary: Stop the requesting user remembering about a particular room. + description: |- + This API stops a user remembering about a particular room. + + In general, history is a first class citizen in Matrix. After this API + is called, however, a user will no longer be able to retrieve history + for this room. If all users on a homeserver forget a room, the room is + eligible for deletion from that homeserver. + + If the user is currently joined to the room, they will implicitly leave + the room as part of this API call. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to forget. + required: true + x-example: "!au1ba7o:matrix.org" + responses: + 200: + description: |- + The room has been forgotten. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml new file mode 100644 index 000000000..23b1c3d0c --- /dev/null +++ b/api/client-server/v1/list_public_rooms.yaml @@ -0,0 +1,97 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +paths: + "/publicRooms": + get: + summary: Lists the public rooms on the server. + description: |- + Lists the public rooms on the server. + + This API returns paginated responses. + responses: + 200: + description: A list of the rooms on the server. + schema: + type: object + description: A list of the rooms on the server. + properties: + chunk: + title: "PublicRoomsChunks" + type: array + description: |- + A paginated chunk of public rooms. + items: + type: object + title: "PublicRoomsChunk" + properties: + aliases: + type: array + description: |- + Aliases of the room. May be empty. + items: + type: string + name: + type: string + description: |- + The name of the room, if any. May be null. + num_joined_members: + type: number + description: |- + The number of members joined to the room. + room_id: + type: string + description: |- + The ID of the room. + topic: + type: string + description: |- + The topic of the room, if any. May be null. + world_readable: + type: boolean + description: |- + Whether the room may be viewed by guest users without joining. + guest_can_join: + type: boolean + description: |- + Whether guest users may join the room and participate in it. + If they can, they will be subject to ordinary power level + rules like any other user. + start: + type: string + description: |- + A pagination token for the response. + end: + type: string + description: |- + A pagination token for the response. + examples: + application/json: |- + { + "chunk": [ + { + "aliases": ["#murrays:cheese.bar"], + "guest_can_join": false, + "name": "CHEESE", + "num_joined_members": 37, + "room_id": "!ol19s:bleecker.street", + "topic": "Tasty tasty cheese", + "world_readable": true + } + ], + "start": "p190q", + "end": "p1902" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d10..90b51290a 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -371,7 +371,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - required: ["room_id", "membership"] + required: ["room_id"] 403: description: > You aren't a member of the room and weren't previously a diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml index 90b167b44..a10d61672 100644 --- a/api/client-server/v1/third_party_membership.yaml +++ b/api/client-server/v1/third_party_membership.yaml @@ -84,8 +84,7 @@ paths: { "id_server": "matrix.org", "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" + "address": "cheeky@monkey.com" } properties: id_server: @@ -98,10 +97,7 @@ paths: address: type: string description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] + required: ["id_server", "medium", "address"] responses: 200: description: The user has been invited to join the room. diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema deleted file mode 120000 index b020e6da4..000000000 --- a/api/client-server/v2_alpha/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json new file mode 100644 index 000000000..5a8f52f6f --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "title": "Event", + "properties": { + "content": { + "type": "object", + "title": "EventContent", + "description": "The content of this event. The fields in this object will vary depending on the type of event." + }, + "origin_server_ts": { + "type": "integer", + "format": "int64", + "description": "Timestamp in milliseconds on originating homeserver when this event was sent." + }, + "sender": { + "type": "string", + "description": "The MXID of the user who sent this event." + }, + "state_key": { + "type": "string", + "description": "Optional. This key will only be present for state events. A unique key which defines the overwriting semantics for this piece of room state." + }, + "type": { + "type": "string", + "description": "The type of event." + }, + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "transaction_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 395aed13d..7f4894234 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,8 +5,8 @@ "type": "array", "description": "List of events", "items": { - "title": "Event", - "type": "object" + "type": "object", + "allOf": [{"$ref": "event.json" }] } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json deleted file mode 100644 index fcf148f36..000000000 --- a/api/client-server/v2_alpha/definitions/room_event_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "events": { - "type": "array", - "description": "List of event ids", - "items": { - "type": "string" - } - } - } -} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index ddf8d3416..f27a5746c 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,14 +1,14 @@ { "type": "object", - "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "allOf": [{"$ref":"definitions/event_batch.json"}], "properties": { "limited": { "type": "boolean", - "description": "Whether there are more events on the server" + "description": "True if the number of events returned was limited by the ``limit`` on the filter" }, "prev_batch": { "type": "string", - "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve earlier events" } } } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index a2d5a2b89..0c9d6186f 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -95,33 +95,26 @@ paths: description: |- Updates to rooms. properties: - joined: - title: Joined + join: + title: Joined Rooms type: object + description: |- + The rooms that the user has joined. additionalProperties: title: Joined Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "core-event-schema/event.json" state: title: State type: object description: |- - The state updates for the room. + Updates to the state, between the time indicated by + the ``since`` parameter, and the start of the + ``timeline`` (or all state up to the start of the + ``timeline``, if ``since`` is not given, or + ``full_state`` is true). allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -139,8 +132,8 @@ paths: e.g. typing. allOf: - $ref: "definitions/event_batch.json" - invited: - title: Invited + invite: + title: Invited Rooms type: object description: |- The rooms that the user has been invited to. @@ -166,37 +159,22 @@ paths: ``invite_state``. allOf: - $ref: "definitions/event_batch.json" - archived: - title: Archived + leave: + title: Left rooms type: object description: |- - The rooms that the user has left or been banned from. The - entries in the room_map will lack an ``ephemeral`` key. + The rooms that the user has left or been banned from. additionalProperties: - title: Archived Room + title: Left Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "core-event-schema/event.json" state: title: State type: object description: |- - The state updates for the room up to the point when - the user left. + The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -226,47 +204,43 @@ paths: ] }, "rooms": { - "joined": { + "join": { "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "unsigned": { - "prev_content": {"membership": "invite"} - }, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": { - "body": "I am a fish", - "msgtype": "m.text" - }, - "origin_server_ts": 1417731086797 - } - }, "state": { "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795, + "event_id": "$66697273743031:example.com" + } ] }, "timeline": { "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" + { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "prev_content": {"membership": "invite"}, + "origin_server_ts": 1417731086795, + "event_id": "$7365636s6r6432:example.com" + }, + { + "sender": "@alice:example.com", + "type": "m.room.message", + "age": 124524, + "txn_id": "1234", + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797, + "event_id": "$74686972643033:example.com" + } ], "limited": true, "prev_batch": "t34-23535_0_0" @@ -274,7 +248,6 @@ paths: "ephemeral": { "events": [ { - "room_id": "!726s6s6q:example.com", "type": "m.typing", "content": {"user_ids": ["@alice:example.com"]} } @@ -282,7 +255,7 @@ paths: } } }, - "invited": { + "invite": { "!696r7674:example.com": { "invite_state": { "events": [ @@ -302,6 +275,6 @@ paths: } } }, - "archived": {} + "leave": {} } } diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 93622c3d4..de5973fac 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -9,26 +9,35 @@ Caveats can only be used for reducing the scope of a token, never for increasing Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. -All caveats must take the form: +All caveats must take the form:: -`key` `operator` `value` -where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] -`operator` is a non-empty string which does not contain whitespace -`value` is a non-empty string + key operator value + +where: + - ``key`` is a non-empty string drawn from the character set [A-Za-z0-9_] + - ``operator`` is a non-empty string which does not contain whitespace + - ``value`` is a non-empty string + And these are joined by single space characters. Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | Caveat name | Description | Legal Values | -+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ ++=============+==================================================+================================================================================================+ | gen | Generation of the macaroon caveat spec. | 1 | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | -| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| refresh - only used to authorize a token refresh | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ +| type | The purpose of this macaroon. | - ``access``: used to authorize any action except token refresh | +| | | - ``refresh``: only used to authorize a token refresh | +| | | - ``login``: issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | -| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | -| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | -| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| -| Note that exact equality of time is largely meaningless. | +| | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| | | Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| | | Note that exact equality of time is largely meaningless. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst new file mode 100644 index 000000000..1699f79bd --- /dev/null +++ b/drafts/markjh_end_to_end.rst @@ -0,0 +1,142 @@ +Goals of Key-Distribution in Matrix +=================================== + +* No Central Authority: Users should not need to trust a central authority + when determining the authenticity of keys. + +* Easy to Add New Devices: It should be easy for a user to start using a + new device. + +* Possible to discover MITM: It should be possible for a user to determine if + they are being MITM. + +* Lost Devices: It should be possible for a user to recover if they lose all + their devices. + +* No Copying Keys: Keys should be per device and shouldn't leave the device + they were created on. + +A Possible Mechanism for Key Distribution +========================================= + +Basic API for setting up keys on a server: + +https://github.com/matrix-org/matrix-doc/pull/24 + +Client shouldn't trust the keys unless they have been verified, e.g by +comparing fingerprints. + +If a user adds a new device it should some yet to be specified protocol +communicate with an old device and obtain a cross-signature from the old +device for its public key. + +The new device can then present the cross-signed key to all the devices +that the user is in conversations with. Those devices should then include +the new device into those conversations. + +If the user cannot cross-sign the new key, e.g. because their old device +is lost or stolen. Then they will need to reauthenticate their conversations +out of band, e.g by comparing fingerprints. + + +Goals of End-to-end encryption in Matrix +======================================== + +* Access to Chat History: Users should be able to see the history of a + conversation on a new device. User should be able to control who can + see their chat history and how much of the chat history they can see. + +* Forward Secrecy of Discarded Chat History: Users should be able to discard + history from their device, once they have discarded the history it should be + impossible for an adversary to recover that history. + +* Forward Secrecy of Future Messages: Users should be able to recover from + disclosure of the chat history on their device. + +* Deniablity of Chat History: It should not be possible to prove to a third + party that a given user sent a message. + +* Authenticity of Chat History: It should be possible to prove amoungst + the members of a chat that a message sent by a user was authored by that + user. + + +Bonus Goals: + +* Traffic Analysis: It would be nice if the protocol was resilient to traffic + or metadata analysis. However it's not something we want to persue if it + harms the usability of the protocol. It might be cool if there was a + way for the user to could specify the trade off between performance and + resilience to traffic analysis that they wanted. + + +A Possible Design for Group Chat using Olm +========================================== + +Protecting the secrecy of history +--------------------------------- + +Each message sent by a client has a 32-bit counter. This counter increments +by one for each message sent by the client. This counter is used to advance a +ratchet. The ratchet is split into a vector four 256-bit values, +:math:`R_{n,j}` for :math:`j \in {0,1,2,3}`. The ratchet can be advanced as +follows: + +.. math:: + \begin{align} + R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ + R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_1\left(R_{(i-1),3}\right) + \end{align} + +Where :math:`H_1` and :math:`H_2` are different hash functions. For example +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and +:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. + +So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. +Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. +Every :math:`2^8` iterations :math:`R_{n,3}` is reseeded from :math:`R_{n,2}`. + +This scheme allows the ratchet to be advanced an arbitrary amount forwards +while needing only 1024 hash computations. + +This the value of the ratchet is hashed to generate the keys used to encrypt +each mesage. + +A client can decrypt chat history onwards from the earliest value of the +ratchet it is aware of. But cannot decrypt history from before that point +without reversing the hash function. + +This allows a client to share its ability to decrypt chat history with another +from a point in the conversation onwards by giving a copy of the ratchet at +that point in the conversation. + +A client can discard history by advancing a ratchet to beyond the last message +they want to discard and then forgetting all previous values of the ratchet. + +Proving and denying the authenticity of history +----------------------------------------------- + +Client sign the messages they send using a Ed25519 key generated per +conversation. That key, along with the ratchet key, is distributed +to other clients using 1:1 olm ratchets. Those 1:1 ratchets are started using +Triple Diffie-Hellman which provides authenticity of the messages to the +participants and deniability of the messages to third parties. Therefore +any keys shared over those keys inherit the same levels of deniability and +authenticity. + +Protecting the secrecy of future messages +----------------------------------------- + +A client would need to generate new keys if it wanted to prevent access to +messages beyond a given point in the conversation. It must generate new keys +whenever someone leaves the room. It should generate new keys periodically +anyway. + +The frequency of key generation in a large room may need to be restricted to +keep the frequency of messages broadcast over the individual 1:1 channels +low. diff --git a/drafts/websockets.rst b/drafts/websockets.rst new file mode 100644 index 000000000..bd0ff081d --- /dev/null +++ b/drafts/websockets.rst @@ -0,0 +1,285 @@ +WebSockets API +============== + +Introduction +------------ +This document is a proposal for a WebSockets-based client-server API. It is not +intended to replace the REST API, but rather to complement it and provide an +alternative interface for certain operations. + +The primary goal is to offer a more efficient interface than the REST API: by +using a bidirectional protocol such as WebSockets we can avoid the overheads +involved in long-polling (SSL negotiation, HTTP headers, etc). In doing so we +will reduce the latency between server and client by allowing the server to +send events as soon as they arrive, rather than having to wait for a poll from +the client. + +Handshake +--------- +1. Instead of calling ``/sync``, the client makes a websocket request to + ``/_matrix/client/rN/stream``, passing the query parameters ``access_token`` + and ``since``, and optionally ``filter`` - all of which have the same + meaning as for ``/sync``. + + * The client sets the ``Sec-WebSocket-Protocol`` to ``m.json``. (Servers may + offer alternative encodings; at present only the JSON encoding is + specified but in future we will specify alternative encodings.) + +#. The server returns the websocket handshake; the socket is then connected. + +If the server does not return a valid websocket handshake, this indicates that +the server or an intermediate proxy does not support WebSockets. In this case, +the client should fall back to polling the ``/sync`` REST endpoint. + +Example +~~~~~~~ + +Client request: + +.. code:: http + + GET /_matrix/client/v2_alpha/stream?access_token=123456&since=s72594_4483_1934 HTTP/1.1 + Host: matrix.org + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: m.json + Sec-WebSocket-Version: 13 + Origin: https://matrix.org + +Server response: + +.. code:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: m.json + + +Update Notifications +-------------------- +Once the socket is connected, the server begins streaming updates over the +websocket. The server sends Update notifications about new messages or state +changes. To make it easy for clients to parse, Update notifications have the +same structure as the response to ``/sync``: an object with the following +members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +next_batch string The batch token to supply in the ``since`` param of + the next /sync request. This is not required for + streaming of events over the WebSocket, but is + provided so that clients can reconnect if the + socket is disconnected. +presence Presence The updates to the presence status of other users. +rooms Rooms Updates to rooms. +============= ========== =================================================== + +Example +~~~~~~~ +Message from the server: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [] + }, + "rooms": { + "join": {}, + "invite": {}, + "leave": {} + } + } + + +Client-initiated operations +--------------------------- + +The client can perform certain operations by sending a websocket message to +the server. Such a "Request" message should be a JSON-encoded object with +the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string A unique identifier for this request +method string Specifies the name of the operation to be + performed; see below for available operations +param object The parameters for the requested operation. +============= ========== =================================================== + +The server responds to a client Request with a Response message. This is a +JSON-encoded object with the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string The same as the value in the corresponding Request + object. The presence of the ``id`` field + distinguishes a Response message from an Update + notification. +result object On success, the results of the request. +error object On error, an object giving the resons for the + error. This has the same structure as the "standard + error response" for the Matrix API: an object with + the fields ``errcode`` and ``error``. +============= ========== =================================================== + +Request methods +~~~~~~~~~~~~~~~ +It is not intended that all operations which are available via the REST API +will be available via the WebSockets API, but a few simple, common operations +will be exposed. The initial operations will be as follows. + +``ping`` +^^^^^^^^ +This is a no-op which clients may use to keep their connection alive. + +The request ``params`` and the response ``result`` should be empty. + +``send`` +^^^^^^^^ +Send a message event to a room. The parameters are as follows: + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to send the event to +event_type string **Required.** The type of event to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + +The ``id`` from the Request message is used as the transaction ID by the +server. + +``state`` +^^^^^^^^^ +Update the state on a room. + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to set the state in +event_type string **Required.** The type of event to send. +state_key string **Required.** The state_key for the state to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + + +Example +~~~~~~~ +Client request: + +.. code:: json + + { + "id": "12345", + "method": "send", + "params": { + "room_id": "!d41d8cd:matrix.org", + "event_type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "hello" + } + } + } + +Server response: + +.. code:: json + + { + "id": "12345", + "result": { + "event_id": "$66697273743031:matrix.org" + } + } + +Alternative server response, in case of error: + +.. code:: json + + { + "id": "12345", + "error": { + "errcode": "M_MISSING_PARAM", + "error": "Missing parameter: event_type" + } + } + + +Rationale +--------- +Alternatives to WebSockets include HTTP/2, CoAP, and simply rolling our own +protocol over raw TCP sockets. However, the need to implement browser-based +clients essentially reduces our choice to WebSockets. HTTP/2 streams will +probably provide an interesting alternative in the future, but current browsers +do not appear to give javascript applications low-level access to the protocol. + +Concerning the continued use of the JSON encoding: we prefer to focus on the +transition to WebSockets initially. Replacing JSON with a compact +representation such as CBOR, MessagePack, or even just compressed JSON will be +a likely extension for the future. The support for negotiation of subprotocols +within WebSockets should make this a simple transition once time permits. + +The number of methods available for client requests is deliberately limited, as +each method requires code to be written to map it onto the equivalent REST +implementation. Some REST methods - for instance, user registration and login - +would be pointless to expose via WebSockets. It is likely, however, that we +will increate the number of methods available via the WebSockets API as it +becomes clear which would be most useful. + +Open questions +-------------- + +Throttling +~~~~~~~~~~ +At least in v2 sync, clients are inherently self-throttling - if they do not +poll quickly enough, events will be dropped from the next result. This proposal +raises the possibility that events will be produced more quickly than they can +be sent to the client; backlogs will build up on the server and/or in the +intermediate network, which will not only lead to high latency on events being +delivered, but will lead to responses to client requests also being delayed. + +We may need to implement some sort of throttling mechanism by which the server +can start to drop events. The difficulty is in knowing when to start dropping +events. A few ideas: + +* Use websocket pings to measure the RTT; if it starts to increase, start + dropping events. But this requires knowledge of the base RTT, and a useful + model of what constitutes an excessive increase. + +* Have the client acknowledge each batch of events, and use a window to ensure + the number of outstanding batches is limited. This is annoying as it requires + the client to have to acknowledge batches - and it's not clear what the right + window size is: we want a big window for long fat networks (think of mobile + clients), but a small one for one with lower latency. + +* Start dropping events if the server's TCP buffer starts filling up. This has + the advantage of delegating the congestion-detection to TCP (which already + has a number of algorithms to deal with it, to greater or lesser + effectiveness), but relies on homeservers being hosted on OSes which use + sensible TCP congestion-avoidance algorithms, and more critically, an ability + to read the fill level of the TCP send buffer. diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 81057049c..25b301097 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], diff --git a/jenkins.sh b/jenkins.sh index 0b217e58a..f5ed3b15a 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -7,3 +7,12 @@ set -ex (cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) + +: ${GOPATH:=${WORKSPACE}/.gopath} +mkdir -p "${GOPATH}" +export GOPATH +go get github.com/hashicorp/golang-lru +go get gopkg.in/fsnotify.v1 + +(cd scripts/continuserv && go build) +(cd scripts/speculator && go build) diff --git a/scripts/matrix-org-gendoc.sh b/scripts/add-matrix-org-stylings.sh similarity index 62% rename from scripts/matrix-org-gendoc.sh rename to scripts/add-matrix-org-stylings.sh index 5716786d2..ef4e1014a 100755 --- a/scripts/matrix-org-gendoc.sh +++ b/scripts/add-matrix-org-stylings.sh @@ -1,41 +1,20 @@ -#! /bin/bash +#!/bin/bash -eu -if [ -z "$1" ]; then - echo "Expected /includes/head.html file as 1st arg." - exit 1 -fi - -if [ -z "$2" ]; then - echo "Expected /includes/nav.html file as 2nd arg." - exit 1 -fi - -if [ -z "$3" ]; then - echo "Expected /includes/footer.html file as 3rd arg." - exit 1 -fi - - -HEADER=$1 -NAV_BAR=$2 -FOOTER=$3 - -if [ ! -f $HEADER ]; then - echo $HEADER " does not exist" - exit 1 +if [[ $# != 1 || ! -d $1 ]]; then + echo >&2 "Usage: $0 include_dir" + exit 1 fi -if [ ! -f $NAV_BAR ]; then - echo $NAV_BAR " does not exist" - exit 1 -fi +HEADER="$1/head.html" +NAV_BAR="$1/nav.html" +FOOTER="$1/footer.html" -if [ ! -f $FOOTER ]; then - echo $FOOTER " does not exist" +for f in "$1"/{head,nav,footer}.html; do + if [[ ! -e "${f}" ]]; then + echo >&2 "Need ${f} to exist" exit 1 -fi - -python gendoc.py + fi +done perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s##$header diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed36a5a78..28a115284 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -238,6 +238,18 @@ def rst2html(i, o): ) +def addAnchors(path): + with open(path, "r") as f: + lines = f.readlines() + + replacement = replacement = r'

\n\1' + with open(path, "w") as f: + for line in lines: + line = re.sub(r'()', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write(line + "\n") + + def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: @@ -387,6 +399,7 @@ def main(target_name, keep_intermediates): shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") + addAnchors("gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: cleanup_env() diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 7f86bd628..0c9a2bfa7 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -63,20 +64,22 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" - matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" + permissionsOwnerFull = 0700 ) func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, directory) + if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil { + return "", fmt.Errorf("error making directory %s: %v", directory, err) + } + args := []string{"clone", url, directory} if shared { - cmd.Args = append(cmd.Args, "--shared") + args = append(args, "--shared") } - - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("error cloning repo: %v", err) + if err := runGitCommand(directory, args); err != nil { + return "", err } return directory, nil } @@ -85,16 +88,13 @@ func gitCheckout(path, sha string) error { return runGitCommand(path, []string{"checkout", sha}) } -func gitFetch(path string) error { - return runGitCommand(path, []string{"fetch"}) -} - func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path - err := cmd.Run() - if err != nil { - return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String()) } return nil } @@ -126,8 +126,7 @@ func generate(dir string) error { cmd.Dir = path.Join(dir, "scripts") var b bytes.Buffer cmd.Stderr = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return nil @@ -139,17 +138,35 @@ func writeError(w http.ResponseWriter, code int, err error) { } type server struct { + mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL matrixDocCloneURL string } +func (s *server) updateBase() error { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"fetch"}) +} + +// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL. +func (s *server) canCheckout(sha string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) - if err != nil { - return + if !s.canCheckout(sha) { + err = s.updateBase() + if err != nil { + return + } } + s.mu.Lock() dst, err = gitClone(s.matrixDocCloneURL, true) + s.mu.Unlock() if err != nil { return } @@ -167,7 +184,9 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b + s.mu.Lock() err := cmd.Run() + s.mu.Unlock() if err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } @@ -178,12 +197,14 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - originHead, err := s.getSHAOf("origin/master") - if err != nil { + // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. + // This is to deal with cases like where github is down but we still want to serve the spec. + if headSha, err := s.lookupHeadSHA(); headSha == "" { writeError(w, 500, err) return + } else { + sha = headSha } - sha = originHead } else { pr, err := lookupPullRequest(*req.URL, "/spec") if err != nil { @@ -220,6 +241,25 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { specCache.Add(sha, b) } +// lookupHeadSHA looks up what origin/master's HEAD SHA is. +// It attempts to `git fetch` before doing so. +// If this fails, it may still return a stale sha, but will also return an error. +func (s *server) lookupHeadSHA() (sha string, retErr error) { + retErr = s.updateBase() + if retErr != nil { + log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr) + } + originHead, err := s.getSHAOf("origin/master") + if err != nil { + retErr = err + } + sha = originHead + if retErr != nil && originHead != "" { + log.Printf("Successfully fell back to possibly stale sha: %s", sha) + } + return +} + func checkAuth(pr *PullRequest) error { if !pr.User.IsTrusted() { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login) @@ -383,9 +423,9 @@ func main() { if err != nil { log.Fatal(err) } - s := server{masterCloneDir} + s := server{matrixDocCloneURL: masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) - http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", forceHTML(listPulls)) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index b02dbf28c..ba5578bcb 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -870,42 +870,22 @@ following values: ``invite`` This room can only be joined if you were invited. -{{membership_http_api}} +{{inviting_http_api}} + +{{joining_http_api}} Leaving rooms ~~~~~~~~~~~~~ -.. TODO-spec - HS deleting rooms they are no longer a part of. Not implemented. - - This is actually Very Tricky. If all clients a HS is serving leave a room, - the HS will no longer get any new events for that room, because the servers - who get the events are determined on the *membership list*. There should - probably be a way for a HS to lurk on a room even if there are 0 of their - members in the room. - - Grace period before deletion? - - Under what conditions should a room NOT be purged? - - A user can leave a room to stop receiving events for that room. A user must have been invited to or have joined the room before they are eligible to leave the room. Leaving a room to which the user has been invited rejects the invite. +Once a user leaves a room, it will no longer appear on the |initialSync|_ API. Whether or not they actually joined the room, if the room is an "invite-only" room they will need to be re-invited before they can re-join -the room. To leave a room, a request should be made to -|/rooms//leave|_ with:: +the room. - {} - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "leave" - } - -See the `Room events`_ section for more information on ``m.room.member``. Once a -user has left a room, that room will no longer appear on the |initialSync|_ API. -If all members in a room leave, that room becomes eligible for deletion. +{{leaving_http_api}} Banning users in a room ~~~~~~~~~~~~~~~~~~~~~~~ @@ -931,6 +911,11 @@ member's state, by making a request to "membership": "ban" } +Listing rooms +~~~~~~~~~~~~~ + +{{list_public_rooms_http_api}} + Profiles -------- diff --git a/specification/events.rst b/specification/events.rst index a1aece1c4..5a0031155 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -30,12 +30,14 @@ formatted for federation by: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, ``origin``, ``prev_state``. * Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has ellapsed since the event was sent. -* Adding a ``prev_content`` to the ``unsigned`` object if the event is - a ``state event`` which gives previous content of that state key. + milliseconds that has elapsed since the event was sent. +* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the + event is a ``state event``, which give the previous content and previous + sender of that state key * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``transaction_id`` if the event was sent by the client requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af93..1f393981c 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -32,6 +32,7 @@ retrieving events: * `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ * `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ * `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ +* `GET /rooms/:room_id/initialSync <#get-matrix-client-api-v1-rooms-roomid-initialsync>`_ There is also a special version of the `GET /events <#get-matrix-client-api-v1-events>`_ endpoint: @@ -51,6 +52,11 @@ sending events: Guest clients *do* need to join rooms in order to send events to them. +The following API endpoints are allowed to be accessed by guest accounts for +their own account maintenance: + +* `PUT /profile/:user_id/displayname <#put-matrix-client-api-v1-profile-userid-displayname>`_ + Server behaviour ---------------- Servers are required to only return events to guest accounts for rooms where diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 4d268631d..d8e3d4d92 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -127,13 +127,10 @@ It is important for user privacy that leaking the mapping between a matrix user ID and a third party identifier is hard. In particular, being able to look up all third party identifiers from a matrix user ID (and accordingly, being able to link each third party identifier) should be avoided wherever possible. -To this end, when implementing this API care should be taken to avoid -adding links between these two identifiers as room events. This mapping can be -unintentionally created by specifying the third party identifier in the -``display_name`` field of the ``m.room.third_party_invite`` event, and then -observing which matrix user ID joins the room using that invite. Clients SHOULD -set ``display_name`` to a value other than the third party identifier, e.g. the -invitee's common name. +To this end, the third party identifier is not put in any event, rather an +opaque display name provided by the identity server is put into the events. +Clients should not remember or display third party identifiers from invites, +other than for the use of the inviter themself. Homeservers are not required to trust any particular identity server(s). It is generally a client's responsibility to decide which identity servers it trusts, diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 26e040ca2..cc9426e2b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -533,6 +533,164 @@ part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. + +To join a room:: + + GET .../make_join// + Response: JSON encoding of a join proto-event + + PUT .../send_join// + Response: JSON encoding of the state of the room at the time of the event + +Performs the room join handshake. For more information, see "Joining Rooms" +below. + +Joining Rooms +------------- + +When a new user wishes to join room that the user's homeserver already knows +about, the homeserver can immediately determine if this is allowable by +inspecting the state of the room, and if it is acceptable, it can generate, +sign, and emit a new ``m.room.member`` state event adding the user into that +room. When the homeserver does not yet know about the room it cannot do this +directly. Instead, it must take a longer multi-stage handshaking process by +which it first selects a remote homeserver which is already participating in +that room, and uses it to assist in the joining process. This is the remote +join handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting the +room alias the user is requesting to join with, and a homeserver where existing +room members are already present (referred to as the "resident" server). + +In summary, the remote join handshake consists of the joining server querying +the directory server for information about the room alias; receiving a room ID +and a list of join candidates. The joining server then requests information +about the room from one of the residents. It uses this information to construct +a ``m.room.member`` event which it finally sends to a resident server. + +Conceptually these are three different roles of homeserver. In practice the +directory server is likely to be resident in the room, and so may be selected +by the joining server to be the assisting resident. Likewise, it is likely that +the joining server picks the same candidate resident for both phases of event +construction, though in principle any valid candidate may be used at each time. +Thus, any join handshake can potentially involve anywhere from two to four +homeservers, though most in practice will use just two. + +:: + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response + +The first part of the handshake usually involves using the directory server to +request the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver could +optimise this step away by picking the origin server of that invite message as +the join candidate. However, the joining server should be aware that the origin +server of the invite might since have left the room, so should be prepared to +fall back on the regular join flow if this optimisation fails. + +Once the joining server has the room ID and the join candidates, it then needs +to obtain enough information about the room to fill in the required fields of +the ``m.room.member`` event. It obtains this by selecting a resident from the +candidate list, and requesting the ``make_join`` endpoint using a ``GET`` +request, specifying the room ID and the user ID of the new member who is +attempting to join. + +The resident server replies to this request with a JSON-encoded object having a +single key called ``event``; within this is an object whose fields contain some +of the information that the joining server will need. Despite its name, this +object is not a full event; notably it does not need to be hashed or signed by +the resident homeserver. The required fields are: + +==================== ======== ============ + Key Type Description +==================== ======== ============ +``type`` String The value ``m.room.member`` +``auth_events`` List An event-reference list containing the + authorization events that would allow this member + to join +``content`` Object The event content +``depth`` Integer (this field must be present but is ignored; it + may be 0) +``event_id`` String A new event ID specified by the resident + homeserver +``origin`` String The name of the resident homeserver +``origin_server_ts`` Integer A timestamp added by the resident homeserver +``prev_events`` List An event-reference list containing the immediate + predecessor events +``room_id`` String The room ID of the room +``sender`` String The user ID of the joining member +``state_key`` String The user ID of the joining member +==================== ======== ============ + +The ``content`` field itself must be an object, containing: + +============== ====== ============ + Key Type Description +============== ====== ============ +``membership`` String The value ``join`` +============== ====== ============ + +The joining server now has sufficient information to construct the real join +event from these protoevent fields. It copies the values of most of them, +adding (or replacing) the following fields: + +==================== ======= ============ + Key Type Description +==================== ======= ============ +``event_id`` String A new event ID specified by the joining homeserver +``origin`` String The name of the joining homeserver +``origin_server_ts`` Integer A timestamp added by the joining homeserver +==================== ======= ============ + +.. TODO-spec + - Why does the protoevent have an event_id, only for the real event to ignore + it and specify a different one? We should definitely pick one or the other. + +This will be a true event, so the joining server should apply the event-signing +algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` +fields. + +To complete the join handshake, the joining server must now submit this new +event to an resident homeserver, by using the ``send_join`` endpoint. This is +invoked using the room ID and the event ID of the new member event. + +The resident homeserver then accepts this event into the room's event graph, +and responds to the joining server with the full set of state for the newly- +joined room. This is returned as a two-element list, whose first element is the +integer 200, and whose second element is an object which contains the +following keys: + +============== ===== ============ + Key Type Description +============== ===== ============ +``auth_chain`` List A list of events giving the authorization chain for this + join event +``state`` List A complete list of the prevailing state events at the + instant just before accepting the new ``m.room.member`` + event +============== ===== ============ + +.. TODO-spec + - (paul) I don't really understand why the full auth_chain events are given + here. What purpose does it serve expanding them out in full, when surely + they'll appear in the state anyway? + Backfilling ----------- .. NOTE:: @@ -763,6 +921,7 @@ Querying directory information:: servers: list of strings giving the join candidates The list of join candidates is a list of server names that are likely to hold -the given room; these are servers that the requesting server may wish to try -joining with. This list may or may not include the server answering the query. +the given room; these are servers that the requesting server may wish to use as +resident servers as part of the remote join handshake. This list may or may not +include the server answering the query. diff --git a/templating/build.py b/templating/build.py index a35d8a085..77fecf91f 100755 --- a/templating/build.py +++ b/templating/build.py @@ -93,6 +93,27 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return '\n\n'.join(output_lines) + def fieldwidths(input, keys, defaults=[], default_width=15): + """ + A template filter to help in the generation of tables. + + Given a list of rows, returns a list giving the maximum length of the + values in each column. + + :param list[dict[str, str]] input: a list of rows. Each row should be a + dict with the keys given in ``keys``. + :param list[str] keys: the keys corresponding to the table columns + :param list[int] defaults: for each column, the default column width. + :param int default_width: if ``defaults`` is shorter than ``keys``, this + will be used as a fallback + """ + def colwidth(key, default): + return reduce(max, (len(row[key]) for row in input), + default if default is not None else default_width) + + results = map(colwidth, keys, defaults) + return results + # make Jinja aware of the templates and filters env = Environment( loader=FileSystemLoader(in_mod.exports["templates"]), @@ -102,6 +123,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): env.filters["indent"] = indent env.filters["indent_block"] = indent_block env.filters["wrap"] = wrap + env.filters["fieldwidths"] = fieldwidths # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 3f16be3da..8d8c8f0ca 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,17 +1,8 @@ +{% import 'tables.tmpl' as tables -%} + {{common_event.title}} Fields {{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} -================== ================= =========================================== - Key Type Description -================== ================= =========================================== -{% for row in common_event.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc | indent(18 - (row.type|length)) |wrap(43) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fbefe17f3..cd165ff06 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.type}}`` {{(4 + event.type | length) * title_kind}} *{{event.typeof}}* @@ -7,18 +9,7 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ================= =========================================== - {{table.title or "Content"}} Key Type Description -======================= ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} -{% endfor -%} -======================= ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example{% if examples | length > 1 %}s{% endif %}: diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 86eacb145..1253e66eb 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} {% if "alias_for_path" in endpoint -%} @@ -12,18 +14,11 @@ {{":Requires auth: Yes." if endpoint.requires_auth else "" }} Request format: - -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== -{% for loc in endpoint.req_param_by_loc -%} -*{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- -{% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} -{% endfor -%} -{% endfor -%} -=========================================== ================= =========================================== +{% if (endpoint.req_param_by_loc | length) %} +{{ tables.split_paramtable(endpoint.req_param_by_loc) }} +{% else %} +`No parameters` +{% endif %} {% if endpoint.res_tables|length > 0 -%} Response format: @@ -31,18 +26,7 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ========================= ========================================== - Param Type Description -======================= ========================= ========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (42 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} -{% endfor -%} -======================= ========================= ========================================== +{{ tables.paramtable(table.rows) }} {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 18d3492bc..87cf4a193 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,21 +1,12 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.msgtype}}`` {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/tables.tmpl b/templating/matrix_templates/templates/tables.tmpl new file mode 100644 index 000000000..aba6c0c49 --- /dev/null +++ b/templating/matrix_templates/templates/tables.tmpl @@ -0,0 +1,104 @@ +{# + # A set of macros for generating RST tables + #} + + +{# + # write a table for a list of parameters. + # + # 'rows' is the list of parameters. Each row should have the keys + # 'key', 'type', and 'desc'. + #} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} +{{ split_paramtable({None: rows}, titles) }} +{% endmacro %} + + +{# + # write a table for the request parameters, split by location. + # 'rows_by_loc' is a map from location to a list of parameters. + # + # As a special case, if a key of 'rows_by_loc' is 'None', no title row is + # written for that location. This is used by the standard 'paramtable' macro. + #} +{% macro split_paramtable(rows_by_loc, + titles=["Parameter", "Type", "Description"]) -%} + +{% set rowkeys = ['key', 'type', 'desc'] %} +{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %} + +{# We need the rows flattened into a single list. Abuse the 'sum' filter to + # join arrays instead of add numbers. -#} +{% set flatrows = rows_by_loc.values()|sum(start=[]) -%} + +{# Figure out the widths of the columns. The last column is always 50 characters + # wide; the others default to 10, but stretch if there is wider text in the + # column. -#} +{% set fieldwidths = (([titlerow] + flatrows) | + fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} + +{{ tableheader(fieldwidths) }} +{{ tablerow(fieldwidths, titlerow, rowkeys) }} +{{ tableheader(fieldwidths) }} +{% for loc in rows_by_loc -%} + +{% if loc != None -%} +{{ tablespan(fieldwidths, "*" ~ loc ~ " parameters*") }} +{% endif -%} + +{% for row in rows_by_loc[loc] -%} +{{ tablerow(fieldwidths, row, rowkeys) }} +{% endfor -%} +{% endfor -%} + +{{ tableheader(fieldwidths) }} +{% endmacro %} + + + +{# + # Write a table header row, for the given column widths + #} +{% macro tableheader(widths) -%} +{% for arg in widths -%} +{{"="*arg}} {% endfor -%} +{% endmacro %} + + + +{# + # Write a normal table row. Each of 'widths' and 'keys' should be sequences + # of the same length; 'widths' defines the column widths, and 'keys' the + # attributes of 'row' to look up for values to put in the columns. + #} +{% macro tablerow(widths, row, keys) -%} +{% for key in keys -%} +{% set value=row[key] -%} +{% if not loop.last -%} + {# the first few columns need space after them -#} + {{ value }}{{" "*(1+widths[loop.index0]-value|length) -}} +{% else -%} + {# the last column needs wrapping and indenting (by the sum of the widths of + the preceding columns, plus the number of preceding columns (for the + separators)) -#} + {{ value | wrap(widths[loop.index0]) | + indent_block(widths[0:-1]|sum + loop.index0) -}} +{% endif -%} +{% endfor -%} +{% endmacro %} + + + + +{# + # write a tablespan row. This is a single value which spans the entire table. + #} +{% macro tablespan(widths, value) -%} +{{value}} +{# we write a trailing space to stop the separator being misinterpreted + # as a header line. -#} +{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} + + + + diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 94435c52e..73997c4e4 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -122,10 +122,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "x-pattern", "string" ) value_type = "{%s: %s}" % (key, nested_object[0]["title"]) + value_id = "%s: %s" % (key, nested_object[0]["title"]) if not nested_object[0].get("no-table"): tables += nested_object else: - value_type = "{string: %s}" % prop_val + value_type = "{string: %s}" % (prop_val,) + value_id = "string: %s" % (prop_val,) else: nested_object = get_json_schema_object_fields( props[key_name], @@ -133,6 +135,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) if not nested_object[0].get("no-table"): tables += nested_object @@ -145,12 +148,14 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) tables += nested_object else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): value_type = " or ".join(value_type) value_type = "[%s]" % value_type + value_id = "%s" % (value_type,) array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: @@ -164,6 +169,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals ) else: value_type = props[key_name]["type"] + value_id = props[key_name]["type"] if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -184,6 +190,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals fields["rows"].append({ "key": key_name, "type": value_type, + "id": value_id, "required": required, "desc": desc, "req_str": "**Required.** " if required else "" @@ -313,10 +320,16 @@ class MatrixUnits(Units): if req_tables > 1: for table in req_tables[1:]: - nested_key_name = [ - s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) - ][0] + nested_key_name = { + "key": s["key"] + for rtable in req_tables + for s in rtable["rows"] + if s["id"] == table["title"] + }.get("key", None) + + if nested_key_name is None: + raise Exception("Failed to find table for %r" % (table["title"],)) + for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"])