diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/inviting.yaml similarity index 61% rename from api/client-server/v1/membership.yaml rename to api/client-server/v1/inviting.yaml index c089e154..0a028710 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/inviting.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - title: "Matrix Client-Server v1 Room Membership API" + title: "Matrix Client-Server v1 Room Joining API" version: "1.0.0" host: localhost:8008 schemes: @@ -18,55 +18,6 @@ securityDefinitions: name: access_token in: query paths: - "/rooms/{roomId}/join": - post: - summary: Start the requesting user participating in a particular room. - description: |- - This API starts a user participating in a particular room, if that user - is allowed to participate in that room. After this call, the client is - allowed to see all current state events in the room, and all subsequent - events associated with the room until the user leaves the room. - - After a user has joined a room, the room will appear as an entry in the - response of the |initialSync| API. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier or room alias to join. - required: true - x-example: "#monkeys:matrix.org" - responses: - 200: - description: |- - The room has been joined. - - The joined room ID must be returned in the ``room_id`` field. - examples: - application/json: |- - {"room_id": "!d41d8cd:matrix.org"} - schema: - type: object - 403: - description: |- - You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - - - The room is invite-only and the user was not invited. - - The user has been banned from the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" - x-alias: - canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" - aliases: - - /join/{roomId} - # With an extra " " to disambiguate from the 3pid invite endpoint # The extra space makes it sort first for what I'm sure is a good reason. "/rooms/{roomId}/invite ": diff --git a/api/client-server/v1/joining.yaml b/api/client-server/v1/joining.yaml new file mode 100644 index 00000000..20fdbd65 --- /dev/null +++ b/api/client-server/v1/joining.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Inviting API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + The joined room ID must be returned in the ``room_id`` field. + examples: + application/json: |- + {"room_id": "!d41d8cd:matrix.org"} + schema: + type: object + 403: + description: |- + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: + + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/api/client-server/v1/leaving.yaml b/api/client-server/v1/leaving.yaml new file mode 100644 index 00000000..e81d812b --- /dev/null +++ b/api/client-server/v1/leaving.yaml @@ -0,0 +1,92 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Leaving API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/leave": + post: + summary: Stop the requesting user participating in a particular room. + description: |- + This API stops a user participating in a particular room. + + If the user was already in the room, they will no longer be able to see + new events in the room. If the room requires an invite to join, they + will need to be re-invited before they can re-join. + + If the user was invited to the room, but had not joined, this call + serves to reject the invite. + + The user will still be allowed to retrieve history from the room which + they were previously allowed to see. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to leave. + required: true + x-example: "!nkl290a:matrix.org" + responses: + 200: + description: |- + The room has been left. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/rooms/{roomId}/forget": + post: + summary: Stop the requesting user remembering about a particular room. + description: |- + This API stops a user remembering about a particular room. + + In general, history is a first class citizen in Matrix. After this API + is called, however, a user will no longer be able to retrieve history + for this room. If all users on a homeserver forget a room, the room is + eligible for deletion from that homeserver. + + If the user is currently joined to the room, they will implicitly leave + the room as part of this API call. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier to forget. + required: true + x-example: "!au1ba7o:matrix.org" + responses: + 200: + description: |- + The room has been forgotten. + examples: + application/json: |- + {} + schema: + type: object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema deleted file mode 120000 index b020e6da..00000000 --- a/api/client-server/v2_alpha/core-event-schema +++ /dev/null @@ -1 +0,0 @@ -../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event.json b/api/client-server/v2_alpha/definitions/event.json new file mode 100644 index 00000000..5a8f52f6 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "title": "Event", + "properties": { + "content": { + "type": "object", + "title": "EventContent", + "description": "The content of this event. The fields in this object will vary depending on the type of event." + }, + "origin_server_ts": { + "type": "integer", + "format": "int64", + "description": "Timestamp in milliseconds on originating homeserver when this event was sent." + }, + "sender": { + "type": "string", + "description": "The MXID of the user who sent this event." + }, + "state_key": { + "type": "string", + "description": "Optional. This key will only be present for state events. A unique key which defines the overwriting semantics for this piece of room state." + }, + "type": { + "type": "string", + "description": "The type of event." + }, + "unsigned": { + "type": "object", + "title": "Unsigned", + "description": "Information about this event which was not sent by the originating homeserver", + "properties": { + "age": { + "type": "integer", + "format": "int64", + "description": "Time in milliseconds since the event was sent." + }, + "prev_content": { + "title": "EventContent", + "type": "object", + "description": "Optional. The previous ``content`` for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "replaces_state": { + "type": "string", + "description": "Optional. The event_id of the previous event for this state. This will be present only for state events appearing in the ``timeline``. If this is not a state event, or there is no previous content, this key will be missing." + }, + "transaction_id": { + "type": "string", + "description": "Optional. The transaction ID set when this message was sent. This key will only be present for message events sent by the device calling this API." + } + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 395aed13..7f489423 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,8 +5,8 @@ "type": "array", "description": "List of events", "items": { - "title": "Event", - "type": "object" + "type": "object", + "allOf": [{"$ref": "event.json" }] } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json deleted file mode 100644 index fcf148f3..00000000 --- a/api/client-server/v2_alpha/definitions/room_event_batch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "events": { - "type": "array", - "description": "List of event ids", - "items": { - "type": "string" - } - } - } -} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index ddf8d341..f27a5746 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,14 +1,14 @@ { "type": "object", - "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "allOf": [{"$ref":"definitions/event_batch.json"}], "properties": { "limited": { "type": "boolean", - "description": "Whether there are more events on the server" + "description": "True if the number of events returned was limited by the ``limit`` on the filter" }, "prev_batch": { "type": "string", - "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve earlier events" } } } diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index f5c99508..e9f12a32 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -95,33 +95,26 @@ paths: description: |- Updates to rooms. properties: - joined: - title: Joined + join: + title: Joined Rooms type: object + description: |- + The rooms that the user has joined. additionalProperties: title: Joined Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "core-event-schema/event.json" state: title: State type: object description: |- - The state updates for the room. + Updates to the state, between the time indicated by + the ``since`` parameter, and the start of the + ``timeline`` (or all state up to the start of the + ``timeline``, if ``since`` is not given, or + ``full_state`` is true). allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -140,15 +133,15 @@ paths: allOf: - $ref: "definitions/event_batch.json" account_data: - title: Private User Data + title: Account Data type: object description: |- The private data that this user has attached to this room. allOf: - $ref: "definitions/event_batch.json" - invited: - title: Invited + invite: + title: Invited Rooms type: object description: |- The rooms that the user has been invited to. @@ -174,37 +167,22 @@ paths: ``invite_state``. allOf: - $ref: "definitions/event_batch.json" - archived: - title: Archived + leave: + title: Left rooms type: object description: |- - The rooms that the user has left or been banned from. The - entries in the room_map will lack an ``ephemeral`` key. + The rooms that the user has left or been banned from. additionalProperties: - title: Archived Room + title: Left Room type: object properties: - event_map: - title: EventMap - type: object - description: |- - A map from event ID to events for this room. The - events are referenced from the ``timeline`` and - ``state`` keys for this room. - additionalProperties: - title: Event - description: An event object. - type: object - allOf: - - $ref: "core-event-schema/event.json" state: title: State type: object description: |- - The state updates for the room up to the point when - the user left. + The state updates for the room up to the start of the timeline. allOf: - - $ref: "definitions/room_event_batch.json" + - $ref: "definitions/event_batch.json" timeline: title: Timeline type: object @@ -244,47 +222,43 @@ paths: ] }, "rooms": { - "joined": { + "join": { "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "unsigned": { - "prev_content": {"membership": "invite"} - }, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": { - "body": "I am a fish", - "msgtype": "m.text" - }, - "origin_server_ts": 1417731086797 - } - }, "state": { "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795, + "event_id": "$66697273743031:example.com" + } ] }, "timeline": { "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" + { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "prev_content": {"membership": "invite"}, + "origin_server_ts": 1417731086795, + "event_id": "$7365636s6r6432:example.com" + }, + { + "sender": "@alice:example.com", + "type": "m.room.message", + "age": 124524, + "txn_id": "1234", + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797, + "event_id": "$74686972643033:example.com" + } ], "limited": true, "prev_batch": "t34-23535_0_0" @@ -313,7 +287,7 @@ paths: } } }, - "invited": { + "invite": { "!696r7674:example.com": { "invite_state": { "events": [ @@ -333,6 +307,6 @@ paths: } } }, - "archived": {} + "leave": {} } } diff --git a/drafts/markjh_end_to_end.rst b/drafts/markjh_end_to_end.rst new file mode 100644 index 00000000..1699f79b --- /dev/null +++ b/drafts/markjh_end_to_end.rst @@ -0,0 +1,142 @@ +Goals of Key-Distribution in Matrix +=================================== + +* No Central Authority: Users should not need to trust a central authority + when determining the authenticity of keys. + +* Easy to Add New Devices: It should be easy for a user to start using a + new device. + +* Possible to discover MITM: It should be possible for a user to determine if + they are being MITM. + +* Lost Devices: It should be possible for a user to recover if they lose all + their devices. + +* No Copying Keys: Keys should be per device and shouldn't leave the device + they were created on. + +A Possible Mechanism for Key Distribution +========================================= + +Basic API for setting up keys on a server: + +https://github.com/matrix-org/matrix-doc/pull/24 + +Client shouldn't trust the keys unless they have been verified, e.g by +comparing fingerprints. + +If a user adds a new device it should some yet to be specified protocol +communicate with an old device and obtain a cross-signature from the old +device for its public key. + +The new device can then present the cross-signed key to all the devices +that the user is in conversations with. Those devices should then include +the new device into those conversations. + +If the user cannot cross-sign the new key, e.g. because their old device +is lost or stolen. Then they will need to reauthenticate their conversations +out of band, e.g by comparing fingerprints. + + +Goals of End-to-end encryption in Matrix +======================================== + +* Access to Chat History: Users should be able to see the history of a + conversation on a new device. User should be able to control who can + see their chat history and how much of the chat history they can see. + +* Forward Secrecy of Discarded Chat History: Users should be able to discard + history from their device, once they have discarded the history it should be + impossible for an adversary to recover that history. + +* Forward Secrecy of Future Messages: Users should be able to recover from + disclosure of the chat history on their device. + +* Deniablity of Chat History: It should not be possible to prove to a third + party that a given user sent a message. + +* Authenticity of Chat History: It should be possible to prove amoungst + the members of a chat that a message sent by a user was authored by that + user. + + +Bonus Goals: + +* Traffic Analysis: It would be nice if the protocol was resilient to traffic + or metadata analysis. However it's not something we want to persue if it + harms the usability of the protocol. It might be cool if there was a + way for the user to could specify the trade off between performance and + resilience to traffic analysis that they wanted. + + +A Possible Design for Group Chat using Olm +========================================== + +Protecting the secrecy of history +--------------------------------- + +Each message sent by a client has a 32-bit counter. This counter increments +by one for each message sent by the client. This counter is used to advance a +ratchet. The ratchet is split into a vector four 256-bit values, +:math:`R_{n,j}` for :math:`j \in {0,1,2,3}`. The ratchet can be advanced as +follows: + +.. math:: + \begin{align} + R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\ + R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\ + R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\ + R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\ + R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\ + R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\ + R_{i,3} &= H_1\left(R_{(i-1),3}\right) + \end{align} + +Where :math:`H_1` and :math:`H_2` are different hash functions. For example +:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and +:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`. + +So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`. +Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`. +Every :math:`2^8` iterations :math:`R_{n,3}` is reseeded from :math:`R_{n,2}`. + +This scheme allows the ratchet to be advanced an arbitrary amount forwards +while needing only 1024 hash computations. + +This the value of the ratchet is hashed to generate the keys used to encrypt +each mesage. + +A client can decrypt chat history onwards from the earliest value of the +ratchet it is aware of. But cannot decrypt history from before that point +without reversing the hash function. + +This allows a client to share its ability to decrypt chat history with another +from a point in the conversation onwards by giving a copy of the ratchet at +that point in the conversation. + +A client can discard history by advancing a ratchet to beyond the last message +they want to discard and then forgetting all previous values of the ratchet. + +Proving and denying the authenticity of history +----------------------------------------------- + +Client sign the messages they send using a Ed25519 key generated per +conversation. That key, along with the ratchet key, is distributed +to other clients using 1:1 olm ratchets. Those 1:1 ratchets are started using +Triple Diffie-Hellman which provides authenticity of the messages to the +participants and deniability of the messages to third parties. Therefore +any keys shared over those keys inherit the same levels of deniability and +authenticity. + +Protecting the secrecy of future messages +----------------------------------------- + +A client would need to generate new keys if it wanted to prevent access to +messages beyond a given point in the conversation. It must generate new keys +whenever someone leaves the room. It should generate new keys periodically +anyway. + +The frequency of key generation in a large room may need to be restricted to +keep the frequency of messages broadcast over the individual 1:1 channels +low. diff --git a/drafts/websockets.rst b/drafts/websockets.rst new file mode 100644 index 00000000..bd0ff081 --- /dev/null +++ b/drafts/websockets.rst @@ -0,0 +1,285 @@ +WebSockets API +============== + +Introduction +------------ +This document is a proposal for a WebSockets-based client-server API. It is not +intended to replace the REST API, but rather to complement it and provide an +alternative interface for certain operations. + +The primary goal is to offer a more efficient interface than the REST API: by +using a bidirectional protocol such as WebSockets we can avoid the overheads +involved in long-polling (SSL negotiation, HTTP headers, etc). In doing so we +will reduce the latency between server and client by allowing the server to +send events as soon as they arrive, rather than having to wait for a poll from +the client. + +Handshake +--------- +1. Instead of calling ``/sync``, the client makes a websocket request to + ``/_matrix/client/rN/stream``, passing the query parameters ``access_token`` + and ``since``, and optionally ``filter`` - all of which have the same + meaning as for ``/sync``. + + * The client sets the ``Sec-WebSocket-Protocol`` to ``m.json``. (Servers may + offer alternative encodings; at present only the JSON encoding is + specified but in future we will specify alternative encodings.) + +#. The server returns the websocket handshake; the socket is then connected. + +If the server does not return a valid websocket handshake, this indicates that +the server or an intermediate proxy does not support WebSockets. In this case, +the client should fall back to polling the ``/sync`` REST endpoint. + +Example +~~~~~~~ + +Client request: + +.. code:: http + + GET /_matrix/client/v2_alpha/stream?access_token=123456&since=s72594_4483_1934 HTTP/1.1 + Host: matrix.org + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: m.json + Sec-WebSocket-Version: 13 + Origin: https://matrix.org + +Server response: + +.. code:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: m.json + + +Update Notifications +-------------------- +Once the socket is connected, the server begins streaming updates over the +websocket. The server sends Update notifications about new messages or state +changes. To make it easy for clients to parse, Update notifications have the +same structure as the response to ``/sync``: an object with the following +members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +next_batch string The batch token to supply in the ``since`` param of + the next /sync request. This is not required for + streaming of events over the WebSocket, but is + provided so that clients can reconnect if the + socket is disconnected. +presence Presence The updates to the presence status of other users. +rooms Rooms Updates to rooms. +============= ========== =================================================== + +Example +~~~~~~~ +Message from the server: + +.. code:: json + + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [] + }, + "rooms": { + "join": {}, + "invite": {}, + "leave": {} + } + } + + +Client-initiated operations +--------------------------- + +The client can perform certain operations by sending a websocket message to +the server. Such a "Request" message should be a JSON-encoded object with +the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string A unique identifier for this request +method string Specifies the name of the operation to be + performed; see below for available operations +param object The parameters for the requested operation. +============= ========== =================================================== + +The server responds to a client Request with a Response message. This is a +JSON-encoded object with the following members: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +id string The same as the value in the corresponding Request + object. The presence of the ``id`` field + distinguishes a Response message from an Update + notification. +result object On success, the results of the request. +error object On error, an object giving the resons for the + error. This has the same structure as the "standard + error response" for the Matrix API: an object with + the fields ``errcode`` and ``error``. +============= ========== =================================================== + +Request methods +~~~~~~~~~~~~~~~ +It is not intended that all operations which are available via the REST API +will be available via the WebSockets API, but a few simple, common operations +will be exposed. The initial operations will be as follows. + +``ping`` +^^^^^^^^ +This is a no-op which clients may use to keep their connection alive. + +The request ``params`` and the response ``result`` should be empty. + +``send`` +^^^^^^^^ +Send a message event to a room. The parameters are as follows: + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to send the event to +event_type string **Required.** The type of event to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + +The ``id`` from the Request message is used as the transaction ID by the +server. + +``state`` +^^^^^^^^^ +Update the state on a room. + +============= ========== =================================================== +Parameter Type Description +============= ========== =================================================== +room_id string **Required.** The room to set the state in +event_type string **Required.** The type of event to send. +state_key string **Required.** The state_key for the state to send. +content object **Required.** The content of the event. +============= ========== =================================================== + +The result is as follows: + +============= ========== =================================================== +Key Type Description +============= ========== =================================================== +event_id string A unique identifier for the event. +============= ========== =================================================== + + +Example +~~~~~~~ +Client request: + +.. code:: json + + { + "id": "12345", + "method": "send", + "params": { + "room_id": "!d41d8cd:matrix.org", + "event_type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "hello" + } + } + } + +Server response: + +.. code:: json + + { + "id": "12345", + "result": { + "event_id": "$66697273743031:matrix.org" + } + } + +Alternative server response, in case of error: + +.. code:: json + + { + "id": "12345", + "error": { + "errcode": "M_MISSING_PARAM", + "error": "Missing parameter: event_type" + } + } + + +Rationale +--------- +Alternatives to WebSockets include HTTP/2, CoAP, and simply rolling our own +protocol over raw TCP sockets. However, the need to implement browser-based +clients essentially reduces our choice to WebSockets. HTTP/2 streams will +probably provide an interesting alternative in the future, but current browsers +do not appear to give javascript applications low-level access to the protocol. + +Concerning the continued use of the JSON encoding: we prefer to focus on the +transition to WebSockets initially. Replacing JSON with a compact +representation such as CBOR, MessagePack, or even just compressed JSON will be +a likely extension for the future. The support for negotiation of subprotocols +within WebSockets should make this a simple transition once time permits. + +The number of methods available for client requests is deliberately limited, as +each method requires code to be written to map it onto the equivalent REST +implementation. Some REST methods - for instance, user registration and login - +would be pointless to expose via WebSockets. It is likely, however, that we +will increate the number of methods available via the WebSockets API as it +becomes clear which would be most useful. + +Open questions +-------------- + +Throttling +~~~~~~~~~~ +At least in v2 sync, clients are inherently self-throttling - if they do not +poll quickly enough, events will be dropped from the next result. This proposal +raises the possibility that events will be produced more quickly than they can +be sent to the client; backlogs will build up on the server and/or in the +intermediate network, which will not only lead to high latency on events being +delivered, but will lead to responses to client requests also being delayed. + +We may need to implement some sort of throttling mechanism by which the server +can start to drop events. The difficulty is in knowing when to start dropping +events. A few ideas: + +* Use websocket pings to measure the RTT; if it starts to increase, start + dropping events. But this requires knowledge of the base RTT, and a useful + model of what constitutes an excessive increase. + +* Have the client acknowledge each batch of events, and use a window to ensure + the number of outstanding batches is limited. This is annoying as it requires + the client to have to acknowledge batches - and it's not clear what the right + window size is: we want a big window for long fat networks (think of mobile + clients), but a small one for one with lower latency. + +* Start dropping events if the server's TCP buffer starts filling up. This has + the advantage of delegating the congestion-detection to TCP (which already + has a number of algorithms to deal with it, to greater or lesser + effectiveness), but relies on homeservers being hosted on OSes which use + sensible TCP congestion-avoidance algorithms, and more critically, an ability + to read the fill level of the TCP send buffer. diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 81057049..25b30109 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event may also include an ``invite_room_state`` key **outside the** ``content`` **key**. If present, this contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 05b6ff3c..ba5578bc 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -870,42 +870,22 @@ following values: ``invite`` This room can only be joined if you were invited. -{{membership_http_api}} +{{inviting_http_api}} + +{{joining_http_api}} Leaving rooms ~~~~~~~~~~~~~ -.. TODO-spec - HS deleting rooms they are no longer a part of. Not implemented. - - This is actually Very Tricky. If all clients a HS is serving leave a room, - the HS will no longer get any new events for that room, because the servers - who get the events are determined on the *membership list*. There should - probably be a way for a HS to lurk on a room even if there are 0 of their - members in the room. - - Grace period before deletion? - - Under what conditions should a room NOT be purged? - - A user can leave a room to stop receiving events for that room. A user must have been invited to or have joined the room before they are eligible to leave the room. Leaving a room to which the user has been invited rejects the invite. +Once a user leaves a room, it will no longer appear on the |initialSync|_ API. Whether or not they actually joined the room, if the room is an "invite-only" room they will need to be re-invited before they can re-join -the room. To leave a room, a request should be made to -|/rooms//leave|_ with:: - - {} - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "leave" - } +the room. -See the `Room events`_ section for more information on ``m.room.member``. Once a -user has left a room, that room will no longer appear on the |initialSync|_ API. -If all members in a room leave, that room becomes eligible for deletion. +{{leaving_http_api}} Banning users in a room ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/specification/events.rst b/specification/events.rst index a1aece1c..5a003115 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -30,12 +30,14 @@ formatted for federation by: ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, ``origin``, ``prev_state``. * Adding an ``age`` to the ``unsigned`` object which gives the time in - milliseconds that has ellapsed since the event was sent. -* Adding a ``prev_content`` to the ``unsigned`` object if the event is - a ``state event`` which gives previous content of that state key. + milliseconds that has elapsed since the event was sent. +* Adding ``prev_content`` and ``prev_sender`` to the ``unsigned`` object if the + event is a ``state event``, which give the previous content and previous + sender of that state key * Adding a ``redacted_because`` to the ``unsigned`` object if the event was redacted which gives the event that redacted it. -* Adding a ``transaction_id`` if the event was sent by the client requesting it. +* Adding a ``transaction_id`` to the ``unsigned`` object if the event was sent + by the client requesting it. Events in responses for APIs with the /v1 prefix are generated from an event formatted for the /v2 prefix by: diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 26e040ca..cc9426e2 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -533,6 +533,164 @@ part of the path specifies the kind of query being made, and its query arguments have a meaning specific to that kind of query. The response is a JSON-encoded object whose meaning also depends on the kind of query. + +To join a room:: + + GET .../make_join// + Response: JSON encoding of a join proto-event + + PUT .../send_join// + Response: JSON encoding of the state of the room at the time of the event + +Performs the room join handshake. For more information, see "Joining Rooms" +below. + +Joining Rooms +------------- + +When a new user wishes to join room that the user's homeserver already knows +about, the homeserver can immediately determine if this is allowable by +inspecting the state of the room, and if it is acceptable, it can generate, +sign, and emit a new ``m.room.member`` state event adding the user into that +room. When the homeserver does not yet know about the room it cannot do this +directly. Instead, it must take a longer multi-stage handshaking process by +which it first selects a remote homeserver which is already participating in +that room, and uses it to assist in the joining process. This is the remote +join handshake. + +This handshake involves the homeserver of the new member wishing to join +(referred to here as the "joining" server), the directory server hosting the +room alias the user is requesting to join with, and a homeserver where existing +room members are already present (referred to as the "resident" server). + +In summary, the remote join handshake consists of the joining server querying +the directory server for information about the room alias; receiving a room ID +and a list of join candidates. The joining server then requests information +about the room from one of the residents. It uses this information to construct +a ``m.room.member`` event which it finally sends to a resident server. + +Conceptually these are three different roles of homeserver. In practice the +directory server is likely to be resident in the room, and so may be selected +by the joining server to be the assisting resident. Likewise, it is likely that +the joining server picks the same candidate resident for both phases of event +construction, though in principle any valid candidate may be used at each time. +Thus, any join handshake can potentially involve anywhere from two to four +homeservers, though most in practice will use just two. + +:: + + Client Joining Directory Resident + Server Server Server + + join request --> + | + directory request -------> + <---------- directory response + | + make_join request -----------------------> + <------------------------------- make_join response + | + send_join request -----------------------> + <------------------------------- send_join response + | + <---------- join response + +The first part of the handshake usually involves using the directory server to +request the room ID and join candidates. This is covered in more detail on the +directory server documentation, below. In the case of a new user joining a +room as a result of a received invite, the joining user's homeserver could +optimise this step away by picking the origin server of that invite message as +the join candidate. However, the joining server should be aware that the origin +server of the invite might since have left the room, so should be prepared to +fall back on the regular join flow if this optimisation fails. + +Once the joining server has the room ID and the join candidates, it then needs +to obtain enough information about the room to fill in the required fields of +the ``m.room.member`` event. It obtains this by selecting a resident from the +candidate list, and requesting the ``make_join`` endpoint using a ``GET`` +request, specifying the room ID and the user ID of the new member who is +attempting to join. + +The resident server replies to this request with a JSON-encoded object having a +single key called ``event``; within this is an object whose fields contain some +of the information that the joining server will need. Despite its name, this +object is not a full event; notably it does not need to be hashed or signed by +the resident homeserver. The required fields are: + +==================== ======== ============ + Key Type Description +==================== ======== ============ +``type`` String The value ``m.room.member`` +``auth_events`` List An event-reference list containing the + authorization events that would allow this member + to join +``content`` Object The event content +``depth`` Integer (this field must be present but is ignored; it + may be 0) +``event_id`` String A new event ID specified by the resident + homeserver +``origin`` String The name of the resident homeserver +``origin_server_ts`` Integer A timestamp added by the resident homeserver +``prev_events`` List An event-reference list containing the immediate + predecessor events +``room_id`` String The room ID of the room +``sender`` String The user ID of the joining member +``state_key`` String The user ID of the joining member +==================== ======== ============ + +The ``content`` field itself must be an object, containing: + +============== ====== ============ + Key Type Description +============== ====== ============ +``membership`` String The value ``join`` +============== ====== ============ + +The joining server now has sufficient information to construct the real join +event from these protoevent fields. It copies the values of most of them, +adding (or replacing) the following fields: + +==================== ======= ============ + Key Type Description +==================== ======= ============ +``event_id`` String A new event ID specified by the joining homeserver +``origin`` String The name of the joining homeserver +``origin_server_ts`` Integer A timestamp added by the joining homeserver +==================== ======= ============ + +.. TODO-spec + - Why does the protoevent have an event_id, only for the real event to ignore + it and specify a different one? We should definitely pick one or the other. + +This will be a true event, so the joining server should apply the event-signing +algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` +fields. + +To complete the join handshake, the joining server must now submit this new +event to an resident homeserver, by using the ``send_join`` endpoint. This is +invoked using the room ID and the event ID of the new member event. + +The resident homeserver then accepts this event into the room's event graph, +and responds to the joining server with the full set of state for the newly- +joined room. This is returned as a two-element list, whose first element is the +integer 200, and whose second element is an object which contains the +following keys: + +============== ===== ============ + Key Type Description +============== ===== ============ +``auth_chain`` List A list of events giving the authorization chain for this + join event +``state`` List A complete list of the prevailing state events at the + instant just before accepting the new ``m.room.member`` + event +============== ===== ============ + +.. TODO-spec + - (paul) I don't really understand why the full auth_chain events are given + here. What purpose does it serve expanding them out in full, when surely + they'll appear in the state anyway? + Backfilling ----------- .. NOTE:: @@ -763,6 +921,7 @@ Querying directory information:: servers: list of strings giving the join candidates The list of join candidates is a list of server names that are likely to hold -the given room; these are servers that the requesting server may wish to try -joining with. This list may or may not include the server answering the query. +the given room; these are servers that the requesting server may wish to use as +resident servers as part of the remote join handshake. This list may or may not +include the server answering the query. diff --git a/templating/build.py b/templating/build.py index a35d8a08..77fecf91 100755 --- a/templating/build.py +++ b/templating/build.py @@ -93,6 +93,27 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return '\n\n'.join(output_lines) + def fieldwidths(input, keys, defaults=[], default_width=15): + """ + A template filter to help in the generation of tables. + + Given a list of rows, returns a list giving the maximum length of the + values in each column. + + :param list[dict[str, str]] input: a list of rows. Each row should be a + dict with the keys given in ``keys``. + :param list[str] keys: the keys corresponding to the table columns + :param list[int] defaults: for each column, the default column width. + :param int default_width: if ``defaults`` is shorter than ``keys``, this + will be used as a fallback + """ + def colwidth(key, default): + return reduce(max, (len(row[key]) for row in input), + default if default is not None else default_width) + + results = map(colwidth, keys, defaults) + return results + # make Jinja aware of the templates and filters env = Environment( loader=FileSystemLoader(in_mod.exports["templates"]), @@ -102,6 +123,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): env.filters["indent"] = indent env.filters["indent_block"] = indent_block env.filters["wrap"] = wrap + env.filters["fieldwidths"] = fieldwidths # load up and parse the lowest single units possible: we don't know or care # which spec section will use it, we just need it there in memory for when diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 3f16be3d..8d8c8f0c 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,17 +1,8 @@ +{% import 'tables.tmpl' as tables -%} + {{common_event.title}} Fields {{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} -================== ================= =========================================== - Key Type Description -================== ================= =========================================== -{% for row in common_event.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc | indent(18 - (row.type|length)) |wrap(43) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(common_event.rows, ["Key", "Type", "Description"]) }} diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl index fb876440..324a8f3e 100644 --- a/templating/matrix_templates/templates/events.tmpl +++ b/templating/matrix_templates/templates/events.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.type}}`` {{(4 + event.type | length) * title_kind}} *{{event.typeof}}* @@ -7,18 +9,7 @@ {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ================= =========================================== - {{table.title or "Content"}} Key Type Description -======================= ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(42)}} -{% endfor -%} -======================= ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index d7258e98..1253e66e 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,3 +1,5 @@ +{% import 'tables.tmpl' as tables -%} + ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} {% if "alias_for_path" in endpoint -%} @@ -13,17 +15,7 @@ Request format: {% if (endpoint.req_param_by_loc | length) %} -=========================================== ================= =========================================== - Parameter Value Description -=========================================== ================= =========================================== -{% for loc in endpoint.req_param_by_loc -%} -*{{loc}} parameters* ---------------------------------------------------------------------------------------------------------- -{% for param in endpoint.req_param_by_loc[loc] -%} -{{param.key}}{{param.type|indent(44-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(62)}} -{% endfor -%} -{% endfor -%} -=========================================== ================= =========================================== +{{ tables.split_paramtable(endpoint.req_param_by_loc) }} {% else %} `No parameters` {% endif %} @@ -34,18 +26,7 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -======================= ========================= ========================================== - Param Type Description -======================= ========================= ========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (42 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(24-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(26 - (row.type|length))) |indent_block(50)}} -{% endfor -%} -======================= ========================= ========================================== +{{ tables.paramtable(table.rows) }} {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 18d3492b..87cf4a19 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,21 +1,12 @@ +{% import 'tables.tmpl' as tables -%} + ``{{event.msgtype}}`` {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== - {{table.title or "Content"}} Key Type Description -================== ================= =========================================== -{% for row in table.rows -%} -{# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} -{# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} -{# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} -{% endfor -%} -================== ================= =========================================== +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} {% endfor %} Example: diff --git a/templating/matrix_templates/templates/tables.tmpl b/templating/matrix_templates/templates/tables.tmpl new file mode 100644 index 00000000..aba6c0c4 --- /dev/null +++ b/templating/matrix_templates/templates/tables.tmpl @@ -0,0 +1,104 @@ +{# + # A set of macros for generating RST tables + #} + + +{# + # write a table for a list of parameters. + # + # 'rows' is the list of parameters. Each row should have the keys + # 'key', 'type', and 'desc'. + #} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} +{{ split_paramtable({None: rows}, titles) }} +{% endmacro %} + + +{# + # write a table for the request parameters, split by location. + # 'rows_by_loc' is a map from location to a list of parameters. + # + # As a special case, if a key of 'rows_by_loc' is 'None', no title row is + # written for that location. This is used by the standard 'paramtable' macro. + #} +{% macro split_paramtable(rows_by_loc, + titles=["Parameter", "Type", "Description"]) -%} + +{% set rowkeys = ['key', 'type', 'desc'] %} +{% set titlerow = {'key': titles[0], 'type': titles[1], 'desc': titles[2]} %} + +{# We need the rows flattened into a single list. Abuse the 'sum' filter to + # join arrays instead of add numbers. -#} +{% set flatrows = rows_by_loc.values()|sum(start=[]) -%} + +{# Figure out the widths of the columns. The last column is always 50 characters + # wide; the others default to 10, but stretch if there is wider text in the + # column. -#} +{% set fieldwidths = (([titlerow] + flatrows) | + fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} + +{{ tableheader(fieldwidths) }} +{{ tablerow(fieldwidths, titlerow, rowkeys) }} +{{ tableheader(fieldwidths) }} +{% for loc in rows_by_loc -%} + +{% if loc != None -%} +{{ tablespan(fieldwidths, "*" ~ loc ~ " parameters*") }} +{% endif -%} + +{% for row in rows_by_loc[loc] -%} +{{ tablerow(fieldwidths, row, rowkeys) }} +{% endfor -%} +{% endfor -%} + +{{ tableheader(fieldwidths) }} +{% endmacro %} + + + +{# + # Write a table header row, for the given column widths + #} +{% macro tableheader(widths) -%} +{% for arg in widths -%} +{{"="*arg}} {% endfor -%} +{% endmacro %} + + + +{# + # Write a normal table row. Each of 'widths' and 'keys' should be sequences + # of the same length; 'widths' defines the column widths, and 'keys' the + # attributes of 'row' to look up for values to put in the columns. + #} +{% macro tablerow(widths, row, keys) -%} +{% for key in keys -%} +{% set value=row[key] -%} +{% if not loop.last -%} + {# the first few columns need space after them -#} + {{ value }}{{" "*(1+widths[loop.index0]-value|length) -}} +{% else -%} + {# the last column needs wrapping and indenting (by the sum of the widths of + the preceding columns, plus the number of preceding columns (for the + separators)) -#} + {{ value | wrap(widths[loop.index0]) | + indent_block(widths[0:-1]|sum + loop.index0) -}} +{% endif -%} +{% endfor -%} +{% endmacro %} + + + + +{# + # write a tablespan row. This is a single value which spans the entire table. + #} +{% macro tablespan(widths, value) -%} +{{value}} +{# we write a trailing space to stop the separator being misinterpreted + # as a header line. -#} +{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} + + + + diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index adb7f427..e5a7c319 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -122,10 +122,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "x-pattern", "string" ) value_type = "{%s: %s}" % (key, nested_object[0]["title"]) + value_id = "%s: %s" % (key, nested_object[0]["title"]) if not nested_object[0].get("no-table"): tables += nested_object else: - value_type = "{string: %s}" % prop_val + value_type = "{string: %s}" % (prop_val,) + value_id = "string: %s" % (prop_val,) else: nested_object = get_json_schema_object_fields( props[key_name], @@ -133,6 +135,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) if not nested_object[0].get("no-table"): tables += nested_object @@ -145,12 +148,14 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] + value_id = "%s" % (nested_object[0]["title"],) tables += nested_object else: value_type = props[key_name]["items"]["type"] if isinstance(value_type, list): value_type = " or ".join(value_type) value_type = "[%s]" % value_type + value_id = "%s" % (value_type,) array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: @@ -164,6 +169,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals ) else: value_type = props[key_name]["type"] + value_id = props[key_name]["type"] if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" @@ -184,6 +190,7 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals fields["rows"].append({ "key": key_name, "type": value_type, + "id": value_id, "required": required, "desc": desc, "req_str": "**Required.** " if required else "" @@ -313,10 +320,16 @@ class MatrixUnits(Units): if req_tables > 1: for table in req_tables[1:]: - nested_key_name = [ - s["key"] for s in req_tables[0]["rows"] if - s["type"] == ("{%s}" % (table["title"],)) - ][0] + nested_key_name = { + "key": s["key"] + for rtable in req_tables + for s in rtable["rows"] + if s["id"] == table["title"] + }.get("key", None) + + if nested_key_name is None: + raise Exception("Failed to find table for %r" % (table["title"],)) + for row in table["rows"]: row["key"] = "%s.%s" % (nested_key_name, row["key"])