From 13eddd456f8277c9aec1743f5541d43f38b016a1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 26 Oct 2015 15:52:45 +0000 Subject: [PATCH 01/36] Proposal for adding full_state param to v2 sync --- api/client-server/v2_alpha/sync.yaml | 20 ++++++++++++++++++-- specification/client_server_api.rst | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 266f27bc..5acbf583 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -22,7 +22,7 @@ paths: summary: Synchronise the client's state and receive new messages. description: |- Synchronise the client's state with the latest state on the server. - Client's use this API when they first log in to get an initial snapshot + Clients use this API when they first log in to get an initial snapshot of the state on the server, and then continue to call this API to get incremental deltas to the state, and to receive new messages. security: @@ -40,6 +40,22 @@ paths: description: |- A point in time to continue a sync from. x-example: "s72594_4483_1934" + - in: query + name: full_state + type: boolean + description: |- + Controls whether to include the full state for all rooms the user + is a member of. + + If this is set to ``true``, then all state events will be returned, + even if ``since`` is non-empty. The timeline will still be limited + by the ``since`` parameter. + + If ``false``, and ``since`` is non-empty, only state which has + changed since the point indicated by ``since`` will be returned. + + By default, this is ``false``. + x-example: "false" - in: query name: set_presence type: string @@ -135,7 +151,7 @@ paths: type: object description: |- The state of a room that the user has been invited - to. These state events may only have the `sender``, + to. These state events may only have the ``sender``, ``type``, ``state_key`` and ``content`` keys present. These events do not replace any state that the client already has for the room, for example if diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 09c6765c..b02dbf28 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -659,6 +659,8 @@ This API also returns an ``end`` token which can be used with the event stream. {{sync_http_api}} +{{v2_sync_http_api}} + Getting events for a room ~~~~~~~~~~~~~~~~~~~~~~~~~ From 71874870c8b66aa1f71247b74d9bb6453880e015 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Oct 2015 17:25:33 +0000 Subject: [PATCH 02/36] Enable syntax highlighting for example http requests --- scripts/css/codehighlight.css | 14 ++++++++++++-- .../matrix_templates/templates/http-api.tmpl | 4 +++- templating/matrix_templates/units.py | 12 +++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/scripts/css/codehighlight.css b/scripts/css/codehighlight.css index 5c9b0c36..fafc43f4 100644 --- a/scripts/css/codehighlight.css +++ b/scripts/css/codehighlight.css @@ -1,6 +1,16 @@ pre.code .comment, code .comment { color: green } pre.code .keyword, code .keyword { color: darkred; font-weight: bold } pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold } -pre.code .literal.number, code .literal.number { color: blue } pre.code .name.tag, code .name.tag { color: darkgreen } -pre.code .literal.string, code .literal.string { color: darkblue } +pre.code .literal, code .literal { color: darkblue } +pre.code .literal.number, code .literal.number { color: blue } + + +/* HTTP Methods have class "name function" */ +pre.code.http .name.function, code.http .name.function { color: black; font-weight: bold } +/* HTTP Paths have class "name namespace" */ +pre.code.http .name.namespace, code.http .name.namespace { color: darkgreen } +/* HTTP "HTTP" strings have class "keyword reserved" */ +pre.code.http .keyword.reserved, code.http .keyword.reserved { color: black; font-weight: bold } +/* HTTP Header names have class "name attribute" */ +pre.code.http .name.attribute, code.http .name.attribute { color: black; font-weight: bold } diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 2812cf7b..86eacb14 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -47,7 +47,9 @@ Response format: {% endfor %} {% endif -%} -Example request:: +Example request: + +.. code:: http {{endpoint.example.req | indent_block(2)}} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c4342141..be113b9c 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -336,9 +336,15 @@ class MatrixUnits(Units): elif param["in"] == "query": qps[param["name"]] = param["x-example"] query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) - endpoint["example"]["req"] = "%s %s%s\n%s" % ( - method.upper(), path_template, query_string, body - ) + if body: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( + method.upper(), path_template, query_string, body + ) + else: + endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % ( + method.upper(), path_template, query_string + ) + else: self.log( "The following parameters are missing examples :( \n %s" % From ece42688d089efbbe69e4d6c7b07bb2d878ec7b3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 26 Oct 2015 19:20:58 +0000 Subject: [PATCH 03/36] Clarify the interaction between full_state and timeout. --- api/client-server/v2_alpha/sync.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 5acbf583..01308405 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -49,7 +49,9 @@ paths: If this is set to ``true``, then all state events will be returned, even if ``since`` is non-empty. The timeline will still be limited - by the ``since`` parameter. + by the ``since`` parameter. In this case, the ``timeout`` parameter + will be ignored and the query will return immediately, possibly with + an empty timeline. If ``false``, and ``since`` is non-empty, only state which has changed since the point indicated by ``since`` will be returned. From 745e60757768b296c3136b761b244e1167c32a2b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 28 Oct 2015 14:49:50 +0000 Subject: [PATCH 04/36] Use 'room' or 'room ID' instead of 'context' when describing federation protocol --- specification/server_server_api.rst | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66367cb0..26e040ca 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -15,9 +15,9 @@ There are three main kinds of communication that occur between home servers: Persisted Data Units (PDUs): These events are broadcast from one home server to any others that have - joined the same "context" (namely, a Room ID). They are persisted in + joined the same room (identified by Room ID). They are persisted in long-term storage and record the history of messages and state for a - context. + room. Like email, it is the responsibility of the originating server of a PDU to deliver that event to its recipient servers. However PDUs are signed @@ -26,7 +26,7 @@ Persisted Data Units (PDUs): Ephemeral Data Units (EDUs): These events are pushed between pairs of home servers. They are not - persisted and are not part of the history of a "context", nor does the + persisted and are not part of the history of a room, nor does the receiving home server have to reply to them. Queries: @@ -338,11 +338,11 @@ PDUs All PDUs have: -- An ID -- A context +- An ID to identify the PDU itself +- A room ID that it relates to - A declaration of their type -- A list of other PDU IDs that have been seen recently on that context - (regardless of which origin sent them) +- A list of other PDU IDs that have been seen recently in that room (regardless + of which origin sent them) Required PDU Fields @@ -351,7 +351,7 @@ Required PDU Fields ==================== ================== ======================================= Key Type Description ==================== ================== ======================================= -``context`` String Event context identifier +``context`` String Room identifier ``user_id`` String The ID of the user sending the PDU ``origin`` String DNS name of homeserver that created this PDU @@ -363,7 +363,7 @@ Required PDU Fields ``content`` Object The content of the PDU. ``prev_pdus`` List of (String, The originating homeserver, PDU ids and String, Object) hashes of the most recent PDUs the - Triplets homeserver was aware of for the context + Triplets homeserver was aware of for the room when it made this PDU ``depth`` Integer The maximum depth of the previous PDUs plus one @@ -440,7 +440,7 @@ keys exist to support this: EDUs ---- -EDUs, by comparison to PDUs, do not have an ID, a context, or a list of +EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and destination home server names, and the actual nested content. @@ -491,23 +491,23 @@ Retrieves a given PDU from the server. The response will contain a single new Transaction, inside which will be the requested PDU. -To fetch all the state of a given context:: +To fetch all the state of a given room:: - GET .../state// + GET .../state// Response: JSON encoding of a single Transaction containing multiple PDUs -Retrieves a snapshot of the entire current state of the given context. The +Retrieves a snapshot of the entire current state of the given room. The response will contain a single Transaction, inside which will be a list of PDUs that encode the state. -To backfill events on a given context:: +To backfill events on a given room:: - GET .../backfill// + GET .../backfill// Query args: v, limit Response: JSON encoding of a single Transaction containing multiple PDUs Retrieves a sliding-window history of previous PDUs that occurred on the given -context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that +room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that preceded it are retrieved, up to a total number given by the "limit" argument. These are then returned in a new Transaction containing all of the PDUs. From 6c1df04b4b99e027ba0750af785a335b08133fc8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 29 Oct 2015 02:11:47 +0000 Subject: [PATCH 05/36] typo --- api/client-server/v2_alpha/filter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 37a0a3aa..0c2761b7 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -56,7 +56,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] }, - "emphemeral": { + "ephemeral": { "types": ["m.receipt", "m.typing"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -120,7 +120,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] }, - "emphemeral": { + "ephemeral": { "types": ["m.receipt", "m.typing"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From d067e50af5990177199a6f3adb0617c2a8ee8836 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:38:33 +0000 Subject: [PATCH 06/36] Document the differences in event formatting between the v1 and v2 client APIs --- specification/events.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/specification/events.rst b/specification/events.rst index 7d64882b..d2428b6b 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -12,6 +12,41 @@ server-server and application-service APIs, and are described below. {{common_state_event_fields}} +Differences between /v1 and /v2 events +-------------------------------------- + +There are a few differences between how events are formatted for sending +between servers over federation and how they are formatted for sending between +a server and its clients. + +Additionally there are a few differences between the format of events in the +responses to client APIs with a /v1 prefix and responses APIs with a /v2 +prefix. + +Events in responses for APIs with the /v2 prefix are generated from an event +formatted for federation by: + +* Removing the following keys: + ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``, + ``origin``, ``prev_state``. +* Adding an ``age`` to the ``unsigned`` object which gives the time in + milliseconds that has ellapsed since the event was sent. +* Adding a ``prev_content`` to the ``unsigned`` object if the event is + a ``state event`` which gives previous content of that state key. +* Adding a ``redacted_because`` to the ``unsigned`` object if the event was + redacted which gives the event that redacted it. +* Adding a ``transaction_id`` if the event was sent by the client requesting it. + +Events in responses for APIs with the /v1 prefix are generated from an event +formatted for the /v2 prefix by: + +* Moving the folling keys from the ``unsigned`` object to the top level event + object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. +* Removing the ``unsigned`` object. +* If the event was an ``m.room.member`` with ``membership`` set to ``invite`` + then adding a ``invite_room_state`` key to the top level event object. + + Size limits ----------- From 451801bf38159aacc5ee2e05d0230a5b89382712 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:40:05 +0000 Subject: [PATCH 07/36] Add an example of ``prev_content`` in ``unsigned`` to v2 /sync --- api/client-server/v2_alpha/sync.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 01308405..a2d5a2b8 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -241,6 +241,9 @@ paths: "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "join"}, + "unsigned": { + "prev_content": {"membership": "invite"} + }, "origin_server_ts": 1417731086795 }, "$74686972643033:example.com": { From d297d83151cf44b785ca5d671ddf4bc8de5f4974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Oct 2015 18:45:53 +0000 Subject: [PATCH 08/36] Mention that sender is renamed to user_id in v1 --- specification/events.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/events.rst b/specification/events.rst index d2428b6b..a1aece1c 100644 --- a/specification/events.rst +++ b/specification/events.rst @@ -43,6 +43,7 @@ formatted for the /v2 prefix by: * Moving the folling keys from the ``unsigned`` object to the top level event object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``. * Removing the ``unsigned`` object. +* Rename the ``sender`` key to ``user_id``. * If the event was an ``m.room.member`` with ``membership`` set to ``invite`` then adding a ``invite_room_state`` key to the top level event object. From 91eb25b76d12b7936583e7a3c482d5b3c066245e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Oct 2015 15:45:46 +0000 Subject: [PATCH 09/36] Include the full schema for an http API in the docs by resolving references to other files --- templating/matrix_templates/units.py | 66 +++++++++++++++++++++------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index be113b9c..74aee26c 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -28,7 +28,25 @@ ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" -def get_json_schema_object_fields(obj, enforce_title=False): +def resolve_references(path, schema): + if isinstance(schema, dict): + result = {} + for key, value in schema.items(): + if key == "$ref": + path = os.path.join(os.path.dirname(path), value) + with open(path) as f: + schema = json.load(f) + return resolve_references(path, schema) + else: + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + +def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) if obj.get("type") != "object": @@ -36,9 +54,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): "get_json_schema_object_fields: Object %s isn't an object." % obj ) if enforce_title and not obj.get("title"): - raise Exception( - "get_json_schema_object_fields: Nested object %s doesn't have a title." % obj - ) + # Force a default titile of "NO_TITLE" to make it obvious in the + # specification output which parts of the schema are missing a title + obj["title"] = 'NO_TITLE' required_keys = obj.get("required") if not required_keys: @@ -73,9 +91,15 @@ def get_json_schema_object_fields(obj, enforce_title=False): "Object %s has no properties or parents." % obj ) if not props: # parents only + if include_parents: + if obj["title"] == "NO_TITLE" and parents[0].get("title"): + obj["title"] = parents[0].get("title") + props = parents[0].get("properties") + + if not props: return [{ "title": obj["title"], - "parent": parents[0]["$ref"], + "parent": parents[0].get("$ref"), "no-table": True }] @@ -91,7 +115,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): if prop_val == "object": nested_object = get_json_schema_object_fields( props[key_name]["additionalProperties"], - enforce_title=True + enforce_title=True, + include_parents=include_parents, ) key = props[key_name]["additionalProperties"].get( "x-pattern", "string" @@ -103,8 +128,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( - props[key_name], - enforce_title=True + props[key_name], + enforce_title=True, + include_parents=include_parents, ) value_type = "{%s}" % nested_object[0]["title"] @@ -114,8 +140,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): # if the items of the array are objects then recurse if props[key_name]["items"]["type"] == "object": nested_object = get_json_schema_object_fields( - props[key_name]["items"], - enforce_title=True + props[key_name]["items"], + enforce_title=True, + include_parents=include_parents, ) value_type = "[%s]" % nested_object[0]["title"] tables += nested_object @@ -159,7 +186,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): class MatrixUnits(Units): - def _load_swagger_meta(self, api, group_name): + def _load_swagger_meta(self, filepath, api, group_name): endpoints = [] for path in api["paths"]: for method in api["paths"][path]: @@ -262,7 +289,10 @@ class MatrixUnits(Units): if is_array_of_objects: req_obj = req_obj["items"] - req_tables = get_json_schema_object_fields(req_obj) + req_tables = get_json_schema_object_fields( + resolve_references(filepath, req_obj), + include_parents=True, + ) if req_tables > 1: for table in req_tables[1:]: @@ -379,7 +409,10 @@ class MatrixUnits(Units): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_json_schema_object_fields(schema) + res_tables = get_json_schema_object_fields( + resolve_references(filepath, schema), + include_parents=True, + ) for table in res_tables: if "no-table" not in table: endpoint["res_tables"].append(table) @@ -445,13 +478,16 @@ class MatrixUnits(Units): if not filename.endswith(".yaml"): continue self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: + filepath = os.path.join(path, filename) + with open(filepath, "r") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") if is_v2: group_name = "v2_" + group_name api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) + api["__meta"] = self._load_swagger_meta( + filepath, api, group_name + ) apis[group_name] = api return apis From 2e3a0b4e00798b672fe3282e24cc1e1c34cd8b0b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 2 Nov 2015 15:26:06 +0000 Subject: [PATCH 10/36] Specify guest accounts --- api/client-server/v2_alpha/registration.yaml | 17 +++++++++++++++++ templating/build.py | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/api/client-server/v2_alpha/registration.yaml b/api/client-server/v2_alpha/registration.yaml index 2bd86e73..681654ae 100644 --- a/api/client-server/v2_alpha/registration.yaml +++ b/api/client-server/v2_alpha/registration.yaml @@ -17,7 +17,24 @@ paths: summary: Register for an account on this homeserver. description: |- Register for an account on this homeserver. + + There are two kinds of user account: + + - `user` accounts. These accounts may use the full API described in this specification. + + - `guest` accounts. These accounts may have limited permissions and may not be supported by all servers. + parameters: + - in: query + name: kind + type: string + x-example: guest + required: false + default: user + enum: + - guest + - user + description: The kind of account to register. Defaults to `user`. - in: body name: body schema: diff --git a/templating/build.py b/templating/build.py index b91e1da2..a35d8a08 100755 --- a/templating/build.py +++ b/templating/build.py @@ -84,6 +84,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) output_lines = [wrapper.fill(line) for line in input_lines] + + for i in range(len(output_lines)): + line = output_lines[i] + in_bullet = line.startswith("- ") + if in_bullet: + output_lines[i] = line.replace("\n", "\n " + initial_indent) + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters From ebc02371095be8789051ac0a47de99803f961606 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:35:44 +0000 Subject: [PATCH 11/36] Add the missing titles to the schema --- .../v1/definitions/push_rule.json | 3 +- .../v1/definitions/push_ruleset.json | 37 +++++++++++-------- .../v2_alpha/definitions/event_batch.json | 1 + .../schema/v1/core-event-schema/event.json | 1 + .../v1/core-event-schema/state_event.json | 1 + event-schemas/schema/v1/m.room.member | 1 + 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/definitions/push_rule.json b/api/client-server/v1/definitions/push_rule.json index 4df93f67..d10f11f8 100644 --- a/api/client-server/v1/definitions/push_rule.json +++ b/api/client-server/v1/definitions/push_rule.json @@ -1,4 +1,5 @@ { + "title": "PushRule", "type": "object", "properties": { "default": { @@ -17,4 +18,4 @@ "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v1/definitions/push_ruleset.json b/api/client-server/v1/definitions/push_ruleset.json index e0372701..529ebeed 100644 --- a/api/client-server/v1/definitions/push_ruleset.json +++ b/api/client-server/v1/definitions/push_ruleset.json @@ -1,60 +1,65 @@ { - "type": "object", + "type": "object", "properties": { "content": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "override": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "sender": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "underride": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" - }, + }, "room": { "items": { - "type": "object", + "type": "object", + "title": "PushRule", "allOf": [ { "$ref": "push_rule.json" } ] - }, + }, "type": "array" } } -} \ No newline at end of file +} diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index 75762d75..395aed13 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -5,6 +5,7 @@ "type": "array", "description": "List of events", "items": { + "title": "Event", "type": "object" } } diff --git a/event-schemas/schema/v1/core-event-schema/event.json b/event-schemas/schema/v1/core-event-schema/event.json index e73aec80..f9103715 100644 --- a/event-schemas/schema/v1/core-event-schema/event.json +++ b/event-schemas/schema/v1/core-event-schema/event.json @@ -5,6 +5,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body." }, "type": { diff --git a/event-schemas/schema/v1/core-event-schema/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json index 88b4900a..5809cf7f 100644 --- a/event-schemas/schema/v1/core-event-schema/state_event.json +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -11,6 +11,7 @@ "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'." }, "prev_content": { + "title": "EventContent", "type": "object", "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing." } diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 45f2ad70..0bd8cd1e 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -8,6 +8,7 @@ "properties": { "content": { "type": "object", + "title": "EventContent", "properties": { "membership": { "type": "string", From 8322151661f8c8a7cb6cfa3fe1d22ab83ce55642 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 3 Nov 2015 19:42:49 +0000 Subject: [PATCH 12/36] Don't put a space when appending the "Must be" strings to the desciption if there isn't a description, otherwise it will mess up the indent --- templating/matrix_templates/units.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 74aee26c..9aa9489a 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -164,12 +164,16 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if props[key_name].get("enum"): if len(props[key_name].get("enum")) > 1: value_type = "enum" + if desc: + desc += " " desc += ( - " One of: %s" % json.dumps(props[key_name]["enum"]) + "One of: %s" % json.dumps(props[key_name]["enum"]) ) else: + if desc: + desc += " " desc += ( - " Must be '%s'." % props[key_name]["enum"][0] + "Must be '%s'." % props[key_name]["enum"][0] ) if isinstance(value_type, list): value_type = " or ".join(value_type) From e49ea9015f4ee5f86b2d401190ec39517c5ae49b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:39:36 +0000 Subject: [PATCH 13/36] Deduplicate tables with the same title --- templating/matrix_templates/units.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9aa9489a..9cc68f49 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -185,7 +185,17 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "desc": desc, "req_str": "**Required.** " if required else "" }) - return tables + + titles = set() + filtered = [] + for table in tables: + if table.get("title") in titles: + continue + + titles.add(table.get("title")) + filtered.append(table) + + return filtered class MatrixUnits(Units): From 8070489080afdc08179087c64b1f6a4a261c224d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:44:20 +0000 Subject: [PATCH 14/36] Handle lists of types in arrays --- templating/matrix_templates/units.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 9cc68f49..cec5b83e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -147,7 +147,10 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals value_type = "[%s]" % nested_object[0]["title"] tables += nested_object else: - value_type = "[%s]" % props[key_name]["items"]["type"] + value_type = props[key_name]["items"]["type"] + if isinstance(value_type, list): + value_type = " or ".join(value_type) + value_type = "[%s]" % value_type array_enums = props[key_name]["items"].get("enum") if array_enums: if len(array_enums) > 1: From c7199463704709ca4c0dc8a3b60572210f464c84 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 4 Nov 2015 11:51:50 +0000 Subject: [PATCH 15/36] Enable syntax highlighting for message type examples --- templating/matrix_templates/templates/msgtypes.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index f7862451..18d3492b 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -18,6 +18,8 @@ ================== ================= =========================================== {% endfor %} -Example:: +Example: + +.. code:: json {{example | jsonify(4, 4)}} From 2fd5fc39a032dbce80a3aba4361f79f553cc5b85 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 10:55:11 +0000 Subject: [PATCH 16/36] Add spec for calculating display names for rooms and users Merged from https://github.com/matrix-org/matrix-doc/pull/145 --- specification/modules/instant_messaging.rst | 125 +++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 09fcb843..7a2142dd 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -116,12 +116,58 @@ of the client-server API will resolve this by attaching the transaction ID of th sending request to the event itself. +Calculating the display name for a user +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show the human-readable display name of a room member as +part of a membership list, or when they send a message. However, different +members may have conflicting display names. Display names MUST be disambiguated +before showing them to the user, in order to prevent spoofing of other users. + +To ensure this is done consistently across clients, clients SHOULD use the +following algorithm to calculate a disambiguated display name for a given user: + +1. Inspect the ``m.room.member`` state event for the relevant user id. +2. If the ``m.room.member`` state event has no ``displayname`` field, or if + that field has a ``null`` value, use the raw user id as the display + name. Otherwise: +3. If the ``m.room.member`` event has a ``displayname`` which is unique among + members of the room with ``membership: join`` or ``membership: invite``, use + the given ``displayname`` as the user-visible display name. Otherwise: +4. The ``m.room.member`` event has a non-unique ``displayname``. This should be + disambiguated using the user id, for example "display name + (@id:homeserver.org)". + +Developers should take note of the following when implementing the above +algorithm: + +* The user-visible display name of one member can be affected by changes in the + state of another member. For example, if ``@user1:matrix.org`` is present in + a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins + the room, also with ``displayname: Alice``, *both* users must be given + disambiguated display names. Similarly, when one of the users then changes + their display name, there is no longer a clash, and *both* users can be given + their chosen display name. Clients should be alert to this possibility and + ensure that all affected users are correctly renamed. + +* The display name of a room may also be affected by changes in the membership + list. This is due to the room name sometimes being based on user display + names (see `Calculating the display name for a room`_). + +* If the entire membership list is searched for clashing display names, this + leads to an O(N^2) implementation for building the list of room members. This + will be very inefficient for rooms with large numbers of members. It is + recommended that client implementations maintain a hash table mapping from + ``displayname`` to a list of room members using that name. Such a table can + then be used for efficient calculation of whether disambiguation is needed. + + Displaying membership information with messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Clients may wish to show the display name and avatar URL of the room member who sent a message. This can be achieved by inspecting the ``m.room.member`` state -event for that user ID. +event for that user ID (see `Calculating the display name for a user`_). When a user paginates the message history, clients may wish to show the **historical** display name and avatar URL for a room member. This is possible @@ -133,6 +179,83 @@ events update the old state. When paginated events are processed sequentially, the old state represents the state of the room *at the time the event was sent*. This can then be used to set the historical display name and avatar URL. + +Calculating the display name for a room +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clients may wish to show a human-readable name for a room. There are a number +of possibilities for choosing a useful name. To ensure that rooms are named +consistently across clients, clients SHOULD use the following algorithm to +choose a name: + +1. If the room has an `m.room.name`_ state event, use the name given by that + event. + +#. If the room has an `m.room.canonical_alias`_ state event, use the alias + given by that event. + +#. If neither of the above events are present, a name should be composed based + on the members of the room. Clients should consider `m.room.member`_ events + for users other than the logged-in user, with ``membership: join`` or + ``membership: invite``. + + .. _active_members: + + i. If there is only one such event, the display name for the room should be + the `disambiguated display name`_ of the corresponding user. + + #. If there are two such events, they should be lexicographically sorted by + their ``state_key`` (i.e. the corresponding user IDs), and the display + name for the room should be the `disambiguated display name`_ of both + users: " and ", or a localised variant thereof. + + #. If there are three or more such events, the display name for the room + should be based on the disambiguated display name of the user + corresponding to the first such event, under a lexicographical sorting + according to their ``state_key``. The display name should be in the + format " and others" (or a localised variant thereof), where N + is the number of `m.room.member`_ events with ``membership: join`` or + ``membership: invite``, excluding the logged-in user and "user1". + + For example, if Alice joins a room, where Bob (whose user id is + ``@superuser:example.com``), Carol (user id ``@carol:example.com``) and + Dan (user id ``@dan:matrix.org``) are in conversation, Alice's + client should show the room name as "Carol and 2 others". + + .. TODO-spec + Sorting by user_id certainly isn't ideal, as IDs at the start of the + alphabet will end up dominating room names: they will all be called + "Arathorn and 15 others". Furthermore - user_ids are not necessarily + ASCII, which means we need to either specify a collation order, or specify + how to choose one. + + Ideally we might sort by the time when the user was first invited to, or + first joined the room. But we don't have this information. + +#. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and + no active members other than the current user, clients should consider + ``m.room.member`` events with ``membership: leave``. If such events exist, a + display name such as "Empty room (was and others)" (or a + localised variant thereof) should be used, following similar rules as for + active members (see `above `_). + +#. A complete absence of ``m.room.name``, ``m.room.canonical_alias``, and + ``m.room.member`` events is likely to indicate a problem with creating the + room or synchronising the state table; however clients should still handle + this situation. A display name such as "Empty room" (or a localised variant + thereof) should be used in this situation. + +.. _`disambiguated display name`: `Calculating the display name for a user`_ + +Clients SHOULD NOT use `m.room.aliases`_ events as a source for room names, as +it is difficult for clients to agree on the best alias to use, and aliases can +change unexpectedly. + +.. TODO-spec + How can we make this less painful for clients to implement, without forcing + an English-language implementation on them all? + + Server behaviour ---------------- From 8cba11b1cd6f70906aa3f8ce03d4332d469af13f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Nov 2015 11:06:31 +0000 Subject: [PATCH 17/36] Add some links to spec bugs for display names Just added a couple of TODO comments to useful jira bugs --- specification/modules/instant_messaging.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 7a2142dd..a58c762f 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -138,6 +138,11 @@ following algorithm to calculate a disambiguated display name for a given user: disambiguated using the user id, for example "display name (@id:homeserver.org)". + .. TODO-spec + what does it mean for a ``displayname`` to be 'unique'? Are we + case-sensitive? Do we care about homograph attacks? See + https://matrix.org/jira/browse/SPEC-221. + Developers should take note of the following when implementing the above algorithm: @@ -232,6 +237,8 @@ choose a name: Ideally we might sort by the time when the user was first invited to, or first joined the room. But we don't have this information. + See https://matrix.org/jira/browse/SPEC-267 for further discussion. + #. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and no active members other than the current user, clients should consider ``m.room.member`` events with ``membership: leave``. If such events exist, a From 161441fa3ae83db985c51c9f1c8a4cb96494bd45 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 18:11:20 +0000 Subject: [PATCH 18/36] Update 3pid spec based on new implementation --- api/client-server/v1/membership.yaml | 110 +-------------- .../v1/third_party_membership.yaml | 127 ++++++++++++++++++ event-schemas/examples/v1/m.room.member | 17 +-- event-schemas/schema/v1/m.room.member | 20 +-- specification/modules/anonymous_access.rst | 50 +++++++ specification/modules/third_party_invites.rst | 85 ++++++++---- 6 files changed, 246 insertions(+), 163 deletions(-) create mode 100644 api/client-server/v1/third_party_membership.yaml create mode 100644 specification/modules/anonymous_access.rst diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index f8dfdea5..c089e154 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -73,9 +73,12 @@ paths: post: summary: Invite a user to participate in a particular room. description: |- + .. _invite-by-user-id-endpoint: + *Note that there are two forms of this API, which are documented separately. This version of the API requires that the inviter knows the Matrix - identifier of the invitee.* + identifier of the invitee. The other is documented in the* + `third party invites section`_. This API invites a user to participate in a particular room. They do not start participating in the room until they actually join the @@ -86,6 +89,8 @@ paths: If the user was invited to the room, the home server will append a ``m.room.member`` event to the room. + + .. _third party invites section: `invite-by-third-party-id-endpoint`_ security: - accessToken: [] parameters: @@ -132,106 +137,3 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" - - "/rooms/{roomId}/invite": - post: - summary: Invite a user to participate in a particular room. - description: |- - *Note that there are two forms of this API, which are documented separately. - This version of the API does not require that the inviter know the Matrix - identifier of the invitee, and instead relies on third party identifiers. - The homeserver uses an identity server to perform the mapping from - third party identifier to a Matrix identifier.* - - This API invites a user to participate in a particular room. - They do not start participating in the room until they actually join the - room. - - Only users currently in a particular room can invite other users to - join that room. - - If the identity server did know the Matrix user identifier for the - third party identifier, the home server will append a ``m.room.member`` - event to the room. - - If the identity server does not know a Matrix user identifier for the - passed third party identifier, the homeserver will issue an invitation - which can be accepted upon providing proof of ownership of the third - party identifier. This is achieved by the identity server generating a - token, which it gives to the inviting homeserver. The homeserver will - add an ``m.room.third_party_invite`` event into the graph for the room, - containing that token. - - When the invitee binds the invited third party identifier to a Matrix - user ID, the identity server will give the user a list of pending - invitations, each containing: - - - The room ID to which they were invited - - - The token given to the homeserver - - - A signature of the token, signed with the identity server's private key - - - The matrix user ID who invited them to the room - - If a token is requested from the identity server, the home server will - append a ``m.room.third_party_invite`` event to the room. - security: - - accessToken: [] - parameters: - - in: path - type: string - name: roomId - description: The room identifier (not alias) to which to invite the user. - required: true - x-example: "!d41d8cd:matrix.org" - - in: body - name: body - required: true - schema: - type: object - example: |- - { - "id_server": "matrix.org", - "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" - } - properties: - id_server: - type: string - description: The hostname+port of the identity server which should be used for third party identifier lookups. - medium: - type: string - # TODO: Link to identity service spec when it eixsts - description: The kind of address being passed in the address field, for example ``email``. - address: - type: string - description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] - responses: - 200: - description: The user has been invited to join the room. - examples: - application/json: |- - {} - schema: - type: object - 403: - description: |- - You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: - - - The invitee has been banned from the room. - - The invitee is already a member of the room. - - The inviter is not currently in the room. - - The inviter's power level is insufficient to invite users to the room. - examples: - application/json: |- - {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} - 429: - description: This request was rate-limited. - schema: - "$ref": "definitions/error.yaml" diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml new file mode 100644 index 00000000..90b167b4 --- /dev/null +++ b/api/client-server/v1/third_party_membership.yaml @@ -0,0 +1,127 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API for third party identifiers" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + description: |- + .. _invite-by-third-party-id-endpoint: + + *Note that there are two forms of this API, which are documented separately. + This version of the API does not require that the inviter know the Matrix + identifier of the invitee, and instead relies on third party identifiers. + The homeserver uses an identity server to perform the mapping from + third party identifier to a Matrix identifier. The other is documented in the* + `joining rooms section`_. + + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + Only users currently in a particular room can invite other users to + join that room. + + If the identity server did know the Matrix user identifier for the + third party identifier, the home server will append a ``m.room.member`` + event to the room. + + If the identity server does not know a Matrix user identifier for the + passed third party identifier, the homeserver will issue an invitation + which can be accepted upon providing proof of ownership of the third + party identifier. This is achieved by the identity server generating a + token, which it gives to the inviting homeserver. The homeserver will + add an ``m.room.third_party_invite`` event into the graph for the room, + containing that token. + + When the invitee binds the invited third party identifier to a Matrix + user ID, the identity server will give the user a list of pending + invitations, each containing: + + - The room ID to which they were invited + + - The token given to the homeserver + + - A signature of the token, signed with the identity server's private key + + - The matrix user ID who invited them to the room + + If a token is requested from the identity server, the home server will + append a ``m.room.third_party_invite`` event to the room. + + .. _joining rooms section: `invite-by-user-id-endpoint`_ + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier (not alias) to which to invite the user. + required: true + x-example: "!d41d8cd:matrix.org" + - in: body + name: body + required: true + schema: + type: object + example: |- + { + "id_server": "matrix.org", + "medium": "email", + "address": "cheeky@monkey.com", + "display_name": "A very cheeky monkey" + } + properties: + id_server: + type: string + description: The hostname+port of the identity server which should be used for third party identifier lookups. + medium: + type: string + # TODO: Link to identity service spec when it eixsts + description: The kind of address being passed in the address field, for example ``email``. + address: + type: string + description: The invitee's third party identifier. + display_name: + type: string + description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. + required: ["id_server", "medium", "address", "display_name"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/event-schemas/examples/v1/m.room.member b/event-schemas/examples/v1/m.room.member index 2c174e8a..e2ca5668 100644 --- a/event-schemas/examples/v1/m.room.member +++ b/event-schemas/examples/v1/m.room.member @@ -3,22 +3,7 @@ "content": { "membership": "join", "avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto", - "displayname": "Alice Margatroid", - "third_party_invite": { - "token": "pc98", - "public_key": "abc123", - "key_validity_url": "https://magic.forest/verifykey", - "signed": { - "mxid": "@alice:localhost", - "token": "pc98", - "signatures": { - "magic.forest": { - "ed25519:0": "poi098" - } - } - }, - "sender": "@zun:zun.soft" - } + "displayname": "Alice Margatroid" }, "invite_room_state": [ { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 0bd8cd1e..81057049 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -1,7 +1,7 @@ { "type": "object", "title": "The current membership state of a user in the room.", - "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if the invite was an ``m.room.third_party_invite`` event, and absent if the invite was an ``m.room.member`` event.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", + "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail. \n\nThe ``third_party_invite`` property will be set if this invite is an ``invite`` event and is the successor of an ``m.room.third_party_invite`` event, and absent otherwise.\n\nThis event also includes an ``invite_room_state`` key **outside the** ``content`` **key**. This contains an array of ``StrippedState`` Events. These events provide information on a few select state events such as the room name.", "allOf": [{ "$ref": "core-event-schema/state_event.json" }], @@ -27,18 +27,6 @@ "type": "object", "title": "Invite", "properties": { - "token": { - "type": "string", - "description": "A token which must be correctly signed, in order to join the room." - }, - "key_validity_url": { - "type": "string", - "description": "A URL which can be fetched, with querystring ``public_key=public_key``, to validate whether the key has been revoked. The URL must return a JSON object containing a boolean property named 'valid'." - }, - "public_key": { - "type": "string", - "description": "A base64-encoded ed25519 key with which token must be signed." - }, "signed": { "type": "object", "title": "signed", @@ -58,13 +46,9 @@ } }, "required": ["mxid", "signatures", "token"] - }, - "sender": { - "type": "string", - "description": "The matrix user ID of the user who send the invite which is being used." } }, - "required": ["token", "key_validity_url", "public_key", "sender", "signed"] + "required": ["signed"] } }, "required": ["membership"] diff --git a/specification/modules/anonymous_access.rst b/specification/modules/anonymous_access.rst new file mode 100644 index 00000000..5a421187 --- /dev/null +++ b/specification/modules/anonymous_access.rst @@ -0,0 +1,50 @@ +Guest access +================ + +.. _module:guest-access: + +It may be desirable to allow users without a fully registered user account to +ephemerally access Matrix rooms. This module specifies limited ways of doing so. + +Note that this is not currently a complete anonymous access solution; in +particular, it only allows servers to provided anonymous access to rooms in +which they are already participating, and relies on individual homeservers to +adhere to the conventions which this module sets, rather than allowing all +participating homeservers to enforce them. + +Events +------ + +{{m_room_guest_accessibility}} + +Client behaviour +---------------- +A client can register for guest access using the FOO endpoint. From that point +on, they can interact with a limited subset of the existing client-server API, +as if they were a fully registered user, using the access token granted to them +by the server. + +These users are only allowed to make calls in relation to rooms which have the +``m.room.history_visibility`` event set to ``world_readable``. + +The APIs they are allowed to hit are: + +/rooms/{roomId}/messages +/rooms/{roomId}/state +/rooms/{roomId}/state/{eventType}/{stateKey} +/events + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. Federation-specific logic should be included +here. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 85538c31..4d268631 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -1,27 +1,30 @@ Third party invites =================== -.. _module:third_party_invites: +.. _module:third-party-invites: This module adds in support for inviting new members to a room where their Matrix user ID is not known, instead addressing them by a third party identifier such as an email address. - There are two flows here; one if a Matrix user ID is known for the third party identifier, and one if not. Either way, the client calls ``/invite`` with the details of the third party identifier. The homeserver asks the identity server whether a Matrix user ID is known for -that identifier. If it is, an invite is simply issued for that user. +that identifier: + +- If it is, an invite is simply issued for that user. -If it is not, the homeserver asks the identity server to record the details of -the invitation, and to notify the client of this pending invitation if it gets -a binding for this identifier in the future. The identity server returns a token -and public key to the homeserver. +- If it is not, the homeserver asks the identity server to record the details of + the invitation, and to notify the invitee's homeserver of this pending invitation if it gets + a binding for this identifier in the future. The identity server returns a token + and public key to the inviting homeserver. -If a client then tries to join the room in the future, it will be allowed to if -it presents both the token, and a signature of that token from the identity -server which can be verified with the public key. +When the invitee's homeserver receives the notification of the binding, it +should insert an ``m.room.member`` event into the room's graph for that user, +with ``content.membership`` = ``invite``, as well as a +``content.third_party_invite`` property which contains proof that the invitee +does indeed own that third party identifier. Events ------ @@ -33,15 +36,18 @@ Client behaviour A client asks a server to invite a user by their third party identifier. +{{third_party_membership_http_api}} + Server behaviour ---------------- All homeservers MUST verify the signature in the event's ``content.third_party_invite.signed`` object. -If a client of the current homeserver is joining by an -``m.room.third_party_invite``, that homesever MUST validate that the public -key used for signing is still valid, by checking ``key_validity_url``. It does +When a homeserver inserts an ``m.room.member`` ``invite`` event into the graph +because of an ``m.room.third_party_invite`` event, +that homesever MUST validate that the public +key used for signing is still valid, by checking ``key_validity_url`` from the ``m.room.third_party_invite``. It does this by making an HTTP GET request to ``key_validity_url``: .. TODO: Link to identity server spec when it exists @@ -84,24 +90,24 @@ membership is questionable. For example: - If room R has two participating homeservers, H1, H2 +#. Room R has two participating homeservers, H1, H2 - And user A on H1 invites a third party identifier to room R +#. User A on H1 invites a third party identifier to room R - H1 asks the identity server for a binding to a Matrix user ID, and has none, - so issues an ``m.room.third_party_invite`` event to the room. +#. H1 asks the identity server for a binding to a Matrix user ID, and has none, + so issues an ``m.room.third_party_invite`` event to the room. - When the third party user validates their identity, they are told about the - invite, and ask their homeserver, H3, to join the room. +#. When the third party user validates their identity, their homeserver H3 + is notified and attempts to issue an ``m.room.member`` event to participate + in the room. - H3 validates the signature in the event's - ``content.third_party_invite.signed`` object. +#. H3 validates the signature given to it by the identity server. - H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` - property *and* check ``key_validity_url``. +#. H3 then asks H1 to join it to the room. H1 *must* validate the ``signed`` + property *and* check ``key_validity_url``. - Having validated these things, H1 writes the join event to the room, and H3 - begins participating in the room. H2 *must* accept this event. +#. Having validated these things, H1 writes the invite event to the room, and H3 + begins participating in the room. H2 *must* accept this event. The reason that no other homeserver may reject the event based on checking ``key_validity_url`` is that we must ensure event acceptance is deterministic. @@ -112,3 +118,32 @@ This relies on participating servers trusting each other, but that trust is already implied by the server-server protocol. Also, the public key signature verification must still be performed, so the attack surface here is minimized. +Security considerations +----------------------- + +There are a number of privary and trust implications to this module. + +It is important for user privacy that leaking the mapping between a matrix user +ID and a third party identifier is hard. In particular, being able to look up +all third party identifiers from a matrix user ID (and accordingly, being able +to link each third party identifier) should be avoided wherever possible. +To this end, when implementing this API care should be taken to avoid +adding links between these two identifiers as room events. This mapping can be +unintentionally created by specifying the third party identifier in the +``display_name`` field of the ``m.room.third_party_invite`` event, and then +observing which matrix user ID joins the room using that invite. Clients SHOULD +set ``display_name`` to a value other than the third party identifier, e.g. the +invitee's common name. + +Homeservers are not required to trust any particular identity server(s). It is +generally a client's responsibility to decide which identity servers it trusts, +not a homeserver's. Accordingly, this API takes identity servers as input from +end users, and doesn't have any specific trusted set. It is possible some +homeservers may want to supply defaults, or reject some identity servers for +*its* users, but no homeserver is allowed to dictate which identity servers +*other* homeservers' users trust. + +There is some risk of denial of service attacks by flooding homeservers or +identity servers with many requests, or much state to store. Defending against +these is left to the implementer's discretion. + From 559747e77a1bac4cd104f9526bfcaa84217651fb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:18:28 +0000 Subject: [PATCH 19/36] speculator: Sent Content-Type: text/html header Go is auto-detecting that this is XML (because for some reason we generate XHTML), and serving it with a Content-Type header text/xml. This causes the browser to render it as XHTML, which gives interesting quirks like extra newlines. This forces the browser to interpret it as HTML. What we should probably do instead of stop generating XHTML and start generating HTML. But in the mean time, this will fix the rendering issues. --- scripts/speculator/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 85fb2596..ef37b931 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -384,9 +384,9 @@ func main() { log.Fatal(err) } s := server{masterCloneDir} - http.HandleFunc("/spec/", s.serveSpec) - http.HandleFunc("/diff/rst/", s.serveRSTDiff) - http.HandleFunc("/diff/html/", s.serveHTMLDiff) + http.HandleFunc("/spec/", forceHTML(s.serveSpec)) + http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) @@ -394,6 +394,13 @@ func main() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } +func forceHTML(h func(w http.ResponseWriter, req *http.Request)) func(w http.ResponseWriter, req *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/html") + h(w, req) + } +} + func serveText(s string) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, req *http.Request) { io.WriteString(w, s) From 937ff046d866b96d068c3e1424689ebb58de99f5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 5 Nov 2015 19:21:16 +0000 Subject: [PATCH 20/36] Force / to be HTML too --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index ef37b931..7f86bd62 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -388,7 +388,7 @@ func main() { http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) - http.HandleFunc("/", listPulls) + http.HandleFunc("/", forceHTML(listPulls)) fmt.Printf("Listening on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From 7f6eafdce52972021643512d09e46eaef88c810d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 14:46:55 +0000 Subject: [PATCH 21/36] continuserv: set Content-Type header --- scripts/continuserv/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95..425a1af9 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -113,6 +113,7 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() b := toServe.Load().([]byte) + w.Header().Set("Content-Type", "text/html") w.Write(b) } From 1be5b856bd2447aa557ada4fcde8b8ddf63a3f70 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:05:07 +0000 Subject: [PATCH 22/36] Preserve text/plain for errors Newlines are nice --- scripts/continuserv/main.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 425a1af9..4613437a 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,7 +25,7 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + toServe atomic.Value // Always contains valid bytesOrErr to serve. May be stale unless wg is zero. wg sync.WaitGroup // Indicates how many updates are pending. mu sync.Mutex // Prevent multiple updates in parallel. ) @@ -112,9 +112,14 @@ func filter(e fsnotify.Event) bool { func serve(w http.ResponseWriter, req *http.Request) { wg.Wait() - b := toServe.Load().([]byte) - w.Header().Set("Content-Type", "text/html") - w.Write(b) + b := toServe.Load().(bytesOrErr) + if b.err != nil { + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(b.err.Error())) + } else { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(b.bytes)) + } } func populateOnce(dir string) { @@ -127,15 +132,15 @@ func populateOnce(dir string) { cmd.Stderr = &b err := cmd.Run() if err != nil { - toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) if err != nil { - toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) return } - toServe.Store(specBytes) + toServe.Store(bytesOrErr{specBytes, nil}) } func doPopulate(ch chan struct{}, dir string) { @@ -160,3 +165,8 @@ func exists(path string) bool { _, err := os.Stat(path) return !os.IsNotExist(err) } + +type bytesOrErr struct { + bytes []byte + err error +} From bbf9e229a7735c1c1ac295169a9f81b83874cc91 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 16:09:09 +0000 Subject: [PATCH 23/36] continuserv: guard concurrent accesses to wg --- scripts/continuserv/main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 573c2c95..2c6072a3 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -25,9 +25,11 @@ import ( var ( port = flag.Int("port", 8000, "Port on which to serve HTTP") - toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. - wg sync.WaitGroup // Indicates how many updates are pending. - mu sync.Mutex // Prevent multiple updates in parallel. + mu sync.Mutex // Prevent multiple updates in parallel. + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + + wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel. + wg sync.WaitGroup // Indicates how many updates are pending. ) func main() { @@ -111,7 +113,9 @@ func filter(e fsnotify.Event) bool { } func serve(w http.ResponseWriter, req *http.Request) { + wgMu.Lock() wg.Wait() + wgMu.Unlock() b := toServe.Load().([]byte) w.Write(b) } @@ -143,7 +147,9 @@ func doPopulate(ch chan struct{}, dir string) { select { case <-ch: if pending == 0 { + wgMu.Lock() wg.Add(1) + wgMu.Unlock() } pending++ case <-time.After(10 * time.Millisecond): From e72151f2c3f8e2825eb2719b3e4598672c134a03 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 6 Nov 2015 18:15:21 +0000 Subject: [PATCH 24/36] Specify guest room access This was reviewed as PR #150 and merged from daniel/anonymousaccess --- api/client-server/v1/guest_events.yaml | 103 ++++++++++++++++++ event-schemas/examples/v1/m.room.guest_access | 12 ++ event-schemas/schema/v1/m.room.guest_access | 30 +++++ specification/modules/guest_access.rst | 81 ++++++++++++++ specification/targets.yaml | 1 + templating/matrix_templates/units.py | 9 +- 6 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 api/client-server/v1/guest_events.yaml create mode 100644 event-schemas/examples/v1/m.room.guest_access create mode 100644 event-schemas/schema/v1/m.room.guest_access create mode 100644 specification/modules/guest_access.rst diff --git a/api/client-server/v1/guest_events.yaml b/api/client-server/v1/guest_events.yaml new file mode 100644 index 00000000..bbb5799a --- /dev/null +++ b/api/client-server/v1/guest_events.yaml @@ -0,0 +1,103 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Sync Guest API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/events": + get: + summary: Listen on the event stream. + description: |- + This will listen for new events related to a particular room and return + them to the caller. This will block until an event is received, or until + the ``timeout`` is reached. + + This API is the same as the non-guest /events endpoint, but can be + called by guest users. + security: + - accessToken: [] + parameters: + - in: query + type: string + name: from + description: |- + The token to stream from. This token is either from a previous + request to this API or from the initial sync API. + required: false + x-example: "s3456_9_0" + - in: query + type: integer + name: timeout + description: The maximum time in milliseconds to wait for an event. + required: false + x-example: "35000" + - in: query + type: array + items: + type: string + name: room_id + description: |- + The room IDs for which events should be returned. + x-example: + - "!somewhere:over" + - "!the:rainbow" + responses: + 200: + description: "The events received, which may be none." + examples: + application/json: |- + { + "start": "s3456_9_0", + "end": "s3457_9_0", + "chunk": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" + }, + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } + schema: + type: object + properties: + start: + type: string + description: |- + A token which correlates to the first value in ``chunk``. This + is usually the same token supplied to ``from=``. + end: + type: string + description: |- + A token which correlates to the last value in ``chunk``. This + token should be used in the next request to ``/events``. + chunk: + type: array + description: "An array of events." + items: + type: object + title: Event + allOf: + - "$ref": "core-event-schema/room_event.json" + 400: + description: "Bad pagination ``from`` parameter." diff --git a/event-schemas/examples/v1/m.room.guest_access b/event-schemas/examples/v1/m.room.guest_access new file mode 100644 index 00000000..8ad4d294 --- /dev/null +++ b/event-schemas/examples/v1/m.room.guest_access @@ -0,0 +1,12 @@ +{ + "age": 242353, + "content": { + "guest_access": "can_join" + }, + "state_key": "", + "origin_server_ts": 1431961217938, + "event_id": "$WLGTSEFSEG:localhost", + "type": "m.room.guest_access", + "room_id": "!Cuyf34gef24u:localhost", + "user_id": "@example:localhost" +} diff --git a/event-schemas/schema/v1/m.room.guest_access b/event-schemas/schema/v1/m.room.guest_access new file mode 100644 index 00000000..a0141f37 --- /dev/null +++ b/event-schemas/schema/v1/m.room.guest_access @@ -0,0 +1,30 @@ +{ + "type": "object", + "title": "Controls whether guest users are allowed to join rooms.", + "description": "This event controls whether guest users are allowed to join rooms. If this event is absent, servers should act as if it is present and has the guest_access value \"forbidden\".", + "allOf": [{ + "$ref": "core-event-schema/state_event.json" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "guest_access": { + "type": "string", + "description": "Whether guests can join the room.", + "enum": ["can_join", "forbidden"] + } + }, + "required": ["guest_access"] + }, + "state_key": { + "type": "string", + "description": "A zero-length string.", + "pattern": "^$" + }, + "type": { + "type": "string", + "enum": ["m.room.guest_access"] + } + } +} diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst new file mode 100644 index 00000000..ac373af9 --- /dev/null +++ b/specification/modules/guest_access.rst @@ -0,0 +1,81 @@ +Guest access +============ + +.. _module:guest-access: + +There are times when it is desirable for clients to be able to interact with +rooms without having to fully register for an account on a homeserver or join +the room. This module specifies how these clients should interact with servers +in order to participate in rooms as guests. + +Guest users retrieve access tokens from a homeserver using the ordinary +`register endpoint <#post-matrix-client-api-v2-alpha-register>`_, specifying +the ``kind`` parameter as ``guest``. They may then interact with the +client-server API as any other user would, but will only have access to a subset +of the API as described the Client behaviour subsection below. +Homeservers may choose not to allow this access at all to their local users, but +have no information about whether users on other homeservers are guests or not. + +This module does not fully factor in federation; it relies on individual +homeservers properly adhering to the rules set out in this module, rather than +allowing all homeservers to enforce the rules on each other. + +Events +------ +{{m_room_guest_access_event}} + +Client behaviour +---------------- +The following API endpoints are allowed to be accessed by guest accounts for +retrieving events: + +* `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ +* `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ +* `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ + +There is also a special version of the +`GET /events <#get-matrix-client-api-v1-events>`_ endpoint: + +{{guest_events_http_api}} + +They will only return events which happened while the room state had the +``m.room.history_visibility`` state event present with ``history_visibility`` +value ``world_readable``. Guest clients do not need to join rooms in order to +receive events for them. + +The following API endpoints are allowed to be accessed by guest accounts for +sending events: + +* `POST /rooms/:room_id/join <#post-matrix-client-api-v1-rooms-roomid-join>`_ +* `PUT /rooms/:room_id/send/m.room.message/:txn_id <#put-matrix-client-api-v1-rooms-roomid-send-eventtype-txnid>`_ + +Guest clients *do* need to join rooms in order to send events to them. + +Server behaviour +---------------- +Servers are required to only return events to guest accounts for rooms where +the room state at the event had the ``m.room.history_visibility`` state event +present with ``history_visibility`` value ``world_readable``. These events may +be returned even if the anonymous user is not joined to the room. + +Servers MUST only allow guest users to join rooms if the ``m.room.guest_access`` +state event is present on the room, and has the ``guest_access`` value +``can_join``. If the ``m.room.guest_access`` event is changed to stop this from +being the case, the server MUST set those users' ``m.room.member`` state to +``leave``. + +Security considerations +----------------------- +Each homeserver manages its own guest accounts itself, and whether an account +is a guest account or not is not information passed from server to server. +Accordingly, any server participating in a room is trusted to properly enforce +the permissions outlined in this section. + +Clients may wish to display to their users that rooms which are +``world_readable`` *may* be showing messages to non-joined users. There is no +way using this module to find out whether any non-joined guest users *do* see +events in the room, or to list or count any guest users. + +Homeservers may want to enable protections such as captchas for guest +registration to prevent spam, denial of service, and similar attacks. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 2482dcfd..8e6a2ce0 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -25,6 +25,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/push.rst - modules/third_party_invites.rst - modules/search.rst + - modules/guest_access.rst title_styles: ["=", "-", "~", "+", "^", "`"] diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index cec5b83e..adb7f427 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -369,7 +369,7 @@ class MatrixUnits(Units): ] if len(params_missing_examples) == 0: path_template = api.get("basePath", "").rstrip("/") + path - qps = {} + qps = [] body = "" for param in single_api.get("parameters", []): if param["in"] == "path": @@ -381,7 +381,12 @@ class MatrixUnits(Units): elif param["in"] == "body": body = param["schema"]["example"] elif param["in"] == "query": - qps[param["name"]] = param["x-example"] + example = param["x-example"] + if type(example) == list: + for value in example: + qps.append((param["name"], value)) + else: + qps.append((param["name"], example)) query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps) if body: endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( From 436a35e9f647fcaa5ef27b48ef26b683440fcb40 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 16:04:31 +0000 Subject: [PATCH 25/36] Document macaroon type=login --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 93622c3d..a7c1b036 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -25,10 +25,13 @@ Specified caveats: | gen | Generation of the macaroon caveat spec. | 1 | | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| refresh - only used to authorize a token refresh | +| | | refresh - only used to authorize a token refresh | +| | | login - issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | -| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | -| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | -| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| -| Note that exact equality of time is largely meaningless. | +| | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | +| | | Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.| +| | | Note that exact equality of time is largely meaningless. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ From 24c2036a35738ebf9c1ebe3b01cef1943a087702 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 9 Nov 2015 17:30:18 +0000 Subject: [PATCH 26/36] 3pid invites: remove mentions of display_name --- api/client-server/v1/third_party_membership.yaml | 8 ++------ specification/modules/third_party_invites.rst | 11 ++++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api/client-server/v1/third_party_membership.yaml b/api/client-server/v1/third_party_membership.yaml index 90b167b4..a10d6167 100644 --- a/api/client-server/v1/third_party_membership.yaml +++ b/api/client-server/v1/third_party_membership.yaml @@ -84,8 +84,7 @@ paths: { "id_server": "matrix.org", "medium": "email", - "address": "cheeky@monkey.com", - "display_name": "A very cheeky monkey" + "address": "cheeky@monkey.com" } properties: id_server: @@ -98,10 +97,7 @@ paths: address: type: string description: The invitee's third party identifier. - display_name: - type: string - description: A user-friendly string describing who has been invited. It should not contain the address of the invitee, to avoid leaking mappings between third party identities and matrix user IDs. - required: ["id_server", "medium", "address", "display_name"] + required: ["id_server", "medium", "address"] responses: 200: description: The user has been invited to join the room. diff --git a/specification/modules/third_party_invites.rst b/specification/modules/third_party_invites.rst index 4d268631..d8e3d4d9 100644 --- a/specification/modules/third_party_invites.rst +++ b/specification/modules/third_party_invites.rst @@ -127,13 +127,10 @@ It is important for user privacy that leaking the mapping between a matrix user ID and a third party identifier is hard. In particular, being able to look up all third party identifiers from a matrix user ID (and accordingly, being able to link each third party identifier) should be avoided wherever possible. -To this end, when implementing this API care should be taken to avoid -adding links between these two identifiers as room events. This mapping can be -unintentionally created by specifying the third party identifier in the -``display_name`` field of the ``m.room.third_party_invite`` event, and then -observing which matrix user ID joins the room using that invite. Clients SHOULD -set ``display_name`` to a value other than the third party identifier, e.g. the -invitee's common name. +To this end, the third party identifier is not put in any event, rather an +opaque display name provided by the identity server is put into the events. +Clients should not remember or display third party identifiers from invites, +other than for the use of the inviter themself. Homeservers are not required to trust any particular identity server(s). It is generally a client's responsibility to decide which identity servers it trusts, From c1866ebebc708d0e8a404edbc3f0bb9b0f3d4297 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:26:06 +0000 Subject: [PATCH 27/36] Fix table formatting --- drafts/macaroons_caveats.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index a7c1b036..71c4784e 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -21,14 +21,17 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | Caveat name | Description | Legal Values | -+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ ++=============+==================================================+================================================================================================+ | gen | Generation of the macaroon caveat spec. | 1 | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | type | The purpose of this macaroon. | access - used to authorize any action except token refresh | | | | refresh - only used to authorize a token refresh | | | | login - issued as a very short-lived token by third party login flows; proves that | | | | authentication has happened but doesn't grant any privileges other than being able to be | | | | exchanged for other tokens. | ++-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | | | | Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | From 51fe4a90b6f3e79428f1e1160b7c7df4f906adb9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:28:27 +0000 Subject: [PATCH 28/36] More formatting fixes --- drafts/macaroons_caveats.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 71c4784e..11e36e59 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -9,12 +9,15 @@ Caveats can only be used for reducing the scope of a token, never for increasing Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. -All caveats must take the form: +All caveats must take the form:: -`key` `operator` `value` -where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] -`operator` is a non-empty string which does not contain whitespace -`value` is a non-empty string + key operator value + +where: + - ``key`` is a non-empty string drawn from the character set [A-Za-z0-9_] + - ``operator`` is a non-empty string which does not contain whitespace + - ``value`` is a non-empty string + And these are joined by single space characters. Specified caveats: From c8f6ed11074f489df5562ce614e3113f48ea77d9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Nov 2015 11:31:31 +0000 Subject: [PATCH 29/36] More formatting.. --- drafts/macaroons_caveats.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 11e36e59..de5973fa 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -29,11 +29,11 @@ Specified caveats: +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ -| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | -| | | refresh - only used to authorize a token refresh | -| | | login - issued as a very short-lived token by third party login flows; proves that | -| | | authentication has happened but doesn't grant any privileges other than being able to be | -| | | exchanged for other tokens. | +| type | The purpose of this macaroon. | - ``access``: used to authorize any action except token refresh | +| | | - ``refresh``: only used to authorize a token refresh | +| | | - ``login``: issued as a very short-lived token by third party login flows; proves that | +| | | authentication has happened but doesn't grant any privileges other than being able to be | +| | | exchanged for other tokens. | +-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+ | time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | | | | Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | From 27ffe7bacd15ae559cdfc7bce03a42d00699d7d4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 10 Nov 2015 15:34:32 +0000 Subject: [PATCH 30/36] Don't serve rst diffs as HTML --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 7f86bd62..e1898d7e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -385,7 +385,7 @@ func main() { } s := server{masterCloneDir} http.HandleFunc("/spec/", forceHTML(s.serveSpec)) - http.HandleFunc("/diff/rst/", forceHTML(s.serveRSTDiff)) + http.HandleFunc("/diff/rst/", s.serveRSTDiff) http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff)) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", forceHTML(listPulls)) From d7357ef9b75893631e4a8b1e5e40273885cd6709 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:39:40 +0000 Subject: [PATCH 31/36] Specify /publicRooms --- api/client-server/v1/list_public_rooms.yaml | 85 +++++++++++++++++++ specification/client_server_api.rst | 5 ++ .../matrix_templates/templates/http-api.tmpl | 5 +- 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 api/client-server/v1/list_public_rooms.yaml diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml new file mode 100644 index 00000000..2f189d2e --- /dev/null +++ b/api/client-server/v1/list_public_rooms.yaml @@ -0,0 +1,85 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Creation API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +paths: + "/publicRooms": + get: + summary: Lists the public rooms on the server. + description: |- + Lists the public rooms on the server. + + This API returns paginated responses. + responses: + 200: + description: A list of the rooms on the server. + schema: + type: object + description: A list of the rooms on the server. + properties: + chunk: + title: "PublicRoomsChunks" + type: array + description: |- + A paginated chunk of public rooms. + items: + type: object + title: "PublicRoomsChunk" + properties: + aliases: + type: array + description: |- + Aliases of the room. May be empty. + items: + type: string + name: + type: string + description: |- + The name of the room, if any. May be null. + num_joined_members: + type: number + description: |- + The number of members joined to the room. + room_id: + type: string + description: |- + The ID of the room. + topic: + type: string + description: |- + The topic of the room, if any. May be null. + start: + type: string + description: |- + A pagination token for the response. + end: + type: string + description: |- + A pagination token for the response. + examples: + application/json: |- + { + "chunk": [ + { + "aliases": ["#murrays:cheese.bar"], + "name": "CHEESE", + "num_joined_members": 37, + "room_id": "!ol19s:bleecker.street", + "topic": "Tasty tasty cheese" + } + ], + "start": "p190q", + "end": "p1902" + } + 400: + description: > + The request body is malformed or the room alias specified is already taken. diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index b02dbf28..05b6ff3c 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -931,6 +931,11 @@ member's state, by making a request to "membership": "ban" } +Listing rooms +~~~~~~~~~~~~~ + +{{list_public_rooms_http_api}} + Profiles -------- diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 86eacb14..d7258e98 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -12,7 +12,7 @@ {{":Requires auth: Yes." if endpoint.requires_auth else "" }} Request format: - +{% if (endpoint.req_param_by_loc | length) %} =========================================== ================= =========================================== Parameter Value Description =========================================== ================= =========================================== @@ -24,6 +24,9 @@ Request format: {% endfor -%} {% endfor -%} =========================================== ================= =========================================== +{% else %} +`No parameters` +{% endif %} {% if endpoint.res_tables|length > 0 -%} Response format: From dcf54e11b10b07fd30cd18be127d9cde9b198e8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 11:53:31 +0000 Subject: [PATCH 32/36] Specify /publicRooms world_readable and guest_access Depends on https://github.com/matrix-org/matrix-doc/pull/154 --- api/client-server/v1/list_public_rooms.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/list_public_rooms.yaml b/api/client-server/v1/list_public_rooms.yaml index 2f189d2e..23b1c3d0 100644 --- a/api/client-server/v1/list_public_rooms.yaml +++ b/api/client-server/v1/list_public_rooms.yaml @@ -57,6 +57,16 @@ paths: type: string description: |- The topic of the room, if any. May be null. + world_readable: + type: boolean + description: |- + Whether the room may be viewed by guest users without joining. + guest_can_join: + type: boolean + description: |- + Whether guest users may join the room and participate in it. + If they can, they will be subject to ordinary power level + rules like any other user. start: type: string description: |- @@ -71,10 +81,12 @@ paths: "chunk": [ { "aliases": ["#murrays:cheese.bar"], + "guest_can_join": false, "name": "CHEESE", "num_joined_members": 37, "room_id": "!ol19s:bleecker.street", - "topic": "Tasty tasty cheese" + "topic": "Tasty tasty cheese", + "world_readable": true } ], "start": "p190q", From 740cc66a7cccc366ca34eac1ebf65fd7c9af90b3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 14:01:45 +0000 Subject: [PATCH 33/36] speculator: Fetch before deciding head is fresh --- scripts/speculator/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e1898d7e..e467018e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -178,6 +178,10 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { + if err := gitFetch(s.matrixDocCloneURL); err != nil { + writeError(w, 500, err) + return + } originHead, err := s.getSHAOf("origin/master") if err != nil { writeError(w, 500, err) From 0f0359d9c1049c60dbea365344b40cadc1a50c16 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 15:13:34 +0000 Subject: [PATCH 34/36] speculator: Nessle up some more if statements --- scripts/speculator/main.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index e467018e..97e67c8c 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -74,8 +74,7 @@ func gitClone(url string, shared bool) (string, error) { cmd.Args = append(cmd.Args, "--shared") } - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error cloning repo: %v", err) } return directory, nil @@ -92,8 +91,7 @@ func gitFetch(path string) error { func runGitCommand(path string, args []string) error { cmd := exec.Command("git", args...) cmd.Dir = path - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) } return nil @@ -126,8 +124,7 @@ func generate(dir string) error { cmd.Dir = path.Join(dir, "scripts") var b bytes.Buffer cmd.Stderr = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return nil @@ -167,8 +164,7 @@ func (s *server) getSHAOf(ref string) (string, error) { cmd.Dir = path.Join(s.matrixDocCloneURL) var b bytes.Buffer cmd.Stdout = &b - err := cmd.Run() - if err != nil { + if err := cmd.Run(); err != nil { return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) } return strings.TrimSpace(b.String()), nil From 061105c9dc25d52a01ba260afc9c90bd0158931b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 11 Nov 2015 17:18:58 +0000 Subject: [PATCH 35/36] Guest users are allowed room initialSync --- api/client-server/v1/rooms.yaml | 2 +- specification/modules/guest_access.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9300d7d1..90b51290 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -371,7 +371,7 @@ paths: description: |- Whether this room is visible to the ``/publicRooms`` API or not." - required: ["room_id", "membership"] + required: ["room_id"] 403: description: > You aren't a member of the room and weren't previously a diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af9..ae8d156e 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -32,6 +32,7 @@ retrieving events: * `GET /rooms/:room_id/state <#get-matrix-client-api-v1-rooms-roomid-state>`_ * `GET /rooms/:room_id/state/:event_type/:state_key <#get-matrix-client-api-v1-rooms-roomid-state-eventtype-statekey>`_ * `GET /rooms/:room_id/messages <#get-matrix-client-api-v1-rooms-roomid-messages>`_ +* `GET /rooms/:room_id/initialSync <#get-matrix-client-api-v1-rooms-roomid-initialsync>`_ There is also a special version of the `GET /events <#get-matrix-client-api-v1-events>`_ endpoint: From 29d9c8eec6a0a7da49b78c4885b168662fc04f99 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 12 Nov 2015 12:05:02 +0000 Subject: [PATCH 36/36] Guests are allowed to set displaynames --- specification/modules/guest_access.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index ac373af9..d7359f23 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -51,6 +51,11 @@ sending events: Guest clients *do* need to join rooms in order to send events to them. +The following API endpoints are allowed to be accessed by guest accounts for +their own account maintenance: + +* `PUT /profile/:user_id/displayname <#put-matrix-client-api-v1-profile-userid-displayname>`_ + Server behaviour ---------------- Servers are required to only return events to guest accounts for rooms where