From 365a9076b93500cd87e85e68129e6ba7729f7625 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 12:11:26 +0100 Subject: [PATCH 1/3] Add nested dict template support; Add x-pattern For cases where event schema specify `patternProperties` it would be nice to give that pattern a "human-readable" form rather than a raw regex. This is now supported by specifying `x-pattern` in the value part of the specified pattern e.g. `patternProperties:{ "^.*":{ x-pattern: "$THING", ... } }` Templating had limited record type descriptions limited to value primitives e.g. `{string: integer}`. It now supports inspecting the values recursively if the value is `object`. Updated `m.receipt` to take both these points into account to make it read better. Tweak receipt module text. --- event-schemas/schema/v1/m.receipt | 9 ++++--- specification/modules/receipts.rst | 37 ++++++++++++---------------- templating/matrix_templates/units.py | 26 +++++++++++++++---- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 0f365eed..5be232ad 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -5,17 +5,20 @@ "properties": { "content": { "type": "object", - "description": "The event ids which the receipts relate to.", "patternProperties": { "^\\$": { "type": "object", - "description": "The types of the receipts.", + "x-pattern": "$EVENT_ID", + "description": "The mapping of event ID to receipt type. The event ID is the ID which the receipts relate to and *not* an ID for the receipt itself. The key in the object is an enum which must be ``read``.", "additionalProperties": { "type": "object", - "description": "User ids of the receipts", + "title": "Users", "patternProperties": { "^@": { "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", "properties": { "ts": { "type": "number", diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index e2f83eea..f32c12db 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -6,8 +6,16 @@ have interacted with. For example, which events the user has read. For efficiency this is done as "up to" markers, i.e. marking a particular event as, say, ``read`` indicates the user has read all events *up to* that event. -Client-Server API -~~~~~~~~~~~~~~~~~ +Events +------ + +{{m_receipt_event}} + +Client behaviour +---------------- + + - When clients should send receipts + - What clients should do when they receive these receipts Clients will receive receipts in the following format:: @@ -25,22 +33,6 @@ Clients will receive receipts in the following format:: } } -For example:: - - { - "type": "m.receipt", - "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", - "content": { - "$1435641916114394fHBLK:matrix.org": { - "read": { - "@erikj:jki.re": { "ts": 1436451550453 }, - ... - } - }, - ... - } - } - For efficiency, receipts are batched into one event per room. In the initialSync and v2 sync APIs the receipts are listed in a separate top level ``receipts`` key. Each ``user_id``, ``receipt_type`` pair must be associated with only a @@ -56,9 +48,8 @@ A client can update the markers for its user by issuing a request:: Where the contents of the ``POST`` will be included in the content sent to other users. The server will automatically set the ``ts`` field. - -Server-Server API -~~~~~~~~~~~~~~~~~ +Server behaviour +---------------- Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -75,3 +66,7 @@ format of the EDUs are:: These are always sent as deltas to previously sent receipts. +Security considerations +----------------------- + + diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 50fa784e..473fdd82 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -49,8 +49,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): } tables = [fields] - props = obj.get("properties", obj.get("patternProperties")) parents = obj.get("allOf") + props = obj.get("properties") + if not props: + props = obj.get("patternProperties") + if props: + # try to replace horrible regex key names with pretty x-pattern ones + for key_name in props.keys(): + pretty_key = props[key_name].get("x-pattern") + if pretty_key: + props[pretty_key] = props[key_name] + del props[key_name] if not props and not parents: raise Exception( "Object %s has no properties or parents." % obj @@ -70,10 +79,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): if props[key_name]["type"] == "object": if props[key_name].get("additionalProperties"): # not "really" an object, just a KV store - value_type = ( - "{string: %s}" % - props[key_name]["additionalProperties"]["type"] - ) + prop_val = props[key_name]["additionalProperties"]["type"] + if prop_val == "object": + nested_object = get_json_schema_object_fields( + props[key_name]["additionalProperties"], + enforce_title=True + ) + value_type = "{string: %s}" % nested_object[0]["title"] + if not nested_object[0].get("no-table"): + tables += nested_object + else: + value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( props[key_name], From c972dad8b383a8345864d32ba6997b5da1fd57fa Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 15:41:54 +0100 Subject: [PATCH 2/3] Flesh out receipts module. Add receipts swagger Add templating support for v2 apis. --- api/client-server/v2_alpha/receipts.yaml | 68 +++++++++++++++++++++ event-schemas/examples/v1/m.receipt | 2 +- event-schemas/schema/v1/m.receipt | 35 ++++++----- specification/modules/receipts.rst | 76 ++++++++++++++---------- templating/matrix_templates/units.py | 30 ++++++---- 5 files changed, 151 insertions(+), 60 deletions(-) create mode 100644 api/client-server/v2_alpha/receipts.yaml diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/v2_alpha/receipts.yaml new file mode 100644 index 00000000..4ef435b0 --- /dev/null +++ b/api/client-server/v2_alpha/receipts.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Receipts API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/receipt/{receiptType}/{eventId}": + post: + summary: Send a receipt for the given event ID. + description: |- + This API updates the marker for the given receipt type to the event ID + specified. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room in which to send the event. + required: true + x-example: "!wefuh21ffskfuh345:example.com" + - in: path + type: string + name: receiptType + description: The type of receipt to send. + required: true + x-example: "m.read" + enum: ["m.read"] + - in: path + type: string + name: eventId + description: The event ID to acknowledge up to. + required: true + x-example: "$1924376522eioj:example.com" + - in: body + description: |- + Extra receipt information to attach to ``content`` if any. The + server will automatically set the ``ts`` field. + schema: + type: object + example: |- + {} + responses: + 200: + description: The receipt was sent. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" \ No newline at end of file diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/v1/m.receipt index 83515317..bd0b726c 100644 --- a/event-schemas/examples/v1/m.receipt +++ b/event-schemas/examples/v1/m.receipt @@ -3,7 +3,7 @@ "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", "content": { "$1435641916114394fHBLK:matrix.org": { - "read": { + "m.read": { "@rikj:jki.re": { "ts": 1436451550453 } diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 5be232ad..d0f79ac4 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -9,25 +9,28 @@ "^\\$": { "type": "object", "x-pattern": "$EVENT_ID", - "description": "The mapping of event ID to receipt type. The event ID is the ID which the receipts relate to and *not* an ID for the receipt itself. The key in the object is an enum which must be ``read``.", - "additionalProperties": { - "type": "object", - "title": "Users", - "patternProperties": { - "^@": { - "type": "object", - "title": "Receipt", - "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", - "x-pattern": "$USER_ID", - "properties": { - "ts": { - "type": "number", - "description": "The timestamp the receipt was sent at" + "title": "Receipts", + "description": "The mapping of event ID to a collection of receipts for this event ID. The event ID is the ID of the event being acknowledged and *not* an ID for the receipt itself.", + "properties": { + "m.read": { + "type": "object", + "title": "Users", + "description": "A collection of users who have sent ``m.read`` receipts for this event.", + "patternProperties": { + "^@": { + "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", + "properties": { + "ts": { + "type": "number", + "description": "The timestamp the receipt was sent at." + } } } } - }, - "additionalProperties": false + } } } }, diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index f32c12db..9dabba30 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -1,56 +1,63 @@ Receipts --------- +======== -Receipts are used to publish which events in a room the user or their devices -have interacted with. For example, which events the user has read. For -efficiency this is done as "up to" markers, i.e. marking a particular event -as, say, ``read`` indicates the user has read all events *up to* that event. +This module adds in support for receipts. These receipts are a form of +acknowledgement of an event. This module defines a single acknowledgement: +``m.read`` which indicates that the user has read up to a given event. + +Sending a receipt for each event can result in sending large amounts of traffic +to a homeserver. To prevent this from becoming a problem, receipts are implemented +using "up to" markers. This marker indicates that the acknowledgement applies +to all events "up to and including" the event specified. For example, marking +an event as "read" would indicate that the user had read all events *up to* the +referenced event. Events ------ +Each ``user_id``, ``receipt_type`` pair must be associated with only a +single ``event_id``. {{m_receipt_event}} Client behaviour ---------------- - - When clients should send receipts - - What clients should do when they receive these receipts - -Clients will receive receipts in the following format:: +In v1 ``/initialSync``, receipts are listed in a separate top level ``receipts`` +key. In v2 ``/sync``, receipts are contained in the ``ephemeral`` block for a +room. New receipts that come down the event streams are deltas which update +existing mappings. Clients should replace older receipt acknowledgements based +on ``user_id`` and ``receipt_type`` pairs. For example:: - { - "type": "m.receipt", - "room_id": , - "content": { - : { - : { - : { "ts": , ... }, - ... - } - }, - ... - } - } + Client receives m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $aaa:example.com -For efficiency, receipts are batched into one event per room. In the initialSync -and v2 sync APIs the receipts are listed in a separate top level ``receipts`` -key. Each ``user_id``, ``receipt_type`` pair must be associated with only a -single ``event_id``. New receipts that come down the event streams are deltas. -Deltas update existing mappings, clobbering based on ``user_id``, -``receipt_type`` pairs. + Client receives another m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $bbb:example.com + The client should replace the older acknowledgement for $aaa:example.com with + this one for $bbb:example.com -A client can update the markers for its user by issuing a request:: +Clients should send read receipts when there is some certainty that the event in +question has been **displayed** to the user. Simply receiving an event does not +provide enough certainty that the user has seen the event. The user SHOULD need +to *take some action* such as viewing the room that the event was sent to or +dismissing a notification in order for the event to count as "read". - POST /_matrix/client/v2_alpha/rooms//receipt/read/ +A client can update the markers for its user by interacting with the following +HTTP APIs. -Where the contents of the ``POST`` will be included in the content sent to -other users. The server will automatically set the ``ts`` field. +{{v2_receipts_http_api}} Server behaviour ---------------- +For efficiency, receipts SHOULD be batched into one event per room before +delivering them to clients. + Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -64,9 +71,12 @@ format of the EDUs are:: ... } -These are always sent as deltas to previously sent receipts. +These are always sent as deltas to previously sent receipts. Currently only a +single ```` should be used: ``m.read``. Security considerations ----------------------- +As receipts are sent outside the context of the event graph, there are no +integrity checks performed on the contents of ``m.receipt`` events. diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 473fdd82..eca52acb 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -19,6 +19,7 @@ import yaml V1_CLIENT_API = "../api/client-server/v1" V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" +V2_CLIENT_API = "../api/client-server/v2_alpha" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -336,18 +337,27 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - path = V1_CLIENT_API + paths = [ + V1_CLIENT_API, V2_CLIENT_API + ] apis = {} - for filename in os.listdir(path): - if not filename.endswith(".yaml"): + for path in paths: + is_v2 = (path == V2_CLIENT_API) + if not os.path.exists(V2_CLIENT_API): + self.log("Skipping v2 apis: %s does not exist.", V2_CLIENT_API) continue - self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: - # strip .yaml - group_name = filename[:-5] - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) - apis[group_name] = api + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + self.log("Reading swagger API: %s" % filename) + with open(os.path.join(path, filename), "r") as f: + # strip .yaml + group_name = filename[:-5] + if is_v2: + group_name = "v2_" + group_name + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta(api, group_name) + apis[group_name] = api return apis def load_common_event_fields(self): From 560cd7a58fe6ecdb713dd3cd30b58b3e98a04d3d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 15:54:45 +0100 Subject: [PATCH 3/3] This isn't javascript. s/,/%/ --- templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index eca52acb..71b6acc6 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -344,7 +344,7 @@ class MatrixUnits(Units): for path in paths: is_v2 = (path == V2_CLIENT_API) if not os.path.exists(V2_CLIENT_API): - self.log("Skipping v2 apis: %s does not exist.", V2_CLIENT_API) + self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) continue for filename in os.listdir(path): if not filename.endswith(".yaml"):