diff --git a/api/client-server/read_markers.yaml b/api/client-server/read_markers.yaml new file mode 100644 index 00000000..a74592c5 --- /dev/null +++ b/api/client-server/read_markers.yaml @@ -0,0 +1,79 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Read Marker API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/rooms/{roomId}/read_markers": + post: + summary: Set the position of the read marker for a room. + description: |- + Sets the position of the read marker for a given room, and optionally + the read receipt's location. + operationId: setReadMarker + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room ID to set the read marker in for the user. + required: true + x-example: "!somewhere:domain.com" + - in: body + name: body + description: The read marker and optional read receipt locations. + required: true + schema: + type: object + properties: + "m.fully_read": + type: string + description: |- + The event ID the read marker should be located at. The + event MUST belong to the room. + example: "$somewhere:domain.com" + "m.read": + type: string + description: |- + The event ID to set the read receipt location at. This is + equivalent to calling ``/receipt/m.read/$elsewhere:domain.com`` + and is provided here to save that extra call. + example: "$elsewhere:domain.com" + required: ['m.fully_read'] + responses: + 200: + description: |- + The read marker, and read receipt if provided, have been updated. + schema: + type: object + properties: {} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/errors/rate_limited.yaml" + tags: + - Read Markers diff --git a/changelogs/client_server/newsfragments/1420.feature b/changelogs/client_server/newsfragments/1420.feature new file mode 100644 index 00000000..f7cf687d --- /dev/null +++ b/changelogs/client_server/newsfragments/1420.feature @@ -0,0 +1 @@ +Encrypt file attachments diff --git a/changelogs/client_server/newsfragments/1617.feature b/changelogs/client_server/newsfragments/1617.feature new file mode 100644 index 00000000..b2169dd5 --- /dev/null +++ b/changelogs/client_server/newsfragments/1617.feature @@ -0,0 +1 @@ +Add "rich replies" - a way for users to better represent the conversation thread they are referencing in their messages. diff --git a/changelogs/client_server/newsfragments/1635.feature b/changelogs/client_server/newsfragments/1635.feature new file mode 100644 index 00000000..3a5bb45f --- /dev/null +++ b/changelogs/client_server/newsfragments/1635.feature @@ -0,0 +1 @@ +Add support for read markers. diff --git a/changelogs/client_server/newsfragments/1635.new b/changelogs/client_server/newsfragments/1635.new new file mode 100644 index 00000000..ec57e74b --- /dev/null +++ b/changelogs/client_server/newsfragments/1635.new @@ -0,0 +1 @@ +POST ``/rooms/{roomId}/read_markers`` diff --git a/changelogs/client_server/newsfragments/1642.clarification b/changelogs/client_server/newsfragments/1642.clarification new file mode 100644 index 00000000..0ccb0f3c --- /dev/null +++ b/changelogs/client_server/newsfragments/1642.clarification @@ -0,0 +1 @@ +Clarify and generalise the language used for describing pagination. diff --git a/event-schemas/examples/m.fully_read b/event-schemas/examples/m.fully_read new file mode 100644 index 00000000..8278c6fa --- /dev/null +++ b/event-schemas/examples/m.fully_read @@ -0,0 +1,8 @@ +{ + "$ref": "core/event.json", + "type": "m.fully_read", + "room_id": "!somewhere:domain.com", + "content": { + "event_id": "$someplace:domain.com" + } +} diff --git a/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml b/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml index 4d2a9964..8ff27b1e 100644 --- a/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml +++ b/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml @@ -18,8 +18,16 @@ properties: description: Size of the image in bytes. type: integer thumbnail_url: - description: The URL to a thumbnail of the image. + description: |- + The URL to a thumbnail of the image. Only present if the + thumbnail is unencrypted. type: string + thumbnail_file: + description: |- + Information on the encrypted thumbnail file, as specified in + |encrypted_files|_. Only present if the thumbnail is encrypted. + title: EncryptedFile + type: object thumbnail_info: allOf: - $ref: thumbnail_info.yaml diff --git a/event-schemas/schema/m.fully_read b/event-schemas/schema/m.fully_read new file mode 100644 index 00000000..51a1942f --- /dev/null +++ b/event-schemas/schema/m.fully_read @@ -0,0 +1,29 @@ +{ + "type": "object", + "title": "Read Marker Location Event", + "description": "The current location of the user's read marker in a room. This event appears in the user's room account data for the room the marker is applicable for.", + "allOf": [{ + "$ref": "core-event-schema/event.yaml" + }], + "properties": { + "content": { + "type": "object", + "properties": { + "event_id": { + "type": "string", + "description": "The event the user's read marker is located at in the room." + } + }, + "required": ["event_id"] + }, + "type": { + "type": "string", + "enum": ["m.fully_read"] + }, + "room_id": { + "type": "string", + "description": "The room ID the read marker applies to." + } + }, + "required": ["type", "room_id", "content"] +} diff --git a/event-schemas/schema/m.room.message#m.audio b/event-schemas/schema/m.room.message#m.audio index f15c71a3..c258b85f 100644 --- a/event-schemas/schema/m.room.message#m.audio +++ b/event-schemas/schema/m.room.message#m.audio @@ -27,12 +27,17 @@ properties: - m.audio type: string url: - description: The URL to the audio clip. + description: Required if the file is not encrypted. The URL to the audio clip. type: string + file: + description: |- + Required if the file is encrypted. Information on the encrypted + file, as specified in |encrypted_files|_. + title: EncryptedFile + type: object required: - msgtype - body - - url type: object type: enum: diff --git a/event-schemas/schema/m.room.message#m.file b/event-schemas/schema/m.room.message#m.file index 76e59e09..2fb4fe50 100644 --- a/event-schemas/schema/m.room.message#m.file +++ b/event-schemas/schema/m.room.message#m.file @@ -21,8 +21,16 @@ properties: description: The size of the file in bytes. type: integer thumbnail_url: - description: The URL to the thumbnail of the file. + description: |- + The URL to the thumbnail of the file. Only present if the + thumbnail is unencrypted. type: string + thumbnail_file: + description: |- + Information on the encrypted thumbnail file, as specified in + |encrypted_files|_. Only present if the thumbnail is encrypted. + title: EncryptedFile + type: object thumbnail_info: allOf: - $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml @@ -34,12 +42,17 @@ properties: - m.file type: string url: - description: The URL to the file. + description: Required if the file is unencrypted. The URL to the file. type: string + file: + description: |- + Required if the file is encrypted. Information on the encrypted + file, as specified in |encrypted_files|_. + title: EncryptedFile + type: object required: - msgtype - body - - url - filename type: object type: diff --git a/event-schemas/schema/m.room.message#m.image b/event-schemas/schema/m.room.message#m.image index 1237b8f8..349f78f4 100644 --- a/event-schemas/schema/m.room.message#m.image +++ b/event-schemas/schema/m.room.message#m.image @@ -17,12 +17,17 @@ properties: - m.image type: string url: - description: The URL to the image. + description: Required if the file is unencrypted. The URL to the image. type: string + file: + description: |- + Required if the file is encrypted. Information on the encrypted + file, as specified in |encrypted_files|_. + title: EncryptedFile + type: object required: - msgtype - body - - url type: object type: enum: diff --git a/event-schemas/schema/m.room.message#m.location b/event-schemas/schema/m.room.message#m.location index e8d55769..ffc4edce 100644 --- a/event-schemas/schema/m.room.message#m.location +++ b/event-schemas/schema/m.room.message#m.location @@ -19,8 +19,16 @@ properties: type: object properties: thumbnail_url: - description: The URL to a thumbnail of the location being represented. + description: |- + The URL to the thumbnail of the file. Only present if the + thumbnail is unencrypted. type: string + thumbnail_file: + description: |- + Information on the encrypted thumbnail file, as specified in + |encrypted_files|_. Only present if the thumbnail is encrypted. + title: EncryptedFile + type: object thumbnail_info: allOf: - $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml diff --git a/event-schemas/schema/m.room.message#m.video b/event-schemas/schema/m.room.message#m.video index a0240b54..8a66fdeb 100644 --- a/event-schemas/schema/m.room.message#m.video +++ b/event-schemas/schema/m.room.message#m.video @@ -27,8 +27,16 @@ properties: description: The size of the video in bytes. type: integer thumbnail_url: - description: The URL to an image thumbnail of the video clip. + description: |- + The URL to an image thumbnail of the video clip. Only present if the + thumbnail is unencrypted. type: string + thumbnail_file: + description: |- + Information on the encrypted thumbnail file, as specified in + |encrypted_files|_. Only present if the thumbnail is encrypted. + title: EncryptedFile + type: object thumbnail_info: allOf: - $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml @@ -40,12 +48,17 @@ properties: - m.video type: string url: - description: The URL to the video clip. + description: Required if the file is unencrypted. The URL to the video clip. type: string + file: + description: |- + Required if the file is encrypted. Information on the encrypted + file, as specified in |encrypted_files|_. + title: EncryptedFile + type: object required: - msgtype - body - - url type: object type: enum: diff --git a/scripts/css/basic.css b/scripts/css/basic.css index 6411570e..1112cdfc 100644 --- a/scripts/css/basic.css +++ b/scripts/css/basic.css @@ -318,6 +318,46 @@ table.citation td { border-bottom: none; } +table.colwidths-auto caption { + font-family: 'Inconsolata', monospace; + font-weight: 800; + font-size: 120%; + padding: 5px; + text-align: left; + margin-bottom: 2px; +} + +ol, li { + margin: 0px 0px 0px 30px !important; +} + +p.httpheaders { + font-weight: 800; + font-size: 120%; + padding: 5px; + text-align: left; + margin-bottom: 2px; +} + +table.colwidths-auto { + width:100%; + margin-top: 20px; +} + +table.colwidths-auto tr td:nth-child(1) { + width: 15%; +} + +table.colwidths-auto tr td:nth-child(2) { + width: 15%; + font-family: 'Inconsolata', monospace; +} + +table.colwidths-auto tr td:nth-child(3) { + width: 70%; +} + + /* -- other body styles ----------------------------------------------------- */ ol.arabic { diff --git a/scripts/css/nature.css b/scripts/css/nature.css index 0fdcc55a..5fd4fae5 100644 --- a/scripts/css/nature.css +++ b/scripts/css/nature.css @@ -273,6 +273,7 @@ table { td[colspan]:not([colspan="1"]) { background: #eeeeee; + text-transform: capitalize; } thead { diff --git a/scripts/templating/matrix_templates/templates/common-event-fields.tmpl b/scripts/templating/matrix_templates/templates/common-event-fields.tmpl index 9d0ddac3..f62f59ba 100644 --- a/scripts/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/scripts/templating/matrix_templates/templates/common-event-fields.tmpl @@ -6,8 +6,7 @@ {{common_event.desc}} {% for table in common_event.tables %} -{{"``"+table.title+"``" if table.title else "" }} -{{ tables.paramtable(table.rows, ["Key", "Type", "Description"]) }} +{{ tables.paramtable(table.rows, ["Key", "Type", "Description"], (table.title or "")) }} {% endfor %} diff --git a/scripts/templating/matrix_templates/templates/events.tmpl b/scripts/templating/matrix_templates/templates/events.tmpl index ded33859..0955cf4c 100644 --- a/scripts/templating/matrix_templates/templates/events.tmpl +++ b/scripts/templating/matrix_templates/templates/events.tmpl @@ -5,16 +5,15 @@ {% if (event.typeof | length) %} *{{event.typeof}}* - {{event.typeof_info}} + {{event.typeof_info | indent_block(4)}} {% endif -%} {{event.desc}} {% for table in event.content_fields %} -{{"``"+table.title+"``" if table.title else "" }} -{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"], (table.title or "")) }} {% endfor %} Example{% if examples | length > 1 %}s{% endif %}: diff --git a/scripts/templating/matrix_templates/templates/http-api.tmpl b/scripts/templating/matrix_templates/templates/http-api.tmpl index 461a0b58..0b9207d9 100644 --- a/scripts/templating/matrix_templates/templates/http-api.tmpl +++ b/scripts/templating/matrix_templates/templates/http-api.tmpl @@ -13,13 +13,16 @@ {{":Rate-limited: Yes." if endpoint.rate_limited else "" }} {{":Requires auth: Yes." if endpoint.requires_auth else "" }} -Request format: +.. class:: httpheaders + + Request format: + {% if (endpoint.req_param_by_loc | length) %} {{ tables.split_paramtable(endpoint.req_param_by_loc) }} {% if (endpoint.req_body_tables) %} {% for table in endpoint.req_body_tables -%} -{{"``"+table.title+"``" if table.title else "" }} -{{ tables.paramtable(table.rows) }} +{{ tables.paramtable(table.rows, caption=(table.title or "")) }} + {% endfor -%} {% endif -%} @@ -28,30 +31,41 @@ Request format: {% endif %} {% if endpoint.res_headers is not none -%} -Response headers: + +.. class:: httpheaders + + Response headers: {{ tables.paramtable(endpoint.res_headers.rows) }} {% endif -%} {% if endpoint.res_tables|length > 0 -%} -Response format: + +.. class:: httpheaders + + Response format: {% for table in endpoint.res_tables -%} -{{"``"+table.title+"``" if table.title else "" }} -{{ tables.paramtable(table.rows) }} +{{ tables.paramtable(table.rows, caption=(table.title or "")) }} + {% endfor %} {% endif -%} -Example request: +.. class:: httpheaders + + Example request: .. code:: http {{endpoint.example.req | indent_block(2)}} {% if endpoint.responses|length > 0 -%} -Response{{"s" if endpoint.responses|length > 1 else "" }}: + +.. class:: httpheaders + + Response{{"s" if endpoint.responses|length > 1 else "" }}: {% endif -%} @@ -63,7 +77,9 @@ Response{{"s" if endpoint.responses|length > 1 else "" }}: {% if res["example"] -%} -Example +.. class:: httpheaders + + Example .. code:: json diff --git a/scripts/templating/matrix_templates/templates/msgtypes.tmpl b/scripts/templating/matrix_templates/templates/msgtypes.tmpl index 87cf4a19..060a7db0 100644 --- a/scripts/templating/matrix_templates/templates/msgtypes.tmpl +++ b/scripts/templating/matrix_templates/templates/msgtypes.tmpl @@ -4,9 +4,8 @@ {{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} -{{"``"+table.title+"``" if table.title else "" }} -{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }} +{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"], (table.title or "")) }} {% endfor %} Example: diff --git a/scripts/templating/matrix_templates/templates/tables.tmpl b/scripts/templating/matrix_templates/templates/tables.tmpl index 6d6e4f8e..fde546a7 100644 --- a/scripts/templating/matrix_templates/templates/tables.tmpl +++ b/scripts/templating/matrix_templates/templates/tables.tmpl @@ -8,8 +8,8 @@ # # 'rows' is the list of parameters. Each row should be a TypeTableRow. #} -{% macro paramtable(rows, titles=["Parameter", "Type", "Description"]) -%} -{{ split_paramtable({None: rows}, titles) }} +{% macro paramtable(rows, titles=["Parameter", "Type", "Description"], caption="") -%} +{{ split_paramtable({None: rows}, titles, caption) }} {% endmacro %} @@ -21,7 +21,7 @@ # written for that location. This is used by the standard 'paramtable' macro. #} {% macro split_paramtable(rows_by_loc, - titles=["Parameter", "Type", "Description"]) -%} + titles=["Parameter", "Type", "Description"], caption="") -%} {% set rowkeys = ['key', 'title', 'desc'] %} {% set titlerow = {'key': titles[0], 'title': titles[1], 'desc': titles[2]} %} @@ -36,6 +36,9 @@ {% set fieldwidths = (([titlerow] + flatrows) | fieldwidths(rowkeys[0:-1], [10, 10])) + [50] -%} +{{".. table:: "}}{{ caption }} +{{" :widths: auto"}} +{{""}} {{ tableheader(fieldwidths) }} {{ tablerow(fieldwidths, titlerow, rowkeys) }} {{ tableheader(fieldwidths) }} @@ -59,7 +62,7 @@ # Write a table header row, for the given column widths #} {% macro tableheader(widths) -%} -{% for arg in widths -%} +{{" "}}{% for arg in widths -%} {{"="*arg}} {% endfor -%} {% endmacro %} @@ -71,7 +74,7 @@ # attributes of 'row' to look up for values to put in the columns. #} {% macro tablerow(widths, row, keys) -%} -{% for key in keys -%} +{{" "}}{% for key in keys -%} {% set value=row[key] -%} {% if not loop.last -%} {# the first few columns need space after them -#} @@ -81,7 +84,7 @@ the preceding columns, plus the number of preceding columns (for the separators)) -#} {{ value | wrap(widths[loop.index0]) | - indent_block(widths[0:-1]|sum + loop.index0) -}} + indent_block(widths[0:-1]|sum + loop.index0 + 2) -}} {% endif -%} {% endfor -%} {% endmacro %} @@ -93,10 +96,10 @@ # write a tablespan row. This is a single value which spans the entire table. #} {% macro tablespan(widths, value) -%} -{{value}} +{{" "}}{{value}} {# we write a trailing space to stop the separator being misinterpreted # as a header line. -#} -{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} +{{" "}}{{"-"*(widths|sum + widths|length -1)}} {% endmacro %} diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 2f17e4c3..549474af 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -138,9 +138,6 @@ Some requests have unique error codes: :``M_INVALID_ROOM_STATE``: Sent when the intial state given to the ``createRoom`` API is invalid. -:``M_BAD_PAGINATION``: - Encountered when specifying bad pagination query parameters. - :``M_THREEPID_IN_USE``: Sent when a threepid given to an API cannot be used because the same threepid is already in use. @@ -1018,39 +1015,21 @@ Pagination is the process of dividing a dataset into multiple discrete pages. Matrix makes use of pagination to allow clients to view extremely large datasets. These datasets are not limited to events in a room (for example clients may want to paginate a list of rooms in addition to events within those rooms). Regardless -of *what* is being paginated, there is a common underlying API which is used to -to give clients a consistent way of selecting subsets of a potentially changing -dataset. Requests pass in ``from``, ``to``, ``dir`` and ``limit`` parameters -which describe where to read from the stream. ``from`` and ``to`` are opaque -textual 'stream tokens' which describe the current position in the dataset. -The ``dir`` parameter is an enum representing the direction of events to return: -either ``f`` orwards or ``b`` ackwards. The response returns new ``start`` and -``end`` stream token values which can then be passed to subsequent requests to -continue pagination. Not all endpoints will make use of all the parameters -outlined here: see the specific endpoint in question for more information. - -Pagination Request Query Parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Query parameters: - from: - $streamtoken - The opaque token to start streaming from. - to: - $streamtoken - The opaque token to end streaming at. Typically, - clients will not know the item of data to end at, so this will usually be - omitted. - limit: - integer - An integer representing the maximum number of items to - return. - dir: - f|b - The direction to return events in. Typically this is ``b`` to paginate - backwards in time. - -'START' and 'END' are placeholder values used in these examples to describe the -start and end of the dataset respectively. - -Unless specified, the default pagination parameters are ``from=START``, -``to=END``, without a limit set. +of what is being paginated, there is a common approach which is used to give +clients an easy way of selecting subsets of a potentially changing dataset. Each +endpoint that uses pagination may use different parameters. However the theme +among them is that they take a ``from`` and ``to`` token, and occasionally +a ``limit`` and ``dir``. Together, these parameters describe the position in a +data set, where ``from`` and ``to`` are known as "stream tokens" matching the +regular expression ``[a-zA-Z0-9.=_-]+``. If supported, the ``dir`` defines the +direction of events to return: either forwards (``f``) or backwards (``b``). +The response may contain tokens that can be used for retrieving results before +or after the returned set. These tokens may be called `start` or `prev_batch` +for retrieving the previous result set, or `end`, `next_batch` or `next_token` +for retrieving the next result set. + +In the following examples, 'START' and 'END' are placeholders to signify the +start and end of the data sets respectively. For example, if an endpoint had events E1 -> E15. The client wants the last 5 events and doesn't know any previous events:: @@ -1067,8 +1046,8 @@ events and doesn't know any previous events:: Another example: a public room list has rooms R1 -> R17. The client is showing 5 -rooms at a time on screen, and is on page 2. They want to -now show page 3 (rooms R11 -> 15):: +rooms at a time on screen, and is on page 2. They want to now show page 3 (rooms +R11 -> 15):: S E | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | stream token @@ -1086,20 +1065,17 @@ token from the initial request was '9' which corresponded to R10. When the 2nd request was made, R10 did not appear again, even though from=9 was specified. If you know the token, you already have the data. -Pagination Response -~~~~~~~~~~~~~~~~~~~ - -Responses to pagination requests MUST follow the format:: - - { - "chunk": [ ... , Responses , ... ], - "start" : $streamtoken, - "end" : $streamtoken - } - -Where $streamtoken is an opaque token which can be used in another query to -get the next set of results. The "start" and "end" keys can only be omitted if -the complete dataset is provided in "chunk". +Responses for pagination-capable endpoints SHOULD have a ``chunk`` array alongside +the applicable stream tokens to represent the result set. + +In general, when the end of a result set is reached the applicable stream token +will be excluded from the response. For example, if a user was backwards-paginating +events in a room they'd eventually reach the first event in the room. In this scenario, +the ``prev_batch`` token would be excluded from the response. Some paginated +endpoints are open-ended in one direction, such as endpoints which expose an event +stream for an active room. In this case, it is not possible for the client to reach +the true "end" of the data set and therefore should always be presented with a token +to keep moving forwards. .. _`filter`: diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index d0ed201a..f0bf285f 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -39,3 +39,10 @@ Client Behaviour ---------------- {{account_data_cs_http_api}} + + +Server Behaviour +---------------- + +Servers MUST reject clients from setting account data for event types that +the server manages. Currently, this only includes `m.fully_read`_. diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 170b70f9..68434e15 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -224,6 +224,145 @@ process: .. |device_lists| replace:: ``device_lists`` .. _`device_lists`: `device_lists_sync`_ + +Sending encrypted attachments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When encryption is enabled in a room, files should be uploaded encrypted on +the homeserver. + +In order to achieve this, a client should generate a single-use 256-bit AES +key, and encrypt the file using AES-CTR. The counter should be 64-bit long, +starting at 0 and prefixed by a random 64-bit Initialization Vector (IV), which +together form a 128-bit unique counter block. + +.. Warning:: + An IV must never be used multiple times with the same key. This implies that + if there are multiple files to encrypt in the same message, typically an + image and its thumbnail, the files must not share both the same key and IV. + +Then, the encrypted file can be uploaded to the homeserver. +The key and the IV must be included in the room event along with the resulting +``mxc://`` in order to allow recipients to decrypt the file. As the event +containing those will be Megolm encrypted, the server will never have access to +the decrypted file. + +A hash of the ciphertext must also be included, in order to prevent the homeserver from +changing the file content. + +A client should send the data as an encrypted ``m.room.message`` event, using +either ``m.file`` as the msgtype, or the appropriate msgtype for the file +type. The key is sent using the `JSON Web Key`_ format, with a `W3C +extension`_. + +.. anchor for link from m.message api spec +.. |encrypted_files| replace:: End-to-end encryption +.. _encrypted_files: + +Extensions to ``m.message`` msgtypes +<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +This module adds ``file`` and ``thumbnail_file`` properties, of type +``EncryptedFile``, to ``m.message`` msgtypes that reference files, such as +`m.file`_ and `m.image`_, replacing the ``url`` and ``thumbnail_url`` +properties. + +.. todo: generate this from a swagger definition? + +``EncryptedFile`` + +========= ================ ===================================================== +Parameter Type Description +========= ================ ===================================================== +url string **Required.** The URL to the file. +key JWK **Required.** A `JSON Web Key`_ object. +iv string **Required.** The Initialisation Vector used by + AES-CTR, encoded as unpadded base64. +hashes {string: string} **Required.** A map from an algorithm name to a hash + of the ciphertext, encoded as unpadded base64. Clients + should support the SHA-256 hash, which uses the key + ``sha256``. +v string **Required.** Version of the encrypted attachments + protocol. Must be ``v2``. +========= ================ ===================================================== + +``JWK`` + +========= ========= ============================================================ +Parameter Type Description +========= ========= ============================================================ +key string **Required.** Key type. Must be ``oct``. +key_opts [string] **Required.** Key operations. Must at least contain + ``encrypt`` and ``decrypt``. +alg string **Required.** Algorithm. Must be ``A256CTR``. +k string **Required.** The key, encoded as urlsafe unpadded base64. +ext boolean **Required.** Extractable. Must be ``true``. This is a + `W3C extension`_. +========= ========= ============================================================ + +Example: + +.. code :: json + + { + "content": { + "body": "something-important.jpg", + "file": { + "url": "mxc://domain.com/FHyPlCeYUSFFxlgbQYZmoEoe", + "mimetype": "image/jpeg", + "v": "v2", + "key": { + "alg": "A256CTR", + "ext": true, + "k": "aWF6-32KGYaC3A_FEUCk1Bt0JA37zP0wrStgmdCaW-0", + "key_ops": ["encrypt","decrypt"], + "kty": "oct" + }, + "iv": "w+sE15fzSc0AAAAAAAAAAA", + "hashes": { + "sha256": "fdSLu/YkRx3Wyh3KQabP3rd6+SFiKg5lsJZQHtkSAYA" + } + }, + "info": { + "mimetype": "image/jpeg", + "h": 1536, + "size": 422018, + "thumbnail_file": { + "hashes": { + "sha256": "/NogKqW5bz/m8xHgFiH5haFGjCNVmUIPLzfvOhHdrxY" + }, + "iv": "U+k7PfwLr6UAAAAAAAAAAA", + "key": { + "alg": "A256CTR", + "ext": true, + "k": "RMyd6zhlbifsACM1DXkCbioZ2u0SywGljTH8JmGcylg", + "key_ops": ["encrypt", "decrypt"], + "kty": "oct" + }, + "mimetype": "image/jpeg", + "url": "mxc://domain.com/pmVJxyxGlmxHposwVSlOaEOv", + "v": "v2" + }, + "thumbnail_info": { + "h": 768, + "mimetype": "image/jpeg", + "size": 211009, + "w": 432 + }, + "w": 864 + }, + "msgtype": "m.image" + }, + "event_id": "$143273582443PhrSn:domain.com", + "origin_server_ts": 1432735824653, + "room_id": "!jEsUZKDJdhlrceRyVU:domain.com", + "sender": "@example:domain.com", + "type": "m.room.message", + "unsigned": { + "age": 1234 + } + } + Claiming one-time keys ~~~~~~~~~~~~~~~~~~~~~~ @@ -583,6 +722,8 @@ Example response: .. _curve25519: https://cr.yp.to/ecdh.html .. _`Olm specification`: http://matrix.org/docs/spec/olm.html .. _`Megolm specification`: http://matrix.org/docs/spec/megolm.html +.. _`JSON Web Key`: https://tools.ietf.org/html/rfc7517#appendix-A.3 +.. _`W3C extension`: https://w3c.github.io/webcrypto/#iana-section-jwk .. _`Signing JSON`: ../appendices.html#signing-json diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 2845dba2..406911ea 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -62,9 +62,9 @@ support an additional ``format`` parameter of ``org.matrix.custom.html``. When this field is present, a ``formatted_body`` with the HTML must be provided. The plain text version of the HTML should be provided in the ``body``. -Clients should limit the HTML they render to avoid Cross-Site Scripting, HTML +Clients should limit the HTML they render to avoid Cross-Site Scripting, HTML injection, and similar attacks. The strongly suggested set of HTML tags to permit, -denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, +denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, ``h2``, ``h3``, ``h4``, ``h5``, ``h6``, ``blockquote``, ``p``, ``a``, ``ul``, ``ol``, ``sup``, ``sub``, ``li``, ``b``, ``i``, ``u``, ``strong``, ``em``, ``strike``, ``code``, ``hr``, ``br``, ``div``, ``table``, ``thead``, ``tbody``, @@ -73,7 +73,7 @@ denying the use and rendering of anything else, is: ``font``, ``del``, ``h1``, Not all attributes on those tags should be permitted as they may be avenues for other disruption attempts, such as adding ``onclick`` handlers or excessively large text. Clients should only permit the attributes listed for the tags below. -Where ``data-mx-bg-color`` and ``data-mx-color`` are listed, clients should +Where ``data-mx-bg-color`` and ``data-mx-color`` are listed, clients should translate the value (a 6-character hex color code) to the appropriate CSS/attributes for the tag. @@ -112,6 +112,13 @@ surrounding text due to Rich Text Editors. HTML included in events should otherw such as having appropriate closing tags, appropriate attributes (considering the custom ones defined in this specification), and generally valid structure. +A special tag, ``mx-reply``, may appear on rich replies (described below) and should be +allowed if, and only if, the tag appears as the very first tag in the ``formatted_body``. +The tag cannot be nested and cannot be located after another tag in the tree. Because the +tag contains HTML, an ``mx-reply`` is expected to have a partner closing tag and should +be treated similar to a ``div``. Clients that support rich replies will end up stripping +the tag and its contents and therefore may wish to exclude the tag entirely. + .. Note:: A future iteration of the specification will support more powerful and extensible message formatting options, such as the proposal `MSC1225 `_. @@ -346,6 +353,192 @@ change unexpectedly. an English-language implementation on them all? See https://matrix.org/jira/browse/SPEC-425. + +Forming relationships between events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In some cases, events may wish to reference other events. This could be to form +a thread of messages for the user to follow along with, or to provide more context +as to what a particular event is describing. Currently, the only kind of relation +defined is a "rich reply" where a user may reference another message to create a +thread-like conversation. + +Relationships are defined under an ``m.relates_to`` key in the event's ``content``. +If the event is of the type ``m.room.encrypted``, the ``m.relates_to`` key MUST NOT +be covered by the encryption and instead be put alongside the encryption information +held in the ``content``. + + +Rich replies +++++++++++++ + +Users may wish to reference another message when forming their own message, and +clients may wish to better embed the referenced message for the user to have a +better context for the conversation being had. This sort of embedding another +message in a message is known as a "rich reply", or occasionally just a "reply". + +A rich reply is formed through use of an ``m.relates_to`` relation for ``m.in_reply_to`` +where a single key, ``event_id``, is used to reference the event being replied to. +The referenced event ID SHOULD belong to the same room where the reply is being sent. +Clients should be cautious of the event ID belonging to another room, or being invalid +entirely. Rich replies can only be constructed in the form of ``m.room.message`` events +with a ``msgtype`` of ``m.text`` or ``m.notice``. Due to the fallback requirements, rich +replies cannot be constructed for types of ``m.emote``, ``m.file``, etc. Rich replies +may reference any other ``m.room.message`` event, however. Rich replies may reference +another event which also has a rich reply, infinitely. + +An ``m.in_reply_to`` relationship looks like the following:: + + { + ... + "type": "m.room.message", + "content": { + "msgtype": "m.text", + "body": "", + "format": "org.matrix.custom.html", + "formatted_body": "", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$another:event.com" + } + } + } + } + + +Fallbacks and event representation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some clients may not have support for rich replies and therefore need a fallback +to use instead. Clients that do not support rich replies should render the event +as if rich replies were not special. + +Clients that do support rich replies MUST provide the fallback format on replies, +and MUST strip the fallback before rendering the reply. Rich replies MUST have +a ``format`` of ``org.matrix.custom.html`` and therefore a ``formatted_body`` +alongside the ``body`` and appropriate ``msgtype``. The specific fallback text +is different for each ``msgtype``, however the general format for the ``body`` is: + +.. code-block:: text + + > <@alice:example.org> This is the original body + + This is where the reply goes + + +The ``formatted_body`` should use the following template: + +.. code-block:: html + + +
+ In reply to + @alice:example.org +
+ +
+ + This is where the reply goes. + + +If the related event does not have a ``formatted_body``, the event's ``body`` should +be considered after encoding any HTML special characters. Note that the ``href`` in +both of the anchors use a `matrix.to URI <../appendices.html#matrix-to-navigation>`_. + +Stripping the fallback +`````````````````````` + +Clients which support rich replies MUST strip the fallback from the event before +rendering the event. This is because the text provided in the fallback cannot be +trusted to be an accurate representation of the event. After removing the fallback, +clients are recommended to represent the event referenced by ``m.in_reply_to`` +similar to the fallback's representation, although clients do have creative freedom +for their user interface. Clients should prefer the ``formatted_body`` over the +``body``, just like with other ``m.room.message`` events. + +To strip the fallback on the ``body``, the client should iterate over each line of +the string, removing any lines that start with the fallback prefix ("> ", +including the space, without quotes) and stopping when a line is encountered without +the prefix. This prefix is known as the "fallback prefix sequence". + +To strip the fallback on the ``formatted_body``, the client should remove the +entirety of the ``mx-reply`` tag. + +Fallback for ``m.text``, ``m.notice``, and unrecognised message types +````````````````````````````````````````````````````````````````````` + +Using the prefix sequence, the first line of the related event's ``body`` should +be prefixed with the user's ID, followed by each line being prefixed with the fallback +prefix sequence. For example:: + + > <@alice:example.org> This is the first line + > This is the second line + + This is the reply + + +The ``formatted_body`` uses the template defined earlier in this section. + +Fallback for ``m.emote`` +```````````````````````` + +Similar to the fallback for ``m.text``, each line gets prefixed with the fallback +prefix sequence. However an asterisk should be inserted before the user's ID, like +so:: + + > * <@alice:example.org> feels like today is going to be a great day + + This is the reply + + +The ``formatted_body`` has a subtle difference for the template where the asterisk +is also inserted ahead of the user's ID: + +.. code-block:: html + + +
+ In reply to + * @alice:example.org +
+ +
+ + This is where the reply goes. + + +Fallback for ``m.image``, ``m.video``, ``m.audio``, and ``m.file`` +`````````````````````````````````````````````````````````````````` + +The related event's ``body`` would be a file name, which may not be very descriptive. +The related event should additionally not have a ``format`` or ``formatted_body`` +in the ``content`` - if the event does have a ``format`` and/or ``formatted_body``, +those fields should be ignored. Because the filename alone may not be descriptive, +the related event's ``body`` should be considered to be ``"sent a file."`` such that +the output looks similar to the following:: + + > <@alice:example.org> sent a file. + + This is the reply + + +.. code-block:: html + + +
+ In reply to + @alice:example.org +
+ sent a file. +
+ + This is where the reply goes. + + +For ``m.image``, the text should be ``"sent an image."``. For ``m.video``, the text +should be ``"sent a video."``. For ``m.audio``, the text should be ``"sent an audio file"``. + + Server behaviour ---------------- @@ -362,4 +555,4 @@ Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross- Scripting (XSS) attacks. This includes room names and topics. .. _`E2E module`: `module:e2e`_ -.. _`Matrix Content (MXC) URI`: `module:content`_ \ No newline at end of file +.. _`Matrix Content (MXC) URI`: `module:content`_ diff --git a/specification/modules/read_markers.rst b/specification/modules/read_markers.rst new file mode 100644 index 00000000..b2f77bc0 --- /dev/null +++ b/specification/modules/read_markers.rst @@ -0,0 +1,67 @@ +.. Copyright 2018 New Vector Ltd +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +Fully read markers +================== + +.. _module:read-markers: + +The history for a given room may be split into three sections: messages the +user has read (or indicated they aren't interested in them), messages the user +might have read some but not others, and messages the user hasn't seen yet. +The "fully read marker" (also known as a "read marker") marks the last event +of the first section, whereas the user's read receipt marks the last event of +the second section. + +Events +------ +The user's fully read marker is kept as an event in the room's `account data`_. +The event may be read to determine the user's current fully read marker location +in the room, and just like other account data events the event will be pushed down +the event stream when updated. + +The fully read marker is kept under an ``m.fully_read`` event. If the event does +not exist on the user's account data, the fully read marker should be considered +to be the user's read receipt location. + +{{m_fully_read_event}} + +Client behaviour +---------------- +The client cannot update fully read markers by directly modifying the ``m.fully_read`` +account data event. Instead, the client must make use of the read markers API +to change the values. + +The read markers API can additionally update the user's read receipt (``m.read``) +location in the same operation as setting the fully read marker location. This is +because read receipts and read markers are commonly updated at the same time, +and therefore the client might wish to save an extra HTTP call. Providing an +``m.read`` location performs the same task as a request to ``/receipts/m.read/$event:domain.com``. + +{{read_markers_cs_http_api}} + +Server behaviour +---------------- +The server MUST prevent clients from setting ``m.fully_read`` directly in +room account data. The server must additionally ensure that it treats the +presence of ``m.read`` in the ``/read_markers`` request the same as how it +would for a request to ``/receipts/m.read/$event:domain.com``. + +Upon updating the ``m.fully_read`` event due to a request to ``/read_markers``, +the server MUST send the updated account data event through to the client via +the event stream (eg: ``/sync``), provided any applicable filters are also +satisfied. + + +.. _`account data`: #client-config diff --git a/specification/targets.yaml b/specification/targets.yaml index 56e9ec34..a2eb5cdd 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -45,6 +45,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst + - modules/read_markers.rst - modules/presence.rst - modules/content_repo.rst - modules/send_to_device.rst