From 902c7d3ea69c56b9c1576756123d98721135af12 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 19 Aug 2015 15:00:22 +0100 Subject: [PATCH 001/223] Add draft macaroon caveat specification --- drafts/macaroons_caveats.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 drafts/macaroons_caveats.rst diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst new file mode 100644 index 000000000..b6920d6cb --- /dev/null +++ b/drafts/macaroons_caveats.rst @@ -0,0 +1,30 @@ +Macaroon Caveats +================ + +Macaroons (http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) are issued by Matrix servers as authorization tokens. Macaroons may be restricted, by adding caveats to them. + +Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. + +Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed. + +All caveats must take the form: + +`key` `operator` `value` +where `key` is a non-empty string drawn from the character set [A-Za-z0-9_] +`operator` is a non-empty string which does not contain whitespace +`value` is a non-empty string +And these are joined by single space characters. + +Specified caveats: + ++-------------+--------------------------------------------------+--------------------------------------------------------------------------------------------+ +| Caveat name | Description | Legal Values | ++-------------+--------------------------------------------------+--------------------------------------------------------------------------------------------+ +| gen | Generation of the macaroon caveat spec. | 1 | +| user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. | +| type | The purpose of this macaroon. | access - used to authorize any action except token refresh | +| refresh - only used to authorize a token refresh | +| time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). | +| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. | +| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. | ++-------------+--------------------------------------------------+--------------------------------------------------------------------------------------------+ From 0440983c4ab74f30a38f27d70c51f47ea1a8ff30 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 28 Aug 2015 10:31:45 +0100 Subject: [PATCH 002/223] Spec exchanging refresh tokens for new access tokens --- specification/10_client_server_api.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index 83c8b7a89..6b8ffbcce 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -1087,10 +1087,33 @@ On success, this returns a JSON object with keys: user_id The fully-qualified Matrix ID that has been registered. access_token - An access token for the new account. + An access token for the account. This token may expire at some point, and if + so, it MAY come with a refersh_token, described below. +refresh_token (optional) + A refresh token may be exchanged for a new access_token as described in + `Refreshing access tokens`. home_server The hostname of the Home Server on which the account has been registered. +Refreshing access tokens +~~~~~~~~~~~~~~~~~~~~~~~~ +Exchanging a refresh token for an access token is done using the request:: + + POST $PREFIX/tokenrefresh + +The body of the POST request is a JSON object containing: + +refresh_token + The refresh token. + +On success, this invalidates the refresh token, so that it cannot be used again, +and returns a JSON object with keys: + +access_token + An access token for the account, as is returned from login. +refresh_token (optional) + A refresh token, as is returned from login. + Changing Password ~~~~~~~~~~~~~~~~~ This section refers to API Version 2. These API calls currently use the prefix From 9515640d5e0ef51e075a0635ebd8261888e04501 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 28 Aug 2015 10:41:49 +0100 Subject: [PATCH 003/223] Add note about no specific 'token expired' error --- specification/10_client_server_api.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index 6b8ffbcce..5dd215d43 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -1114,6 +1114,11 @@ access_token refresh_token (optional) A refresh token, as is returned from login. +There is no specific error message to indicate that a request has failed because +an access token has expired; instead, if a client has reason to believe its +access token is valid, and it receives an auth error, they should attempt to +refresh for a new token on failure, and re-try the request with the new token. + Changing Password ~~~~~~~~~~~~~~~~~ This section refers to API Version 2. These API calls currently use the prefix From 0dc22e721755fa31ca4af105459275eda2762707 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 Aug 2015 14:50:31 +0100 Subject: [PATCH 004/223] Document receipts --- specification/46_receipts.rst | 64 +++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 specification/46_receipts.rst diff --git a/specification/46_receipts.rst b/specification/46_receipts.rst new file mode 100644 index 000000000..43b849478 --- /dev/null +++ b/specification/46_receipts.rst @@ -0,0 +1,64 @@ +Receipts +======== + +Receipts are used to publish which events in a room the user or their devices +have interacted with. For example, which events the user has read. + +For efficiency this is done as "up to" markers, i.e. marking a particular event +as, e.g., read indicates the user has read all events *up to* that event. + +Client-Server API +----------------- + +Clients will receive receipts in the following format:: + + { + "type": "m.receipt", + "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org" + "content": { + "$1435641916114394fHBLK:matrix.org": { + "read": { + "@erikj:jki.re": { "ts": 1436451550453 }, + ... + } + }, + ... + } + } + +For efficiency, receipts are batched into one event per room. In the initialSync +and v2 sync APIs the receipts are listed in a seperate top level receipts key. + +Each ``user_id``, ``receipt_type`` pair must be associated with only a single +``event_id``. + +New receipts that come down the event streams are deltas. Deltas update +existing mappings, clobbering based on ``user_id``, ``receipt_type`` pairs. + + +A client can update the markers for its user by issuing a request:: + + POST /_matrix/client/v2_alpha/rooms//receipt/read/ + +Where the contents will of the POST will be included in the content sent to +other users. The server will automatically set the ``ts`` field. + + +Server-Server API +----------------- + +Receipts are sent across federation as EDUs with type ``m.receipt``. The +format of the EDUs are:: + + { + : { + : { + : { } + }, + .... + }, + ... + } + +These are always sent as deltas to previously sent reciepts. + From 83dfc2bf61ef776ad32f7a897c82a67f95e0caa3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 Aug 2015 15:01:02 +0100 Subject: [PATCH 005/223] Fix some mistakes/typos --- specification/46_receipts.rst | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/specification/46_receipts.rst b/specification/46_receipts.rst index 43b849478..6428d6b5c 100644 --- a/specification/46_receipts.rst +++ b/specification/46_receipts.rst @@ -5,16 +5,32 @@ Receipts are used to publish which events in a room the user or their devices have interacted with. For example, which events the user has read. For efficiency this is done as "up to" markers, i.e. marking a particular event -as, e.g., read indicates the user has read all events *up to* that event. +as, say, ``read`` indicates the user has read all events *up to* that event. Client-Server API ----------------- Clients will receive receipts in the following format:: + { + "type": "m.receipt", + "room_id": , + "content": { + : { + : { + : { "ts": , ... }, + ... + } + }, + ... + } + } + +For example:: + { "type": "m.receipt", - "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org" + "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", "content": { "$1435641916114394fHBLK:matrix.org": { "read": { @@ -27,7 +43,8 @@ Clients will receive receipts in the following format:: } For efficiency, receipts are batched into one event per room. In the initialSync -and v2 sync APIs the receipts are listed in a seperate top level receipts key. +and v2 sync APIs the receipts are listed in a seperate top level ``receipts`` +key. Each ``user_id``, ``receipt_type`` pair must be associated with only a single ``event_id``. @@ -40,7 +57,7 @@ A client can update the markers for its user by issuing a request:: POST /_matrix/client/v2_alpha/rooms//receipt/read/ -Where the contents will of the POST will be included in the content sent to +Where the contents of the ``POST`` will be included in the content sent to other users. The server will automatically set the ``ts`` field. @@ -55,7 +72,7 @@ format of the EDUs are:: : { : { } }, - .... + ... }, ... } From 0414ff4cc346eb171d1a0e793af36c3fb4dac33b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 7 Sep 2015 16:27:18 +0100 Subject: [PATCH 006/223] Output error if command fails --- scripts/gendoc.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f8ec6316b..e3e8d3d44 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -32,17 +32,23 @@ def rst2html(i, o): ) def run_through_template(input): - null = open(os.devnull, 'w') - subprocess.check_output( - [ - 'python', 'build.py', - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ], - stderr=null, - cwd="../templating", - ) + tmpfile = './tmp/output' + try: + with open(tmpfile, 'w') as out: + subprocess.check_output( + [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ], + stderr=out, + cwd="../templating", + ) + except subprocess.CalledProcessError as e: + with open(tmpfile, 'r') as f: + print f.read() + raise def prepare_env(): try: From c74c1ab098876e26efe561a5ecd6a1cc88d76284 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 17:09:41 +0100 Subject: [PATCH 007/223] SPEC-216: Add rooms that have been left to initial sync --- api/client-server/v1/sync.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index a08146b47..777e27db8 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -285,8 +285,11 @@ paths: chunk: type: array description: |- - A list of the most recent messages for this room. This - array will consist of at most ``limit`` elements. + If the user is a member of the room this will be a + list of the most recent messages for this room. If + the user has left the room this will be the + messages that preceeded them leaving. This array + will consist of at most ``limit`` elements. items: type: object title: RoomEvent @@ -296,8 +299,10 @@ paths: state: type: array description: |- - A list of state events representing the current state - of the room. + If the user is a member of the room this will be the + current state of the room as a list of events. If the + user has left the room this will be the state of the + room when they left it. items: title: StateEvent type: object @@ -347,4 +352,4 @@ paths: allOf: - "$ref": "definitions/event.yaml" 404: - description: The event was not found or you do not have permission to read this event. \ No newline at end of file + description: The event was not found or you do not have permission to read this event. From 94b13c01218864fac8f9a8e9ce3544194c589719 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 7 Sep 2015 17:14:36 +0100 Subject: [PATCH 008/223] Show all responses, not just the successful one This still filters out responses lacking either a description or an example --- .../matrix_templates/templates/http-api.tmpl | 17 +++++++++++++++-- templating/matrix_templates/units.py | 16 +++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 5e80f1686..1384daab5 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -46,6 +46,19 @@ Example request:: {{endpoint.example.req | indent_block(2)}} -Example response:: +{% if endpoint.example.responses|length > 0 -%} +Response{{"s" if endpoint.example.responses|length > 1 else "" }}: - {{endpoint.example.res | indent_block(2)}} \ No newline at end of file +{% endif -%} + +{% for res in endpoint.example.responses -%} + +**Status code {{res["code"]}}:** + +{{res["description"]}} + +Example:: + + {{res["example"] | indent_block(2)}} + +{% endfor %} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index d607160fd..42ce3f225 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -177,11 +177,17 @@ class MatrixUnits(Units): endpoint["req_param_by_loc"][p["loc"]] = [] endpoint["req_param_by_loc"][p["loc"]].append(p) - # add example response if it has one - res = single_api["responses"][200] # get the 200 OK response - endpoint["example"]["res"] = res.get("examples", {}).get( - "application/json", "" - ) + endpoint["example"]["responses"] = [] # Ordered list of maps + for code, res in single_api["responses"].items(): + description = res.get("description", "") + example = res.get("examples", {}).get("application/json", "") + if description and example: + endpoint["example"]["responses"].append({ + "code": code, + "description": description, + "example": example, + }) + # form example request if it has one. It "has one" if all params # have either "x-example" or a "schema" with an "example". params_missing_examples = [ From 0a9f61029a82ad67f9b02e054fd5238e976905d3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 11:16:48 +0100 Subject: [PATCH 009/223] Allow for missing responses key --- templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 42ce3f225..5852b438f 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -178,7 +178,7 @@ class MatrixUnits(Units): endpoint["req_param_by_loc"][p["loc"]].append(p) endpoint["example"]["responses"] = [] # Ordered list of maps - for code, res in single_api["responses"].items(): + for code, res in single_api.get("responses", {}).items(): description = res.get("description", "") example = res.get("examples", {}).get("application/json", "") if description and example: From 5031c26f7b4b39ece5bb04df4be38ff0b3000292 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 11:21:19 +0100 Subject: [PATCH 010/223] Make res optional, and rename to good_response --- templating/matrix_templates/units.py | 42 ++++++++++++++++------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 5852b438f..c4ed8358c 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -116,7 +116,8 @@ class MatrixUnits(Units): "res_tables": [], "example": { "req": "", - "res": "" + "responses": [], + "good_response": "" } } self.log(".o.O.o. Endpoint: %s %s" % (method, path)) @@ -177,8 +178,10 @@ class MatrixUnits(Units): endpoint["req_param_by_loc"][p["loc"]] = [] endpoint["req_param_by_loc"][p["loc"]].append(p) - endpoint["example"]["responses"] = [] # Ordered list of maps + good_response = None for code, res in single_api.get("responses", {}).items(): + if not good_response and code == 200: + good_response = res description = res.get("description", "") example = res.get("examples", {}).get("application/json", "") if description and example: @@ -222,22 +225,25 @@ class MatrixUnits(Units): ) # add response params if this API has any. - res_type = Units.prop(res, "schema/type") - if res_type and res_type not in ["object", "array"]: - # response is a raw string or something like that - endpoint["res_tables"].append({ - "title": None, - "rows": [{ - "key": res["schema"].get("name", ""), - "type": res_type, - "desc": res.get("description", "") - }] - }) - elif res_type and Units.prop(res, "schema/properties"): # object - res_tables = get_json_schema_object_fields(res["schema"]) - for table in res_tables: - if "no-table" not in table: - endpoint["res_tables"].append(table) + if good_response: + res_type = Units.prop(good_response, "schema/type") + if res_type and res_type not in ["object", "array"]: + # response is a raw string or something like that + endpoint["res_tables"].append({ + "title": None, + "rows": [{ + "key": good_response["schema"].get("name", ""), + "type": res_type, + "desc": res.get("description", "") + }] + }) + elif res_type and Units.prop(good_response, "schema/properties"): + # response is an object: + schema = good_response["schema"] + res_tables = get_json_schema_object_fields(schema) + for table in res_tables: + if "no-table" not in table: + endpoint["res_tables"].append(table) endpoints.append(endpoint) From 26ebe3e68b413c3151b12885da713f9be556e80b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 16:27:20 +0100 Subject: [PATCH 011/223] Add ability to refer to aliases of endpoints --- .../matrix_templates/templates/http-api.tmpl | 6 ++++++ templating/matrix_templates/units.py | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index 1384daab5..a0b25924f 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -1,5 +1,10 @@ ``{{endpoint.method}} {{endpoint.path}}`` {{(5 + (endpoint.path | length) + (endpoint.method | length)) * title_kind}} +{% if "alias_for_path" in endpoint -%} +``{{endpoint.path}}`` is an alias for `{{endpoint.alias_for_path}}`_. + +.. _`{{endpoint.alias_for_path}}`: #{{endpoint.alias_link}} +{% else -%} {{endpoint.desc | wrap(80)}} @@ -62,3 +67,4 @@ Example:: {{res["example"] | indent_block(2)}} {% endfor %} +{% endif -%} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index c4ed8358c..89b34bb1d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -105,11 +105,24 @@ class MatrixUnits(Units): for path in api["paths"]: for method in api["paths"][path]: single_api = api["paths"][path][method] + + full_path = api.get("basePath", "") + path + alias_for_path = single_api.get("x-alias-for-path", "") + alias_link = single_api.get("x-alias-link", "") + if alias_for_path: + endpoints.append({ + "method": method.upper(), + "path": full_path, + "alias_for_path": alias_for_path, + "alias_link": alias_link + }) + continue + endpoint = { "title": single_api.get("summary", ""), "desc": single_api.get("description", single_api.get("summary", "")), "method": method.upper(), - "path": api.get("basePath", "") + path, + "path": full_path, "requires_auth": "security" in single_api, "rate_limited": 429 in single_api.get("responses", {}), "req_params": [], From d53943c8c2aa266d894d7fa0b5a2a3a8545f5c25 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 16:35:08 +0100 Subject: [PATCH 012/223] Spec /room/{roomId}/join This is just replacing the existing spec with a swagger version. Subsequent pull requests will add 3pid join to this, as well as specing the invite, leave, ban, and kick endpoints. --- api/client-server/v1/membership.yaml | 68 +++++++++++++++++++++++++ specification/10_client_server_api.rst | 38 +------------- templating/matrix_templates/sections.py | 6 +++ 3 files changed, 75 insertions(+), 37 deletions(-) create mode 100644 api/client-server/v1/membership.yaml diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml new file mode 100644 index 000000000..24f3835df --- /dev/null +++ b/api/client-server/v1/membership.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Room Membership API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/room/{roomId}/join": + post: + summary: Start the requesting user participating in a particular room. + description: |- + This API starts a user participating in a particular room, if that user + is allowed to participate in that room. After this call, the client is + allowed to see all current state events in the room, and all subsequent + events associated with the room until the user leaves the room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to join. + required: true + x-example: "#monkeys:matrix.org" + responses: + 200: + description: |- + The room has been joined. + + If the room was joined with an room alias, rather than a room ID, + the room ID must be returned in the room_id field. + + If the room was joined with a room ID, the room_id field must not be + present. + examples: + application/json: |- + {"room_id": "!primates:matrix.org"} + schema: + type: object # empty json object + 403: + description: |- + You do not have permission to join the room. A meaningful errcode and description error text will be returned. Example reasons for rejection are: + - The room is invite-only and the user was not invited. + - The user has been banned from the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + "/join/{roomId}": + post: + x-alias-for-path: "/room/{roomId}/join" + x-alias-link: "post-matrix-client-api-v1-room-roomid-join" diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index 83c8b7a89..d0942ad67 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -880,43 +880,7 @@ certain operations such as kicking, banning and sending state events. See `m.room.power_levels`_ for more information. -Joining rooms -~~~~~~~~~~~~~ -.. TODO-doc What does the home server have to do to join a user to a room? - - See SPEC-30. - -Users need to join a room in order to send and receive events in that room. A -user can join a room by making a request to |/join/|_ with:: - - {} - -Alternatively, a user can make a request to |/rooms//join|_ with the -same request content. This is only provided for symmetry with the other -membership APIs: ``/rooms//invite`` and ``/rooms//leave``. If -a room alias was specified, it will be automatically resolved to a room ID, -which will then be joined. The room ID that was joined will be returned in -response:: - - { - "room_id": "!roomid:domain" - } - -The membership state for the joining user can also be modified directly to be -``join`` by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "join" - } - -See the `Room events`_ section for more information on ``m.room.member``. - -After the user has joined a room, they will receive subsequent events in that -room. This room will now appear as an entry in the |initialSync|_ API. - -Some rooms enforce that a user is *invited* to a room before they can join that -room. Other rooms will allow anyone to join the room even if they have not -received an invite. +{{membership_http_api}} Inviting users ~~~~~~~~~~~~~~ diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 7bc250952..7e21434f7 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -90,6 +90,12 @@ class MatrixSections(Sections): title_kind="~" ) + def render_membership_http_api(self): + return self._render_http_api_group( + "membership", + title_kind="~" + ) + def render_room_events(self): def filterFn(eventType): return ( From e58f816ad39c26461527c2ba47d2db8d544f7cb1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 16:49:14 +0100 Subject: [PATCH 013/223] Add m.receipt schema --- event-schemas/examples/v1/m.receipt | 13 +++++++++ event-schemas/schema/v1/m.receipt | 43 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 event-schemas/examples/v1/m.receipt create mode 100644 event-schemas/schema/v1/m.receipt diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/v1/m.receipt new file mode 100644 index 000000000..83515317a --- /dev/null +++ b/event-schemas/examples/v1/m.receipt @@ -0,0 +1,13 @@ +{ + "type": "m.receipt", + "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", + "content": { + "$1435641916114394fHBLK:matrix.org": { + "read": { + "@rikj:jki.re": { + "ts": 1436451550453 + } + } + } + } +} diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt new file mode 100644 index 000000000..b9d29c300 --- /dev/null +++ b/event-schemas/schema/v1/m.receipt @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "title": "Receipt Event", + "description": "Informs the client of new receipts.", + "properties": { + "content": { + "type": "object", + "description": "The event ids which the receipts relate to.", + "patternProperties": { + "^\$": { + "type": "object", + "description": "The types of the receipts." + "additionalProperties": { + "type": "object", + "description": "User ids of the receipts", + "patternProperties": { + "^@": { + "type": "object", + "properties": { + "ts": { + "type": "number", + "description": "The timestamp the receipt was sent at" + } + } + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "type": { + "type": "string", + "enum": ["m.presence"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["room_id", "type", "content"] +} From 5157c2f52ed6dfc3019b097387d6c3678413a87e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 16:51:08 +0100 Subject: [PATCH 014/223] More escapes --- event-schemas/schema/v1/m.receipt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index b9d29c300..9848f6c68 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -8,7 +8,7 @@ "type": "object", "description": "The event ids which the receipts relate to.", "patternProperties": { - "^\$": { + "^\\$": { "type": "object", "description": "The types of the receipts." "additionalProperties": { From 53756cfd73ed18d74839b16fe4b697f0dd25cc70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 16:52:43 +0100 Subject: [PATCH 015/223] More commas --- event-schemas/schema/v1/m.receipt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 9848f6c68..3f242eb02 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -10,7 +10,7 @@ "patternProperties": { "^\\$": { "type": "object", - "description": "The types of the receipts." + "description": "The types of the receipts.", "additionalProperties": { "type": "object", "description": "User ids of the receipts", From 500b4eb32de4054281b5bbeda69df1dc5f4fdf96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 16:54:14 +0100 Subject: [PATCH 016/223] Typo --- event-schemas/schema/v1/m.receipt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 3f242eb02..4bd9f17b8 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -33,7 +33,7 @@ }, "type": { "type": "string", - "enum": ["m.presence"] + "enum": ["m.receipt"] }, "room_id": { "type": "string" From 90f5dc370b8e00b2f57ba98f0846b8bbbd7c1e12 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 16:49:28 +0100 Subject: [PATCH 017/223] Populate aliases from canonical endpoint Swagger validates badly if you have endpoints without the full specification of things, so instead let's generate them the other way around. --- templating/matrix_templates/units.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 89b34bb1d..8e991fa8a 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -105,19 +105,7 @@ class MatrixUnits(Units): for path in api["paths"]: for method in api["paths"][path]: single_api = api["paths"][path][method] - full_path = api.get("basePath", "") + path - alias_for_path = single_api.get("x-alias-for-path", "") - alias_link = single_api.get("x-alias-link", "") - if alias_for_path: - endpoints.append({ - "method": method.upper(), - "path": full_path, - "alias_for_path": alias_for_path, - "alias_link": alias_link - }) - continue - endpoint = { "title": single_api.get("summary", ""), "desc": single_api.get("description", single_api.get("summary", "")), @@ -260,6 +248,16 @@ class MatrixUnits(Units): endpoints.append(endpoint) + for alias in single_api.get("x-aliases", []): + canonical_alias_link = single_api.get("x-alias-link") + alias_link = single_api.get("x-alias-link", "") + endpoints.append({ + "method": method.upper(), + "path": alias, + "alias_for_path": full_path, + "alias_link": alias_link + }) + return { "base": api.get("basePath"), "group": group_name, From 1feb9565e4727f9c7c10619b84bb5f77976dd513 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 16:51:39 +0100 Subject: [PATCH 018/223] Use other-way-around alias format --- api/client-server/v1/membership.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index 24f3835df..1845ce095 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -62,7 +62,6 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" - "/join/{roomId}": - post: - x-alias-for-path: "/room/{roomId}/join" + x-aliases: + - /join/{roomId} x-alias-link: "post-matrix-client-api-v1-room-roomid-join" From 64cfc00f5055959f4a67d51d29711c7f4b746e97 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 8 Sep 2015 17:07:52 +0100 Subject: [PATCH 019/223] Remove unused fluff --- templating/matrix_templates/units.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 8e991fa8a..24fb3767e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -249,8 +249,7 @@ class MatrixUnits(Units): endpoints.append(endpoint) for alias in single_api.get("x-aliases", []): - canonical_alias_link = single_api.get("x-alias-link") - alias_link = single_api.get("x-alias-link", "") + alias_link = single_api.get("x-alias-link") endpoints.append({ "method": method.upper(), "path": alias, From a92fa6392d1b7eb9dd9c75f6d209506a2898ca30 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 8 Sep 2015 17:14:31 +0100 Subject: [PATCH 020/223] Include patternProperties as normal properties (they basically are just patterns instead of keys; we could probably annotate this more nicely in the future) --- templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index d607160fd..2ae6cd0b8 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -30,7 +30,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): } tables = [fields] - props = obj.get("properties") + props = obj.get("properties", obj.get("patternProperties")) parents = obj.get("allOf") if not props and not parents: raise Exception( From d399e5b93bac92bcbe82a79e6aecb90a92f38fe7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:16:46 +0100 Subject: [PATCH 021/223] Use an alias object rather than flat fields --- templating/matrix_templates/units.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 24fb3767e..d8cc213de 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -248,14 +248,16 @@ class MatrixUnits(Units): endpoints.append(endpoint) - for alias in single_api.get("x-aliases", []): - alias_link = single_api.get("x-alias-link") - endpoints.append({ - "method": method.upper(), - "path": alias, - "alias_for_path": full_path, - "alias_link": alias_link - }) + aliases = single_api.get("x-alias", None) + if aliases: + alias_link = aliases["canonical-link"] + for alias in aliases["aliases"]: + endpoints.append({ + "method": method.upper(), + "path": alias, + "alias_for_path": full_path, + "alias_link": alias_link + }) return { "base": api.get("basePath"), From 04b2b2588f1453fb62d9e86277b0f2cefc5a6207 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:18:23 +0100 Subject: [PATCH 022/223] Fix up formatting and typos --- api/client-server/v1/membership.yaml | 22 +++++++++++----------- specification/10_client_server_api.rst | 10 ++++++++++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index 1845ce095..b86e8b033 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -18,7 +18,7 @@ securityDefinitions: name: access_token in: query paths: - "/room/{roomId}/join": + "/rooms/{roomId}/join": post: summary: Start the requesting user participating in a particular room. description: |- @@ -26,6 +26,9 @@ paths: is allowed to participate in that room. After this call, the client is allowed to see all current state events in the room, and all subsequent events associated with the room until the user leaves the room. + + After a user has joined a room, the room will appear as an entry in the + response of the |initialSync| API. security: - accessToken: [] parameters: @@ -40,19 +43,15 @@ paths: description: |- The room has been joined. - If the room was joined with an room alias, rather than a room ID, - the room ID must be returned in the room_id field. - - If the room was joined with a room ID, the room_id field must not be - present. + The joined room ID must be returned in the room_id field. examples: application/json: |- - {"room_id": "!primates:matrix.org"} + {"room_id": "!d41d8cd:matrix.org"} schema: type: object # empty json object 403: description: |- - You do not have permission to join the room. A meaningful errcode and description error text will be returned. Example reasons for rejection are: + You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: - The room is invite-only and the user was not invited. - The user has been banned from the room. examples: @@ -62,6 +61,7 @@ paths: description: This request was rate-limited. schema: "$ref": "definitions/error.yaml" - x-aliases: - - /join/{roomId} - x-alias-link: "post-matrix-client-api-v1-room-roomid-join" + x-alias: + canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" + aliases: + - /join/{roomId} diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index d0942ad67..1fdb9ae6f 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -879,6 +879,16 @@ The keys contained in ``m.room.power_levels`` determine the levels required for certain operations such as kicking, banning and sending state events. See `m.room.power_levels`_ for more information. +Joining rooms +------------- +Users need to be a member of a room in order to send and receive events in that +room. There are several states in which a user may be, in relation to a room: + + - Unrelated (the user cannot send or receive events in the room) + - Invited (the user has been invited to participate in the room, but is not + yet participating) + - Joined (the user can send and receive events in the room) + - Banned (the user is not allowed to join the room) {{membership_http_api}} From 267d2bd3decd69002044b417784c30389fef3856 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:40:32 +0100 Subject: [PATCH 023/223] Add api/node_modules to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 63ef5bb3f..f4177c790 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ templating/out *.pyc supporting-docs/_site supporting-docs/.sass-cache +api/node_modules From 1b591a023ecc5a167d630a6e1dad88981edcfefc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 14:04:48 +0100 Subject: [PATCH 024/223] Monospace room_id --- api/client-server/v1/membership.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index b86e8b033..d847a78fb 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -43,7 +43,7 @@ paths: description: |- The room has been joined. - The joined room ID must be returned in the room_id field. + The joined room ID must be returned in the ``room_id`` field. examples: application/json: |- {"room_id": "!d41d8cd:matrix.org"} From 2797fac3aa2a45049d3ee1932a7470fbb10b9c3e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 17:55:40 +0100 Subject: [PATCH 025/223] Swagger /room/{roomId}/invite This is a transcription of the current spec, with nothing new or controversial in it. --- api/client-server/v1/membership.yaml | 64 +++++++++++++++++++++++++- specification/10_client_server_api.rst | 33 ++----------- 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index d847a78fb..86e84111e 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -48,7 +48,7 @@ paths: application/json: |- {"room_id": "!d41d8cd:matrix.org"} schema: - type: object # empty json object + type: object 403: description: |- You do not have permission to join the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejection are: @@ -65,3 +65,65 @@ paths: canonical-link: "post-matrix-client-api-v1-rooms-roomid-join" aliases: - /join/{roomId} + + "/rooms/{roomId}/invite": + post: + summary: Invite a user to participate in a particular room. + # It's a crying shame that I don't know how to force line breaks. + description: |- + This API invites a user to participate in a particular room. + They do not start participating in the room until they actually join the + room. + + This serves two purposes; firstly, to notify the user that the room + exists (and that their presence is requested). Secondly, some rooms can + only be joined if a user is invited to join it; sending the invite gives + that user permission to join the room. + + Only users currently in a particular room can invite other users to + join that room. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room identifier or room alias to which to invite the user. + required: true + x-example: "#monkeys:matrix.org" + - in: body + name: user_id + required: true + schema: + type: object + example: |- + { + "user_id": "@cheeky_monkey:matrix.org" + } + properties: + user_id: + type: string + description: The fully qualified user ID of the invitee. + required: ["user_id"] + responses: + 200: + description: The user has been invited to join the room. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 403: + description: |- + You do not have permission to invite the user to the room. A meaningful ``errcode`` and description error text will be returned. Example reasons for rejections are: + - The invitee has been banned from the room. + - The invitee is already a member of the room. + - The inviter is not currently in the room. + - The inviter's power level is insufficient to invite users to the room. + examples: + application/json: |- + {"errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index 1fdb9ae6f..f0d6be42e 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -890,17 +890,9 @@ room. There are several states in which a user may be, in relation to a room: - Joined (the user can send and receive events in the room) - Banned (the user is not allowed to join the room) -{{membership_http_api}} - -Inviting users -~~~~~~~~~~~~~~ -.. TODO-doc Invite-join dance - - Outline invite join dance. What is it? Why is it required? How does it work? - - What does the home server have to do? +Some rooms require that users be invited to it before they can join; others +allow anyone to join. -The purpose of inviting users to a room is to notify them that the room exists -so they can choose to become a member of that room. Some rooms require that all -users who join a room are previously invited to it (an "invite-only" room). Whether a given room is an "invite-only" room is determined by the room config key ``m.room.join_rules``. It can have one of the following values: @@ -910,26 +902,7 @@ key ``m.room.join_rules``. It can have one of the following values: ``invite`` This room can only be joined if you were invited. -Only users who have a membership state of ``join`` in a room can invite new -users to said room. The person being invited must not be in the ``join`` state -in the room. The fully-qualified user ID must be specified when inviting a -user, as the user may reside on a different home server. To invite a user, send -the following request to |/rooms//invite|_, which will manage the -entire invitation process:: - - { - "user_id": "" - } - -Alternatively, the membership state for this user in this room can be modified -directly by sending the following request to -``/rooms//state/m.room.member/``:: - - { - "membership": "invite" - } - -See the `Room events`_ section for more information on ``m.room.member``. +{{membership_http_api}} Leaving rooms ~~~~~~~~~~~~~ From c4acee3bcbf8fc99d93a912f09c565729f4aad39 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 11:16:31 +0100 Subject: [PATCH 026/223] continuserv: Live serves updates to the spec --- .gitignore | 1 + scripts/continuserv/main.go | 129 ++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 scripts/continuserv/main.go diff --git a/.gitignore b/.gitignore index f4177c790..e2250131b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ scripts/gen +scripts/continuserv/continuserv templating/out *.pyc supporting-docs/_site diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go new file mode 100644 index 000000000..26dfc4a11 --- /dev/null +++ b/scripts/continuserv/main.go @@ -0,0 +1,129 @@ +// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. +// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished. +// It does not currently pre-empt stale generations, but will block until they are complete. +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + + fsnotify "gopkg.in/fsnotify.v1" +) + +var toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. +var wg sync.WaitGroup // Indicates how many updates are pending. +var mu sync.Mutex // Prevent multiple updates in parallel. + +func main() { + w, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("Error making watcher: %v", err) + } + + dir, err := os.Getwd() + if err != nil { + log.Fatalf("Error getting wd: %v", err) + } + for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) { + if dir == "/" { + log.Fatalf("Could not find git root") + } + } + + filepath.Walk(dir, makeWalker(w)) + + wg.Add(1) + populateOnce(dir) + + ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes + go doPopulate(ch, dir) + + http.HandleFunc("/", serve) + go http.ListenAndServe(":8000", nil) + + for { + select { + case e := <-w.Events: + if filter(e) { + wg.Add(1) + ch <- struct{}{} + } + } + } +} + +func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { + return func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatalf("Error walking: %v", err) + } + w.Add(path) + return nil + } +} + +// Return true if event should trigger re-population +func filter(e fsnotify.Event) bool { + // vim is *really* noisy about how it writes files + if e.Op != fsnotify.Write { + return false + } + // Avoid some temp files that vim writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + return false + } + + // Avoid infinite cycles being caused by writing actual output + if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") { + return false + } + return true +} + +func serve(w http.ResponseWriter, req *http.Request) { + wg.Wait() + b := toServe.Load().([]byte) + w.Write(b) +} + +func populateOnce(dir string) { + defer wg.Done() + mu.Lock() + defer mu.Unlock() + cmd := exec.Command("python", "gendoc.py") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + return + } + specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + if err != nil { + toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + return + } + toServe.Store(specBytes) +} + +func doPopulate(ch chan struct{}, dir string) { + for _ = range ch { + populateOnce(dir) + } +} + +func exists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} From a54814856357db32f2e430561a9e8fd5aa381024 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 11:21:18 +0100 Subject: [PATCH 027/223] Invite doesn't accept room aliases --- api/client-server/v1/membership.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index 86e84111e..04922656d 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -88,9 +88,9 @@ paths: - in: path type: string name: roomId - description: The room identifier or room alias to which to invite the user. + description: The room identifier (not alias) to which to invite the user. required: true - x-example: "#monkeys:matrix.org" + x-example: "!d41d8cd:matrix.org" - in: body name: user_id required: true From e30272733b48a3afca3c2e5e6fd87ef1a0ff9727 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 11:29:16 +0100 Subject: [PATCH 028/223] continuserv: Make port flag-specified --- scripts/continuserv/main.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 26dfc4a11..c5d623a4a 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -5,6 +5,7 @@ package main import ( "bytes" + "flag" "fmt" "io/ioutil" "log" @@ -20,11 +21,17 @@ import ( fsnotify "gopkg.in/fsnotify.v1" ) -var toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. -var wg sync.WaitGroup // Indicates how many updates are pending. -var mu sync.Mutex // Prevent multiple updates in parallel. +var ( + port = flag.Int("port", 8000, "Port on which to serve HTTP") + + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + wg sync.WaitGroup // Indicates how many updates are pending. + mu sync.Mutex // Prevent multiple updates in parallel. +) func main() { + flag.Parse() + w, err := fsnotify.NewWatcher() if err != nil { log.Fatalf("Error making watcher: %v", err) @@ -48,9 +55,14 @@ func main() { ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes go doPopulate(ch, dir) + go watchFS(ch, w) + http.HandleFunc("/", serve) - go http.ListenAndServe(":8000", nil) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) + +} +func watchFS(ch chan struct{}, w *fsnotify.Watcher) { for { select { case e := <-w.Events: From 55f33736b2efdb21507abb0714cd2ff36357334b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 12:04:12 +0100 Subject: [PATCH 029/223] Preserve newlines in wrapped text --- templating/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templating/build.py b/templating/build.py index dadf91ce5..258e4a174 100755 --- a/templating/build.py +++ b/templating/build.py @@ -78,8 +78,10 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): def wrap(input, wrap=80, initial_indent=""): if len(input) == 0: return initial_indent + input_lines = input.split('\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) - return wrapper.fill(input) + output_lines = [wrapper.fill(line) for line in input_lines] + return '\n'.join(output_lines) # make Jinja aware of the templates and filters env = Environment( From f67e27e843bd812360f095a242be584f40719052 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:20:04 +0100 Subject: [PATCH 030/223] Split on double-newlines not single Otherwise all sorts of tables get horribly broken --- templating/build.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/templating/build.py b/templating/build.py index 258e4a174..6756bed9e 100755 --- a/templating/build.py +++ b/templating/build.py @@ -76,12 +76,10 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return " "*indent + input def wrap(input, wrap=80, initial_indent=""): - if len(input) == 0: - return initial_indent - input_lines = input.split('\n') + input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) output_lines = [wrapper.fill(line) for line in input_lines] - return '\n'.join(output_lines) + return '\n\n'.join(output_lines) # make Jinja aware of the templates and filters env = Environment( From 6a56c8a96559b576aa6925d44d86785a6329f199 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:20:56 +0100 Subject: [PATCH 031/223] Add trailing newline --- api/client-server/v1/presence.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/client-server/v1/presence.yaml b/api/client-server/v1/presence.yaml index 166653414..6f8311e48 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/v1/presence.yaml @@ -206,4 +206,3 @@ paths: title: PresenceEvent allOf: - "$ref": "definitions/event.yaml" - \ No newline at end of file From ad595aea459eedf92f568ff63e0e3522cea8c86c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:21:08 +0100 Subject: [PATCH 032/223] Optionally don't delete intermediate rst files Useful for debugging Hackily hackily implemented :) --- scripts/gendoc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index e3e8d3d44..47141d7d8 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -71,10 +71,11 @@ def main(): run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") - cleanup_env() + if "--nodelete" not in sys.argv: + cleanup_env() if __name__ == '__main__': - if len(sys.argv) > 1: + if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: # we accept no args, so they don't know what they're doing! print "gendoc.py - Generate the Matrix specification as HTML." print "Usage:" From a74fb54053eb9c247a4af1f5fa3fe59e891204d5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:44:41 +0100 Subject: [PATCH 033/223] Correct comment --- scripts/gendoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 47141d7d8..57a047174 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -76,7 +76,7 @@ def main(): if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: - # we accept no args, so they don't know what they're doing! + # we accept almost no args, so they don't know what they're doing! print "gendoc.py - Generate the Matrix specification as HTML." print "Usage:" print " python gendoc.py" From cdc06a8bf1c410147a7cd096c2a9169914285852 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:53:38 +0100 Subject: [PATCH 034/223] Document --nodelete --- scripts/gendoc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 57a047174..e3c3797e4 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -79,9 +79,11 @@ if __name__ == '__main__': # we accept almost no args, so they don't know what they're doing! print "gendoc.py - Generate the Matrix specification as HTML." print "Usage:" - print " python gendoc.py" + print " python gendoc.py [--nodelete]" print "" print "The specification can then be found in the gen/ folder." + print ("If --nodelete was specified, intermediate files will be " + "present in the tmp/ folder.") print "" print "Requirements:" print " - This script requires Jinja2 and rst2html (docutils)." From 99106bb6fefd0b2f44c1ff418491f7d54c52cc8c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 13:54:00 +0100 Subject: [PATCH 035/223] Add explanatory comments Also, remove line which was accidentally removed --- templating/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templating/build.py b/templating/build.py index 6756bed9e..ac3d2491a 100755 --- a/templating/build.py +++ b/templating/build.py @@ -76,6 +76,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return " "*indent + input def wrap(input, wrap=80, initial_indent=""): + if len(input) == 0: + return initial_indent + # TextWrapper collapses newlines into single spaces; we do our own + # splitting on newlines to prevent this, so that newlines can actually + # be intentionally inserted in text. input_lines = input.split('\n\n') wrapper = TextWrapper(initial_indent=initial_indent, width=wrap) output_lines = [wrapper.fill(line) for line in input_lines] From 34091038f39a332c70ac188992437ae7a2e79003 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:19:06 +0100 Subject: [PATCH 036/223] speculator: Tool to preview spec pull requests Not the prettiest thing in the world, and horribly unperformant, but useful :) --- scripts/speculator/README | 4 + scripts/speculator/main.go | 220 +++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 scripts/speculator/README create mode 100644 scripts/speculator/main.go diff --git a/scripts/speculator/README b/scripts/speculator/README new file mode 100644 index 000000000..1da99cfbd --- /dev/null +++ b/scripts/speculator/README @@ -0,0 +1,4 @@ +speculator allows you to preview pull requests to the matrix.org specification. + +To run it, you must install the `go` tool, and run: + `go run main.go` diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go new file mode 100644 index 000000000..59047e680 --- /dev/null +++ b/scripts/speculator/main.go @@ -0,0 +1,220 @@ +// speculator allows you to preview pull requests to the matrix.org specification. +// It serves two HTTP endpoints: +// - /spec/123 which renders the spec as html at pull request 123. +// - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. +// It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "os" + "os/exec" + "path" + "strconv" + "strings" + "syscall" +) + +type PullRequest struct { + Base Commit + Head Commit + User User +} + +type Commit struct { + SHA string + Repo RequestRepo +} + +type RequestRepo struct { + CloneURL string `json:"clone_url"` +} + +type User struct { + Login string +} + +var ( + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + allowedMembers map[string]bool +) + +func gitClone(url string) (string, error) { + dst := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) + cmd := exec.Command("git", "clone", url, dst) + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error cloning repo: %v", err) + } + return dst, nil +} + +func gitCheckout(path, sha string) error { + cmd := exec.Command("git", "checkout", sha) + cmd.Dir = path + err := cmd.Run() + if err != nil { + return fmt.Errorf("error checking out repo: %v", err) + } + return nil +} + +func lookupPullRequest(prNumber string) (PullRequest, error) { + resp, _ := http.Get("https://api.github.com/repos/matrix-org/matrix-doc/pulls/" + prNumber) + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var pr PullRequest + _ = dec.Decode(&pr) + return pr, nil +} + +func generate(dir string) error { + cmd := exec.Command("python", "gendoc.py", "--nodelete") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) + } + return nil +} + +func writeError(w http.ResponseWriter, err error) { + w.WriteHeader(500) + io.WriteString(w, fmt.Sprintf("%v\n", err)) +} + +// generateAt generates spec from repo at sha. +// Returns the path where the generation was done. +func generateAt(repo, sha string) (dst string, err error) { + dst, err = gitClone(repo) + if err != nil { + return + } + + if err = gitCheckout(dst, sha); err != nil { + return + } + + err = generate(dst) + return +} + +func serveSpec(w http.ResponseWriter, req *http.Request) { + parts := strings.Split(req.URL.Path, "/") + if len(parts) != 3 { + w.WriteHeader(400) + io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /pull/123", req.URL.Path)) + return + } + + pr, err := lookupPullRequest(parts[2]) + if err != nil { + writeError(w, err) + return + } + + if !allowedMembers[pr.User.Login] { + w.WriteHeader(403) + io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) + return + } + + dst, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, err) + return + } + + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) + if err != nil { + writeError(w, fmt.Errorf("Error reading spec: %v", err)) + return + } + w.Write(b) +} + +func serveRstDiff(w http.ResponseWriter, req *http.Request) { + parts := strings.Split(req.URL.Path, "/") + if len(parts) != 4 { + w.WriteHeader(400) + io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /diff/rst/123", req.URL.Path)) + return + } + + pr, err := lookupPullRequest(parts[3]) + if err != nil { + writeError(w, err) + return + } + + if !allowedMembers[pr.User.Login] { + w.WriteHeader(403) + io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) + return + } + + base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA) + defer os.RemoveAll(base) + if err != nil { + writeError(w, err) + return + } + + head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + defer os.RemoveAll(head) + if err != nil { + writeError(w, err) + return + } + + diffCmd := exec.Command("diff", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + var diff bytes.Buffer + diffCmd.Stdout = &diff + if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { + writeError(w, fmt.Errorf("error running diff: %v", err)) + return + } + w.Write(diff.Bytes()) +} + +func ignoreExitCodeOne(err error) error { + if err == nil { + return err + } + + if exiterr, ok := err.(*exec.ExitError); ok { + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() == 1 { + return nil + } + } + } + return err +} + +func main() { + flag.Parse() + // It would be great to read this from github, but there's no convenient way to do so. + // Most of these memberships are "private", so would require some kind of auth. + allowedMembers = map[string]bool{ + "dbkr": true, + "erikjohnston": true, + "illicitonion": true, + "Kegsay": true, + "NegativeMjark": true, + } + http.HandleFunc("/spec/", serveSpec) + http.HandleFunc("/diff/rst/", serveRstDiff) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) +} From 6e86cb34d2e8864b69c50faa4ab9b165e47857a7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:25:23 +0100 Subject: [PATCH 037/223] continuserv: Adding README --- scripts/continuserv/README | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 scripts/continuserv/README diff --git a/scripts/continuserv/README b/scripts/continuserv/README new file mode 100644 index 000000000..8ce378501 --- /dev/null +++ b/scripts/continuserv/README @@ -0,0 +1,6 @@ +continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. + +To run it, you must install the `go` tool. You will also need to install fsnotify by running: + `go get gopkg.in/fsnotify.v1` +You can then run continuserv by running: + `go run main.go` From 6ac519d9dce4016b9911b5f6ddc05889b8c2abd2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:25:37 +0100 Subject: [PATCH 038/223] Log on file updates --- scripts/continuserv/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index c5d623a4a..b42247725 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -68,6 +68,7 @@ func watchFS(ch chan struct{}, w *fsnotify.Watcher) { case e := <-w.Events: if filter(e) { wg.Add(1) + fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) ch <- struct{}{} } } From c29aef362e11ed31300f0f66a54536a4644c9a51 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 15:45:21 +0100 Subject: [PATCH 039/223] Error on failure of Watcher.Add --- scripts/continuserv/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index b42247725..e7757c06d 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -80,7 +80,9 @@ func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { if err != nil { log.Fatalf("Error walking: %v", err) } - w.Add(path) + if err := w.Add(path); err != nil { + log.Fatalf("Failed to add watch: %v", err) + } return nil } } From 708f28127057d26e7000f24752dd1a111e1c1d75 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 17:09:44 +0100 Subject: [PATCH 040/223] Add comments about auth --- scripts/speculator/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 59047e680..22fb2ff24 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -123,6 +123,8 @@ func serveSpec(w http.ResponseWriter, req *http.Request) { return } + // We're going to run whatever Python is specified in the pull request, which + // may do bad things, so only trust people we trust. if !allowedMembers[pr.User.Login] { w.WriteHeader(403) io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) @@ -158,6 +160,8 @@ func serveRstDiff(w http.ResponseWriter, req *http.Request) { return } + // We're going to run whatever Python is specified in the pull request, which + // may do bad things, so only trust people we trust. if !allowedMembers[pr.User.Login] { w.WriteHeader(403) io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) From ca7dd49dc600a8d2f3cc0dbccbfd7f81e15185a2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 17:14:53 +0100 Subject: [PATCH 041/223] speculator: Add /healthz endpoint --- scripts/speculator/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 22fb2ff24..3775ad863 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -220,5 +220,12 @@ func main() { } http.HandleFunc("/spec/", serveSpec) http.HandleFunc("/diff/rst/", serveRstDiff) + http.HandleFunc("/healthz", serveText("ok")) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } + +func serveText(s string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, s) + } +} From bf318ffcd53e4b01ea20c4a3bbe566a653360815 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 12:42:41 +0100 Subject: [PATCH 042/223] Add HTTP endpoints to README --- scripts/speculator/README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/README b/scripts/speculator/README index 1da99cfbd..ae00c55ad 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -1,4 +1,8 @@ speculator allows you to preview pull requests to the matrix.org specification. +It serves two HTTP endpoints: + - /spec/123 which renders the spec as html at pull request 123. + - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. + To run it, you must install the `go` tool, and run: `go run main.go` From ba780a589a9311dce856342d978ad3f45d10f392 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 14:40:08 +0100 Subject: [PATCH 043/223] speculator: Add list-pull-request functionality --- scripts/speculator/main.go | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3775ad863..757c48e78 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -24,9 +24,12 @@ import ( ) type PullRequest struct { - Base Commit - Head Commit - User User + Number int + Base Commit + Head Commit + Title string + User User + HTMLURL string `json:"html_url"` } type Commit struct { @@ -39,7 +42,8 @@ type RequestRepo struct { } type User struct { - Login string + Login string + HTMLURL string `json:"html_url"` } var ( @@ -47,6 +51,8 @@ var ( allowedMembers map[string]bool ) +const pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + func gitClone(url string) (string, error) { dst := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) cmd := exec.Command("git", "clone", url, dst) @@ -68,7 +74,7 @@ func gitCheckout(path, sha string) error { } func lookupPullRequest(prNumber string) (PullRequest, error) { - resp, _ := http.Get("https://api.github.com/repos/matrix-org/matrix-doc/pulls/" + prNumber) + resp, _ := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) defer resp.Body.Close() dec := json.NewDecoder(resp.Body) var pr PullRequest @@ -192,6 +198,32 @@ func serveRstDiff(w http.ResponseWriter, req *http.Request) { w.Write(diff.Bytes()) } +func listPulls(w http.ResponseWriter, req *http.Request) { + resp, err := http.Get(pullsPrefix) + if err != nil { + writeError(w, err) + return + } + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var pulls []PullRequest + if err := dec.Decode(&pulls); err != nil { + writeError(w, err) + return + } + if len(pulls) == 0 { + io.WriteString(w, "No pull requests found") + return + } + s := "
    " + for _, pull := range pulls { + s += fmt.Sprintf(`
  • %d: %s: %s: spec rst diff
    • `, + pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number) + } + s += "
" + io.WriteString(w, s) +} + func ignoreExitCodeOne(err error) error { if err == nil { return err @@ -221,6 +253,7 @@ func main() { http.HandleFunc("/spec/", serveSpec) http.HandleFunc("/diff/rst/", serveRstDiff) http.HandleFunc("/healthz", serveText("ok")) + http.HandleFunc("/", listPulls) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } From 20f0284155ed5d1a5cfe8146071f395559e93866 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 14:40:52 +0100 Subject: [PATCH 044/223] speculator: Unified diff for rst diff --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3775ad863..d30c199c6 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -182,7 +182,7 @@ func serveRstDiff(w http.ResponseWriter, req *http.Request) { return } - diffCmd := exec.Command("diff", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) var diff bytes.Buffer diffCmd.Stdout = &diff if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { From b6dab0009d5533161dff92b1bdad3c664d3a5c6f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 14:43:26 +0100 Subject: [PATCH 045/223] speculator: Don't ignore errors --- scripts/speculator/main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 3775ad863..1a474277d 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -67,13 +67,18 @@ func gitCheckout(path, sha string) error { return nil } -func lookupPullRequest(prNumber string) (PullRequest, error) { - resp, _ := http.Get("https://api.github.com/repos/matrix-org/matrix-doc/pulls/" + prNumber) +func lookupPullRequest(prNumber string) (*PullRequest, error) { + resp, err := http.Get("https://api.github.com/repos/matrix-org/matrix-doc/pulls/" + prNumber) defer resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("error getting pulls: %v", err) + } dec := json.NewDecoder(resp.Body) var pr PullRequest - _ = dec.Decode(&pr) - return pr, nil + if err := dec.Decode(&pr); err != nil { + return nil, fmt.Errorf("error decoding pulls: %v", err) + } + return &pr, nil } func generate(dir string) error { From d251f791fbfbe7af675f82689d51b63a79a361ff Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 17:15:03 +0100 Subject: [PATCH 046/223] speculator: Document / --- scripts/speculator/README | 3 ++- scripts/speculator/main.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/README b/scripts/speculator/README index ae00c55ad..6ab84f68b 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -1,6 +1,7 @@ speculator allows you to preview pull requests to the matrix.org specification. -It serves two HTTP endpoints: +It serves the following HTTP endpoints: + - / lists open pull requests - /spec/123 which renders the spec as html at pull request 123. - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index bcbe55146..32c015ac2 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -1,5 +1,6 @@ // speculator allows you to preview pull requests to the matrix.org specification. -// It serves two HTTP endpoints: +// It serves the following HTTP endpoints: +// - / lists open pull requests // - /spec/123 which renders the spec as html at pull request 123. // - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. // It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement. From b2477614658d6b4267ffcb1f35cfc9adb7f388ad Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 14 Sep 2015 10:33:25 +0100 Subject: [PATCH 047/223] .gitignore speculator --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e2250131b..74b1c7d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ scripts/gen scripts/continuserv/continuserv +scripts/speculator/speculator templating/out *.pyc supporting-docs/_site From 0341c30824ed89b64cb09b5eea74f011eed81495 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 14 Sep 2015 10:39:22 +0100 Subject: [PATCH 048/223] Minor typo and formatting fixes --- drafts/macaroons_caveats.rst | 2 +- specification/10_client_server_api.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index b6920d6cb..2a45fe12d 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -1,7 +1,7 @@ Macaroon Caveats ================ -Macaroons (http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) are issued by Matrix servers as authorization tokens. Macaroons may be restricted, by adding caveats to them. +Macaroons (http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index 6a589f8e6..440f98e4b 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -1035,7 +1035,7 @@ user_id The fully-qualified Matrix ID that has been registered. access_token An access token for the account. This token may expire at some point, and if - so, it MAY come with a refersh_token, described below. + so, it MAY come with a refresh_token, described below. refresh_token (optional) A refresh token may be exchanged for a new access_token as described in `Refreshing access tokens`. @@ -1064,7 +1064,7 @@ refresh_token (optional) There is no specific error message to indicate that a request has failed because an access token has expired; instead, if a client has reason to believe its access token is valid, and it receives an auth error, they should attempt to -refresh for a new token on failure, and re-try the request with the new token. +refresh for a new token on failure, and retry the request with the new token. Changing Password ~~~~~~~~~~~~~~~~~ From d9013cab5fb51d341f96e325b15e207859e4c313 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 14 Sep 2015 11:03:54 +0100 Subject: [PATCH 049/223] speculator: Add HTML diffing I started fiddling with re-implementing the perl script in Go to add some new functionality (and avoid the Perl), but it's not yet usable --- scripts/speculator/README | 1 + scripts/speculator/htmldiff.pl | 564 +++++++++++++++++++++++++++++++++ scripts/speculator/main.go | 130 ++++++-- 3 files changed, 661 insertions(+), 34 deletions(-) create mode 100755 scripts/speculator/htmldiff.pl diff --git a/scripts/speculator/README b/scripts/speculator/README index 6ab84f68b..0a9f53fd2 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -4,6 +4,7 @@ It serves the following HTTP endpoints: - / lists open pull requests - /spec/123 which renders the spec as html at pull request 123. - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. + - /diff/html/123 which gives a diff of the spec's HTML at pull request 123. To run it, you must install the `go` tool, and run: `go run main.go` diff --git a/scripts/speculator/htmldiff.pl b/scripts/speculator/htmldiff.pl new file mode 100755 index 000000000..b16890673 --- /dev/null +++ b/scripts/speculator/htmldiff.pl @@ -0,0 +1,564 @@ +#!/usr/bin/perl +# +# htmldiff - present a diff marked version of two html documents +# +# Copyright (c) 1998-2006 MACS, Inc. +# +# Copyright (c) 2007 SiSco, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# See http://www.themacs.com for more information. +# +# usage: htmldiff [[-c] [-l] [-o] oldversion newversion [output]] +# +# -c - disable metahtml comment processing +# -o - disable outputting of old text +# -l - use navindex to create sequence of diffs +# oldversion - the previous version of the document +# newversion - the newer version of the document +# output - a filename to place the output in. If omitted, the output goes to +# standard output. +# +# if invoked with no options or arguments, operates as a CGI script. It then +# takes the following parameters: +# +# oldfile - the URL of the original file +# newfile - the URL of the new file +# mhtml - a flag to indicate whether it should be aware of MetaHTML comments. +# +# requires GNU diff utility +# also requires the perl modules Getopt::Std +# +# NOTE: The markup created by htmldiff may not validate against the HTML 4.0 +# DTD. This is because the algorithm is realtively simple, and there are +# places in the markup content model where the span element is not allowed. +# Htmldiff is NOT aware of these places. +# +# $Source: /u/sources/public/2009/htmldiff/htmldiff.pl,v $ +# $Revision: 1.1 $ +# +# $Log: htmldiff.pl,v $ +# Revision 1.1 2014/01/06 08:04:51 dom +# added copy of htmldiff perl script since aptest.com repo no longer available +# +# Revision 1.5 2008/03/05 13:23:16 ahby +# Fixed a problem with leading whitespace before markup. +# +# Revision 1.4 2007/12/13 13:09:16 ahby +# Updated copyright and license. +# +# Revision 1.3 2007/12/13 12:53:34 ahby +# Changed use of span to ins and del +# +# Revision 1.2 2002/02/13 16:27:23 ahby +# Changed processing model. +# Improved handling of old text and changed styles. +# +# Revision 1.1 2000/07/12 12:20:04 ahby +# Updated to remove empty spans - this fixes validation problems under +# strict. +# +# Revision 1.11 1999/12/08 19:46:45 ahby +# Fixed validation errors introduced by placing markup where it didn't +# belong. +# +# Revision 1.10 1999/10/18 13:42:58 ahby +# Added -o to the usage message. +# +# Revision 1.9 1999/05/04 12:29:11 ahby +# Added an option to turn off the display of old text. +# +# Revision 1.8 1999/04/09 14:37:27 ahby +# Fixed a perl syntax error. +# +# Revision 1.7 1999/04/09 14:35:49 ahby +# Added reference to MACS homepage. +# +# Revision 1.6 1999/04/09 14:35:09 ahby +# Added comment about validity of generated markup. +# +# Revision 1.5 1999/02/22 22:17:54 ahby +# Changed to use stylesheets. +# Changed to rely upon span. +# Changed to work around content model problems. +# +# Revision 1.4 1999/02/08 02:32:22 ahby +# Added a copyright statement. +# +# Revision 1.3 1999/02/08 02:30:40 ahby +# Added header processing. +# +# Revision 1.2 1998/12/10 17:31:31 ahby +# Fixed to escape less-thans in change blocks and to not permit change +# markup within specific elements (like TITLE). +# +# Revision 1.1 1998/11/26 00:09:22 ahby +# Initial revision +# +# + +use Getopt::Std; + +sub usage { + print STDERR "htmldiff [-c] [-o] oldversion newversion [output]\n"; + exit; +} + +sub url_encode { + my $str = shift; + $str =~ s/([\x00-\x1f\x7F-\xFF])/ + sprintf ('%%%02x', ord ($1))/eg; + return $str; +} + +# markit - diff-mark the streams +# +# markit(file1, file2) +# +# markit relies upon GNUdiff to mark up the text. +# +# The markup is encoded using special control sequences: +# +# a block wrapped in control-a is deleted text +# a block wrapped in control-b is old text +# a block wrapped in control-c is new text +# +# The main processing loop attempts to wrap the text blocks in appropriate +# SPANs based upon the type of text that it is. +# +# When the loop encounters a < in the text, it stops the span. Then it outputs +# the element that is defined, then it restarts the span. + +sub markit { + my $retval = ""; + my($file1) = shift; + my($file2) = shift; +# my $old="deleted text: %c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $old="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'"; + my $new="%c'\012'%c'\003'%c'\012'%>%c'\012'%c'\003'%c'\012'"; + my $unchanged="%="; + my $changed="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + if ($opt_o) { + $old = ""; + $changed = "%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'"; + } +# my $old="%c'\002'deleted text:%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\012'%c'\002'"; +# my $new="%c'\002'%c'\012'%c'\002'%>%c'\002'%c'\002'%c'\012'"; +# my $unchanged="%="; +# my $changed="%c'\002'%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\002'%c'\012'%>%c'\012'%c'\002'%c'\002'%c'\012'"; + + my @span; + $span[0]=""; + $span[1]=""; + $span[2]=""; + $span[3]=""; + $span[4]=""; + + my @diffEnd ; + $diffEnd[1] = ''; + $diffEnd[2] = ''; + $diffEnd[3] = ''; + $diffEnd[4] = ''; + + my $diffcounter = 0; + + open(FILE, qq(diff -d --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 |)) || die("Diff failed: $!"); +# system (qq(diff --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 > /tmp/output)); + + my $state = 0; + my $inblock = 0; + my $temp = ""; + my $lineCount = 0; + +# strategy: +# +# process the output of diff... +# +# a link with control A-D means the start/end of the corresponding ordinal +# state (1-4). Resting state is state 0. +# +# While in a state, accumulate the contents for that state. When exiting the +# state, determine if it is appropriate to emit the contents with markup or +# not (basically, if the accumulated buffer contains only empty lines or lines +# with markup, then we don't want to emit the wrappers. We don't need them. +# +# Note that if there is markup in the "old" block, that markup is silently +# removed. It isn't really that interesting, and it messes up the output +# something fierce. + + while () { + my $anchor = $opt_l ? qq[] : "" ; + my $anchorEnd = $opt_l ? q[] : "" ; + $lineCount ++; + if ($state == 0) { # if we are resting and we find a marker, + # then we must be entering a block + if (m/^([\001-\004])/) { + $state = ord($1); + $_ = ""; + } +# if (m/^\001/) { +# $state = 1; +# s/^/$span[1]/; +# } elsif (m/^\002/) { +# $state = 2; +# s/^/$span[2]/; +# } elsif (m/^\003/) { +# $state = 3; +# s/^/$span[3]/; +# } elsif (m/^\004/) { +# $state = 4; +# s/^/$span[4]/; +# } + } else { + # if we are in "old" state, remove markup + if (($state == 1) || ($state == 2)) { + s/\<.*\>//; # get rid of any old markup + s/\/>/g; + } + # if we found another marker, we must be exiting the state + if (m/^([\001-\004])/) { + if ($temp ne "") { + $_ = $span[$state] . $anchor . $temp . $anchorEnd . $diffEnd[$state] . "\n"; + $temp = ""; + } else { + $_ = "" ; + } + $state = 0; + } elsif (m/^\s*\ +.diff-old-a { + font-size: smaller; + color: red; +} + +.diff-new { background-color: yellow; } +.diff-chg { background-color: lime; } +.diff-new:before, +.diff-new:after + { content: "\2191" } +.diff-chg:before, .diff-chg:after + { content: "\2195" } +.diff-old { text-decoration: line-through; background-color: #FBB; } +.diff-old:before, +.diff-old:after + { content: "\2193" } +:focus { border: thin red solid} + +); + if ($opt_t) { + $styles .= q( + +); + + } + + if ($stripheader) { + open(HEADER, ">$headertmp"); + } + + my $incomment = 0; + my $inhead = 1; + open(FILE, $filename) || die("File $filename cannot be opened: $!"); + while () { + if ($inhead == 1) { + if (m/\<\/head/i) { + print HEADER $styles; + } + if (m/\ +); + } + close HEADER; + } else { + print HEADER; + } + } else { + if ($incomment) { + if (m;-->;) { + $incomment = 0; + s/.*-->//; + } else { + next; + } + } + if (m;;) { + s///; + } + if (m; m.call.candidate --------> - [more candidates events] - User answers call - <------ m.call.answer - [...] - <------ m.call.hangup + [..candidates..] --------> + [Answers call] + <--------------- m.call.answer + [Call is active and ongoing] + <--------------- m.call.hangup Or a rejected call: :: - Caller Callee - m.call.invite -----------> - m.call.candidate --------> - [more candidates events] - User rejects call - <------- m.call.hangup + Caller Callee + m.call.invite ------------> + m.call.candidate ---------> + [..candidates..] ---------> + [Rejects call] + <-------------- m.call.hangup Calls are negotiated according to the WebRTC specification. @@ -137,9 +137,8 @@ better experience for the users if their calls are connected if it is clear that their intention is to set up a call with one another. In Matrix, calls are to rooms rather than users (even if those rooms may only -contain one other user) so we consider calls which are to the same room. - -The rules for dealing with such a situation are as follows: +contain one other user) so we consider calls which are to the same room. The +rules for dealing with such a situation are as follows: - If an invite to a room is received whilst the client is preparing to send an invite to the same room, the client should cancel its outgoing call and diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 4b8127aea..3a8825ae8 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -361,8 +361,8 @@ class MatrixUnits(Units): # add typeof base_defs = { - "core#/definitions/room_event": "Message Event", - "core#/definitions/state_event": "State Event" + "core/room_event.json": "Message Event", + "core/state_event.json": "State Event" } if type(json_schema.get("allOf")) == list: schema["typeof"] = base_defs.get( @@ -399,7 +399,6 @@ class MatrixUnits(Units): "`m.room.message msgtypes`_." ) - # Assign state key info if it has some if schema["typeof"] == "State Event": skey_desc = Units.prop( From af32ec194a0fdad546df6f6e35bbd13d8b476c23 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 16:38:58 +0100 Subject: [PATCH 069/223] Move VoIP events to 22_voip_events 20_events is a huge as it is, this makes it easier to find what section you actually want when editing. --- specification/20_events.rst | 66 -------------------------------- specification/22_voip_events.rst | 65 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 66 deletions(-) create mode 100644 specification/22_voip_events.rst diff --git a/specification/20_events.rst b/specification/20_events.rst index ffc23ec41..ce36b040a 100644 --- a/specification/20_events.rst +++ b/specification/20_events.rst @@ -88,69 +88,3 @@ users, they should include the displayname and avatar URL fields in these events so that clients already have these details to hand, and do not have to perform extra roundtrips to query it. -Voice over IP -------------- -Matrix can also be used to set up VoIP calls. This is part of the core -specification, although is at a relatively early stage. Voice (and video) over -Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like -any other event. This means that clients must only send call events to rooms -with exactly two participants as currently the WebRTC standard is based around -two-party communication. - -{{voip_events}} - -Message Exchange -~~~~~~~~~~~~~~~~ -A call is set up with messages exchanged as follows: - -:: - - Caller Callee - [Place Call] - m.call.invite -----------> - m.call.candidate --------> - [..candidates..] --------> - [Answers call] - <--------------- m.call.answer - [Call is active and ongoing] - <--------------- m.call.hangup - -Or a rejected call: - -:: - - Caller Callee - m.call.invite ------------> - m.call.candidate ---------> - [..candidates..] ---------> - [Rejects call] - <-------------- m.call.hangup - -Calls are negotiated according to the WebRTC specification. - - -Glare -~~~~~ -This specification aims to address the problem of two users calling each other -at roughly the same time and their invites crossing on the wire. It is a far -better experience for the users if their calls are connected if it is clear -that their intention is to set up a call with one another. - -In Matrix, calls are to rooms rather than users (even if those rooms may only -contain one other user) so we consider calls which are to the same room. The -rules for dealing with such a situation are as follows: - - - If an invite to a room is received whilst the client is preparing to send an - invite to the same room, the client should cancel its outgoing call and - instead automatically accept the incoming call on behalf of the user. - - If an invite to a room is received after the client has sent an invite to - the same room and is waiting for a response, the client should perform a - lexicographical comparison of the call IDs of the two calls and use the - lesser of the two calls, aborting the greater. If the incoming call is the - lesser, the client should accept this call on behalf of the user. - -The call setup should appear seamless to the user as if they had simply placed -a call and the other party had accepted. Thusly, any media stream that had been -setup for use on a call should be transferred and used for the call that -replaces it. - diff --git a/specification/22_voip_events.rst b/specification/22_voip_events.rst new file mode 100644 index 000000000..d75db7e73 --- /dev/null +++ b/specification/22_voip_events.rst @@ -0,0 +1,65 @@ +Voice over IP +------------- +Matrix can also be used to set up VoIP calls. This is part of the core +specification, although is at a relatively early stage. Voice (and video) over +Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like +any other event. This means that clients must only send call events to rooms +with exactly two participants as currently the WebRTC standard is based around +two-party communication. + +{{voip_events}} + +Message Exchange +~~~~~~~~~~~~~~~~ +A call is set up with messages exchanged as follows: + +:: + + Caller Callee + [Place Call] + m.call.invite -----------> + m.call.candidate --------> + [..candidates..] --------> + [Answers call] + <--------------- m.call.answer + [Call is active and ongoing] + <--------------- m.call.hangup + +Or a rejected call: + +:: + + Caller Callee + m.call.invite ------------> + m.call.candidate ---------> + [..candidates..] ---------> + [Rejects call] + <-------------- m.call.hangup + +Calls are negotiated according to the WebRTC specification. + + +Glare +~~~~~ +This specification aims to address the problem of two users calling each other +at roughly the same time and their invites crossing on the wire. It is a far +better experience for the users if their calls are connected if it is clear +that their intention is to set up a call with one another. + +In Matrix, calls are to rooms rather than users (even if those rooms may only +contain one other user) so we consider calls which are to the same room. The +rules for dealing with such a situation are as follows: + + - If an invite to a room is received whilst the client is preparing to send an + invite to the same room, the client should cancel its outgoing call and + instead automatically accept the incoming call on behalf of the user. + - If an invite to a room is received after the client has sent an invite to + the same room and is waiting for a response, the client should perform a + lexicographical comparison of the call IDs of the two calls and use the + lesser of the two calls, aborting the greater. If the incoming call is the + lesser, the client should accept this call on behalf of the user. + +The call setup should appear seamless to the user as if they had simply placed +a call and the other party had accepted. Thusly, any media stream that had been +setup for use on a call should be transferred and used for the call that +replaces it. From 46d29e9eea31d32e4865554e0c36badcd4006091 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 17 Sep 2015 16:48:07 +0100 Subject: [PATCH 070/223] fix the room api swagger to be valid swagger --- api/client-server/v1/rooms.yaml | 41 +++++++++++++++------------------ 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 9a73bb059..c038dba54 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -43,9 +43,8 @@ paths: - in: path type: string name: stateKey - description: |- - The key of the state to look up. Defaults to the empty string. - required: false + description: The key of the state to look up. Defaults to the empty string. + required: true x-example: "" responses: 200: @@ -159,17 +158,17 @@ paths: "user_id": "@alice:example.com" } ] - schema: - type: array - title: RoomState - description: |- - If the user is a member of the room this will be the - current state of the room as a list of events. - items: - title: StateEvent - type: object - allOf: - - "$ref": "definitions/state_event.yaml" + schema: + type: array + title: RoomState + description: |- + If the user is a member of the room this will be the + current state of the room as a list of events. + items: + title: StateEvent + type: object + allOf: + - "$ref": "events/core/state_event.json" 403: description: You are not joined to the room. @@ -345,7 +344,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "definitions/room_event.yaml" + - "$ref": "events/core/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -358,7 +357,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "definitions/state_event.yaml" + - "$ref": "events/core/state_event.json" visibility: type: string enum: ["private", "public"] @@ -377,7 +376,7 @@ paths: type: string name: roomId description: The room to get the member events for. - required: true, + required: true x-example: "!room:example.com" responses: 200: @@ -419,15 +418,13 @@ paths: ] } schema: - object: - properities: + type: object + properties: chunk: type: array items: title: MemberEvent type: object allOf: - - "$ref": "../../event-schemas/schema/v1/m.room.member" - visibility: - type: string + - "$ref": "events/m.room.member" From faa95e172f0ec7f1f40416b1df094380a72f2c98 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 17:49:57 +0100 Subject: [PATCH 071/223] More formatting fixes; typos; etc --- specification/22_voip_events.rst | 1 + specification/25_application_service_api.rst | 11 ++-- specification/30_server_server_api.rst | 55 ++++++++++---------- specification/31_event_signing.rst | 2 +- specification/40_content_repository.rst | 8 +-- specification/43_push_cs_api.rst | 20 +++---- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/specification/22_voip_events.rst b/specification/22_voip_events.rst index d75db7e73..a54682379 100644 --- a/specification/22_voip_events.rst +++ b/specification/22_voip_events.rst @@ -63,3 +63,4 @@ The call setup should appear seamless to the user as if they had simply placed a call and the other party had accepted. Thusly, any media stream that had been setup for use on a call should be transferred and used for the call that replaces it. + diff --git a/specification/25_application_service_api.rst b/specification/25_application_service_api.rst index 8236b2dea..9b58c8614 100644 --- a/specification/25_application_service_api.rst +++ b/specification/25_application_service_api.rst @@ -66,13 +66,13 @@ An example HS configuration required to pass traffic to the AS is: application service is merely augmenting the room itself (e.g. providing logging or searching facilities). - Namespaces are represented by POSIX extended regular expressions, - e.g.: + e.g: .. code-block:: yaml users: - exclusive: true - regex: @irc.freenode.net/.* + regex: @irc.freenode.net_.* Home Server -> Application Service API @@ -326,7 +326,7 @@ but only if the application service has defined the namespace as ``exclusive``. ID conventions ~~~~~~~~~~~~~~ -.. NOTE:: +.. TODO-spec - Giving HSes the freedom to namespace still feels like the Right Thing here. - Exposing a public API provides the consistency which was the main complaint against namespacing. @@ -345,7 +345,7 @@ types, including: - MSISDNs (``tel``) - Email addresses (``mailto``) - IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04) -- XMPP (xep-0032) +- XMPP (XEP-0032) - SIP URIs (RFC 3261) As a result, virtual user IDs SHOULD relate to their URI counterpart. This @@ -403,6 +403,8 @@ blog comment traffic in & out of matrix Active Application Services ---------------------------- +.. NOTE:: + This section is a work in progress. .. TODO-spec API that provides hooks into the server so that you can intercept and @@ -419,3 +421,4 @@ Policy Servers Enforcing policies ------------------ + diff --git a/specification/30_server_server_api.rst b/specification/30_server_server_api.rst index 1c7bf3ede..f5cadabff 100644 --- a/specification/30_server_server_api.rst +++ b/specification/30_server_server_api.rst @@ -2,10 +2,9 @@ Federation API ============== Matrix home servers use the Federation APIs (also known as server-server APIs) -to communicate with each other. -Home servers use these APIs to push messages to each other in real-time, to -request historic messages from each other, and to query profile and presence -information about users on each other's servers. +to communicate with each other. Home servers use these APIs to push messages to +each other in real-time, to request historic messages from each other, and to +query profile and presence information about users on each other's servers. The APIs are implemented using HTTPS GETs and PUTs between each of the servers. These HTTPS requests are strongly authenticated using public key @@ -21,7 +20,7 @@ Persisted Data Units (PDUs): context. Like email, it is the responsibility of the originating server of a PDU - to deliver that event to its recepient servers. However PDUs are signed + to deliver that event to its recipient servers. However PDUs are signed using the originating server's public key so that it is possible to deliver them through third-party servers. @@ -84,18 +83,19 @@ directly or by querying an intermediate notary server using a response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. -This approach is borrowed from the Perspectives Project -(http://perspectives-project.org/), but modified to include the NACL keys and to -use JSON instead of XML. It has the advantage of avoiding a single trust-root -since each server is free to pick which notary servers they trust and can -corroborate the keys returned by a given notary server by querying other -servers. +This approach is borrowed from the `Perspectives Project`_, but modified to +include the NACL keys and to use JSON instead of XML. It has the advantage of +avoiding a single trust-root since each server is free to pick which notary +servers they trust and can corroborate the keys returned by a given notary +server by querying other servers. + +.. _Perspectives Project: http://perspectives-project.org/ Publishing Keys _______________ Home servers publish the allowed TLS fingerprints and signing keys in a JSON -object at ``/_matrix/key/v2/server/${key_id}``. The response contains a list of +object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the server and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list @@ -510,7 +510,7 @@ To backfill events on a given context:: Retrieves a sliding-window history of previous PDUs that occurred on the given context. Starting from the PDU ID(s) given in the "v" argument, the PDUs that -preceeded it are retrieved, up to a total number given by the "limit" argument. +preceded it are retrieved, up to a total number given by the "limit" argument. These are then returned in a new Transaction containing all of the PDUs. @@ -554,9 +554,7 @@ Every HTTP request made by a homeserver is authenticated using public key digital signatures. The request method, target and body are signed by wrapping them in a JSON object and signing it using the JSON signing algorithm. The resulting signatures are added as an Authorization header with an auth scheme -of X-Matrix. - -Note that the target field should include the full path starting with +of X-Matrix. Note that the target field should include the full path starting with ``/_matrix/...``, including the ``?`` and any query parameters if present, but should not include the leading ``https:``, nor the destination server's hostname. @@ -656,12 +654,12 @@ State Conflict Resolution - How does this work with deleting current state - How do we reject invalid federation traffic? -[[TODO(paul): At this point we should probably have a long description of how -State management works, with descriptions of clobbering rules, power levels, etc -etc... But some of that detail is rather up-in-the-air, on the whiteboard, and -so on. This part needs refining. And writing in its own document as the details -relate to the server/system as a whole, not specifically to server-server -federation.]] + [[TODO(paul): At this point we should probably have a long description of how + State management works, with descriptions of clobbering rules, power levels, etc + etc... But some of that detail is rather up-in-the-air, on the whiteboard, and + so on. This part needs refining. And writing in its own document as the details + relate to the server/system as a whole, not specifically to server-server + federation.]] Presence -------- @@ -677,8 +675,8 @@ Performing a presence update and poll subscription request:: Each should be an object with the following keys: user_id: string containing a User ID presence: "offline"|"unavailable"|"online"|"free_for_chat" - status_msg: (optional) string of freeform text - last_active_ago: miliseconds since the last activity by the user + status_msg: (optional) string of free-form text + last_active_ago: milliseconds since the last activity by the user poll: (optional): list of strings giving User IDs @@ -696,7 +694,7 @@ removed until explicitly requested by a later ``unpoll``. On receipt of a message containing a non-empty ``poll`` list, the receiving server should immediately send the sending server a presence update EDU of its own, containing in a ``push`` list the current state of every user that was in -the orginal EDU's ``poll`` list. +the original EDU's ``poll`` list. Sending a presence invite:: @@ -721,7 +719,7 @@ Rejecting a presence invite:: Content keys - as for m.presence_invite .. TODO-doc - - Explain the timing-based roundtrip reduction mechanism for presence + - Explain the timing-based round-trip reduction mechanism for presence messages - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence @@ -742,8 +740,8 @@ Querying profile information:: field: (optional) string giving a field name Returns: JSON object containing the following keys: - displayname: string of freeform text - avatar_url: string containing an http-scheme URL + displayname: string of free-form text + avatar_url: string containing an HTTP-scheme URL If the query contains the optional ``field`` key, it should give the name of a result field. If such is present, then the result should contain only a field @@ -769,3 +767,4 @@ Querying directory information:: The list of join candidates is a list of server names that are likely to hold the given room; these are servers that the requesting server may wish to try joining with. This list may or may not include the server answering the query. + diff --git a/specification/31_event_signing.rst b/specification/31_event_signing.rst index 9351cd57d..f3a22d1d8 100644 --- a/specification/31_event_signing.rst +++ b/specification/31_event_signing.rst @@ -30,7 +30,7 @@ using this representation. value, # Encode code-points outside of ASCII as UTF-8 rather than \u escapes ensure_ascii=False, - # Remove unecessary white space. + # Remove unnecessary white space. separators=(',',':'), # Sort the keys of dictionaries. sort_keys=True, diff --git a/specification/40_content_repository.rst b/specification/40_content_repository.rst index df04605a1..2c45ced74 100644 --- a/specification/40_content_repository.rst +++ b/specification/40_content_repository.rst @@ -39,10 +39,10 @@ thumbnailing method:: -The thumbnail methods are "crop" and "scale". "scale" trys to return an +The thumbnail methods are "crop" and "scale". "scale" tries to return an image where either the width or the height is smaller than the requested size. The client should then scale and letterbox the image if it needs to -fit within a given rectangle. "crop" trys to return an image where the +fit within a given rectangle. "crop" tries to return an image where the width and height are close to the requested size and the aspect matches the requested size. The client should scale the image if it needs to fit within a given rectangle. @@ -53,8 +53,8 @@ the content. Homeservers may return thumbnails of a different size to that requested. However homeservers should provide exact matches where reasonable. Homeservers must never upscale images. -Security --------- +Security considerations +----------------------- Clients may try to upload very large files. Homeservers should not store files that are too large and should not serve them to clients. diff --git a/specification/43_push_cs_api.rst b/specification/43_push_cs_api.rst index 022f50c3b..b78b44fde 100644 --- a/specification/43_push_cs_api.rst +++ b/specification/43_push_cs_api.rst @@ -70,7 +70,7 @@ Room Rules Sender These rules configure notification behaviour for messages from a specific, named Matrix user ID. The rule_id of Sender rules is always the Matrix user - ID of the user whose messages theyt apply to. + ID of the user whose messages they'd apply to. Underride These are identical to override rules, but have a lower priority than content, room and sender rules. @@ -112,7 +112,7 @@ In addition, all rules may be enabled or disabled. Disabled rules never match. If no rules match an event, the Home Server should not notify for the message (that is to say, the default action is "dont-notify"). Events that the user sent -themself are never alerted for. +themselves are never alerted for. Predefined Rules ---------------- @@ -128,7 +128,7 @@ with these IDs, their semantics should match those given below: { "rule_id": ".m.rule.contains_user_name" - "pattern": "[the lcoal part of the user's Matrix ID]", + "pattern": "[the local part of the user's Matrix ID]", "actions": [ "notify", { @@ -220,7 +220,7 @@ with these IDs, their semantics should match those given below: Push Rules: Actions: -------------------- All rules have an associated list of 'actions'. An action affects if and how a -notification is delievered for a matching event. This standard defines the +notification is delivered for a matching event. This standard defines the following actions, although if Home servers wish to support more, they are free to do so: @@ -241,11 +241,11 @@ set_tweak Actions that have no parameters are represented as a string. Otherwise, they are represented as a dictionary with a key equal to their name and other keys as -their parameters, eg. { "set_tweak": "sound", "value": "default" } +their parameters, e.g. ``{ "set_tweak": "sound", "value": "default" }`` Push Rules: Actions: Tweaks --------------------------- -The 'set_tweak' key action is used to add an entry to the 'tweaks' dictionary +The ``set_tweak`` key action is used to add an entry to the 'tweaks' dictionary that is sent in the notification poke. The following tweaks are defined: sound @@ -275,7 +275,7 @@ do so: event_match This is a glob pattern match on a field of the event. Parameters: - * 'key': The dot-separated field of the event to match, eg. content.body + * 'key': The dot-separated field of the event to match, e.g. content.body * 'pattern': The glob-style pattern to match against. Patterns with no special glob characters should be treated as having asterisks prepended and appended when testing the condition. @@ -295,7 +295,7 @@ room_member_count '>=' or '<='. A prefix of '<' matches rooms where the member count is strictly less than the given number and so forth. If no prefix is present, this matches rooms where the member count is exactly equal to the given - number (ie. the same as '=='). + number (i.e. the same as '=='). Room, Sender, User and Content rules do not have conditions in the same way, but instead have predefined conditions, the behaviour of which can be configured @@ -314,7 +314,7 @@ scope Either 'global' or 'device/' to specify global rules or device rules for the given profile_tag. kind - The kind of rule, ie. 'override', 'underride', 'sender', 'room', 'content'. + The kind of rule, i.e. 'override', 'underride', 'sender', 'room', 'content'. rule_id The identifier for the rule. @@ -330,7 +330,7 @@ after rule. All requests to the push rules API also require an access_token as a query -paraemter. +parameter. The content of the PUT request is a JSON object with a list of actions under the 'actions' key and either conditions (under the 'conditions' key) or the From 620d3dcb266c432e1bc471ad63b6bed57354e46f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 17:51:44 +0100 Subject: [PATCH 072/223] Test commit --- specification/44_push_push_gw_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/44_push_push_gw_api.rst b/specification/44_push_push_gw_api.rst index b182503b5..d7f16144d 100644 --- a/specification/44_push_push_gw_api.rst +++ b/specification/44_push_push_gw_api.rst @@ -1,7 +1,7 @@ HTTP Notification Protocol -------------------------- -This describes the format used by "http" pushers to send notifications of +This describes the format used by "HTTP" pushers to send notifications of events. Notifications are sent as HTTP POST requests to the URL configured when the From 3f9d183c2aaaf5f6d2f6c9bc39a57317145dca1b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 09:25:40 +0100 Subject: [PATCH 073/223] More typo/layout fixes Split out identity servers to a separate file --- specification/44_push_push_gw_api.rst | 8 ++++---- specification/45_typing_notifications.rst | 11 +++++++---- specification/46_receipts.rst | 18 +++++++----------- specification/49_other_non_core_apis.rst | 4 ++-- specification/50_appendices.rst | 8 -------- specification/60_identity_servers.rst | 8 ++++++++ 6 files changed, 28 insertions(+), 29 deletions(-) create mode 100644 specification/60_identity_servers.rst diff --git a/specification/44_push_push_gw_api.rst b/specification/44_push_push_gw_api.rst index d7f16144d..c970e93ef 100644 --- a/specification/44_push_push_gw_api.rst +++ b/specification/44_push_push_gw_api.rst @@ -77,10 +77,10 @@ counts This is a dictionary of the current number of unacknowledged communications for the recipient user. Counts whose value is zero are omitted. unread - The number of unread messages a user has accross all of the rooms they are a + The number of unread messages a user has across all of the rooms they are a member of. missed_calls - The number of unacknowledged missed calls a user has accross all rooms of + The number of unacknowledged missed calls a user has across all rooms of which they are a member. device This is an array of devices that the notification should be sent to. @@ -104,13 +104,13 @@ And additional key is defined but only present on member events: user_is_target This is true if the user receiving the notification is the subject of a member - event (ie. the state_key of the member event is equal to the user's Matrix + event (i.e. the state_key of the member event is equal to the user's Matrix ID). The recipient of an HTTP notification should respond with an HTTP 2xx response when the notification has been processed. If the endpoint returns an HTTP error code, the Home Server should retry for a reasonable amount of time with a -reasonable backoff scheme. +reasonable back-off scheme. The endpoint should return a JSON dictionary as follows:: diff --git a/specification/45_typing_notifications.rst b/specification/45_typing_notifications.rst index 7df9a238e..89f06b48e 100644 --- a/specification/45_typing_notifications.rst +++ b/specification/45_typing_notifications.rst @@ -5,9 +5,10 @@ Client APIs ----------- To set "I am typing for the next N msec":: + PUT .../rooms//typing/ Content: { "typing": true, "timeout": N } - # timeout is in msec; I suggest no more than 20 or 30 seconds + # timeout is in milliseconds; suggested no more than 20 or 30 seconds This should be re-sent by the client to continue informing the server the user is still typing; I suggest a safety margin of 5 seconds before the expected @@ -15,6 +16,7 @@ timeout runs out. Just keep declaring a new timeout, it will replace the old one. To set "I am no longer typing":: + PUT ../rooms//typing/ Content: { "typing": false } @@ -46,13 +48,14 @@ Servers will emit EDUs in the following form:: "content": { "room_id": "!room-id-here:matrix.org", "user_id": "@user-id-here:matrix.org", - "typing": true/false, + "typing": true/false } } Server EDUs don't (currently) contain timing information; it is up to originating HSes to ensure they eventually send "stop" notifications. -((This will eventually need addressing, as part of the wider typing/presence -timer addition work)) +.. TODO + ((This will eventually need addressing, as part of the wider typing/presence + timer addition work)) diff --git a/specification/46_receipts.rst b/specification/46_receipts.rst index 6428d6b5c..a0a90c16e 100644 --- a/specification/46_receipts.rst +++ b/specification/46_receipts.rst @@ -2,9 +2,8 @@ Receipts ======== Receipts are used to publish which events in a room the user or their devices -have interacted with. For example, which events the user has read. - -For efficiency this is done as "up to" markers, i.e. marking a particular event +have interacted with. For example, which events the user has read. For +efficiency this is done as "up to" markers, i.e. marking a particular event as, say, ``read`` indicates the user has read all events *up to* that event. Client-Server API @@ -43,14 +42,11 @@ For example:: } For efficiency, receipts are batched into one event per room. In the initialSync -and v2 sync APIs the receipts are listed in a seperate top level ``receipts`` -key. - -Each ``user_id``, ``receipt_type`` pair must be associated with only a single -``event_id``. - -New receipts that come down the event streams are deltas. Deltas update -existing mappings, clobbering based on ``user_id``, ``receipt_type`` pairs. +and v2 sync APIs the receipts are listed in a separate top level ``receipts`` +key. Each ``user_id``, ``receipt_type`` pair must be associated with only a +single ``event_id``. New receipts that come down the event streams are deltas. +Deltas update existing mappings, clobbering based on ``user_id``, +``receipt_type`` pairs. A client can update the markers for its user by issuing a request:: diff --git a/specification/49_other_non_core_apis.rst b/specification/49_other_non_core_apis.rst index 8855ce9e7..d6315a965 100644 --- a/specification/49_other_non_core_apis.rst +++ b/specification/49_other_non_core_apis.rst @@ -3,10 +3,10 @@ Address book repository .. NOTE:: This section is a work in progress. - Do we even need it? Clients can use out-of-band addressbook servers for now; - this should definitely not be core. .. TODO-spec + Do we even need it? Clients can use out-of-band addressbook servers for now; + this should definitely not be core. - format: POST(?) wodges of json, some possible processing, then return wodges of json on GET. - processing may remove dupes, merge contacts, pepper with extra info (e.g. matrix-ability of contacts), etc. diff --git a/specification/50_appendices.rst b/specification/50_appendices.rst index 295c8f695..de1ac2905 100644 --- a/specification/50_appendices.rst +++ b/specification/50_appendices.rst @@ -128,11 +128,3 @@ Threat: Disclosure to Servers Within Chatroom An attacker could take control of a server within a chatroom to expose message contents or metadata for messages in that room. - -Identity Servers -================ -.. NOTE:: - This section is a work in progress. - -.. TODO-doc Dave - - 3PIDs and identity server, functions diff --git a/specification/60_identity_servers.rst b/specification/60_identity_servers.rst new file mode 100644 index 000000000..6ec013bd7 --- /dev/null +++ b/specification/60_identity_servers.rst @@ -0,0 +1,8 @@ +Identity Servers +================ +.. NOTE:: + This section is a work in progress. + +.. TODO-doc Dave + - 3PIDs and identity server, functions + From 8e7b33ac9989c985ceac1cf7fccddd84c1346ec8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 09:40:53 +0100 Subject: [PATCH 074/223] Fix more typos/spelling errors --- specification/00_basis.rst | 13 ++++++----- specification/10_client_server_api.rst | 32 +++++++++++++------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/specification/00_basis.rst b/specification/00_basis.rst index f6d50802f..384d583b9 100644 --- a/specification/00_basis.rst +++ b/specification/00_basis.rst @@ -79,7 +79,7 @@ The functionality that Matrix provides includes: - Extensible user management (inviting, joining, leaving, kicking, banning) mediated by a power-level based user privilege system. - Extensible room state management (room naming, aliasing, topics, bans) -- Extensible user profile management (avatars, displaynames, etc) +- Extensible user profile management (avatars, display names, etc) - Managing user accounts (registration, login, logout) - Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, Facebook accounts to authenticate, identify and discover users on Matrix. @@ -91,7 +91,7 @@ The functionality that Matrix provides includes: The end goal of Matrix is to be a ubiquitous messaging layer for synchronising arbitrary data between sets of people, devices and services - be that for instant messages, VoIP call setups, or any other objects that need to be -reliably and persistently pushed from A to B in an interoperable and federated +reliably and persistently pushed from A to B in an inter-operable and federated manner. Overview @@ -185,7 +185,7 @@ Event Graphs Events exchanged in the context of a room are stored in a directed acyclic graph (DAG) called an ``event graph``. The partial ordering of this graph gives the chronological ordering of events within the room. Each event in the graph has a -list of zero or more ``parent`` events, which refer to any preceeding events +list of zero or more ``parent`` events, which refer to any preceding events which have no chronological successor from the perspective of the homeserver which created the event. @@ -368,7 +368,8 @@ room). An example of a non-proactive client activity would be a client setting key called ``last_active_ago``, which gives the relative number of milliseconds since the message is generated/emitted that the user was last seen active. -N.B. in v1 API, status/online/idle state are muxed into a single 'presence' field on the m.presence event. +N.B. in v1 API, status/online/idle state are muxed into a single 'presence' +field on the ``m.presence`` event. Presence Lists ~~~~~~~~~~~~~~ @@ -386,7 +387,7 @@ Profiles Users may publish arbitrary key/value data associated with their account - such as a human readable ``display name``, a profile photo URL, contact information -(email address, phone nubers, website URLs etc). +(email address, phone numbers, website URLs etc). In Client-Server API v2, profile data is typed using namespaced keys for interoperability, much like events - e.g. ``m.profile.display_name``. @@ -443,7 +444,7 @@ The ``error`` string will be a human-readable error message, usually a sentence explaining what went wrong. The ``errcode`` string will be a unique string which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error codes should have their namespace first in ALL CAPS, followed by a single _ to -ease seperating the namespace from the error code.. For example, if there was a +ease separating the namespace from the error code.. For example, if there was a custom namespace ``com.mydomain.here``, and a ``FORBIDDEN`` code, the error code should look like ``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the diff --git a/specification/10_client_server_api.rst b/specification/10_client_server_api.rst index f0416c02a..ef77a7dc9 100644 --- a/specification/10_client_server_api.rst +++ b/specification/10_client_server_api.rst @@ -8,11 +8,11 @@ The client-server API provides a simple lightweight API to let clients send messages, control rooms and synchronise conversation history. It is designed to support both lightweight clients which store no state and lazy-load data from the server as required - as well as heavyweight clients which maintain a full -local peristent copy of server state. +local persistent copy of server state. This mostly describes v1 of the Client-Server API as featured in the original September 2014 launch of Matrix, apart from user-interactive authentication where it is -encouraged to move to V2, therefore this is the version documented here. +encouraged to move to v2, therefore this is the version documented here. Version 2 is currently in development (as of Jan-March 2015) as an incremental but backwards-incompatible refinement of Version 1 and will be released shortly. @@ -154,7 +154,7 @@ Matrix client, for example, an email confirmation may be completed when the user clicks on the link in the email. In this case, the client retries the request with an auth dict containing only the session key. The response to this will be the same as if the client were attempting to complete an auth state normally, -ie. the request will either complete or request auth, with the presence or +i.e. the request will either complete or request auth, with the presence or absence of that login stage type in the 'completed' array indicating whether that stage is complete. @@ -204,7 +204,7 @@ Password-based :Type: ``m.login.password`` :Description: - The client submits a username and secret password, both sent in plaintext. + The client submits a username and secret password, both sent in plain-text. To respond to this type, reply with an auth dict as follows:: @@ -247,10 +247,10 @@ service which the home server accepts when logging in, this indirection can be skipped and the "uri" key can be the ``Authorization Request URI``. The client then visits the ``Authorization Request URI``, which then shows the -OAuth2 Allow/Deny prompt. Hitting 'Allow' returns the [XXX: redirects to the?]``redirect URI`` with the -auth code. Home servers can choose any path for the ``redirect URI``. Once the -OAuth flow has completed, the client retries the request with the session only, -as above. +OAuth2 Allow/Deny prompt. Hitting 'Allow' redirects to the ``redirect URI`` with +the auth code. Home servers can choose any path for the ``redirect URI``. Once +the OAuth flow has completed, the client retries the request with the session +only, as above. Email-based (identity server) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -308,7 +308,7 @@ Where ``stage type`` is the type name of the stage it is attempting and ``session id`` is the ID of the session given by the home server. This MUST return an HTML page which can perform this authentication stage. This -page must attempt to call the Javascript function ``window.onAuthDone`` when +page must attempt to call the JavaScript function ``window.onAuthDone`` when the authentication has been completed. Pagination @@ -373,7 +373,7 @@ now show page 3 (rooms R11 -> 15):: Returns: R11,R12,R13,R14,R15 Note that tokens are treated in an *exclusive*, not inclusive, manner. The end -token from the intial request was '9' which corresponded to R10. When the 2nd +token from the initial request was '9' which corresponded to R10. When the 2nd request was made, R10 did not appear again, even though from=9 was specified. If you know the token, you already have the data. @@ -425,9 +425,9 @@ You can visualise the range of events being returned as:: | | start: '1-2-3' end: 'a-b-c' -Now, to receive future events in realtime on the eventstream, you simply GET +Now, to receive future events in real-time on the eventstream, you simply GET $PREFIX/events with a ``from`` parameter of 'a-b-c': in other words passing in the -``end`` token returned by initialsync. The request blocks until new events are +``end`` token returned by initial sync. The request blocks until new events are available or until your specified timeout elapses, and then returns a new paginatable chunk of events alongside new start and end parameters:: @@ -467,7 +467,7 @@ event stream. When the request returns, an ``end`` token is included in the response. This token can be used in the next request to continue where the last request left off. -All events must be deduplicated based on their event ID. +All events must be de-duplicated based on their event ID. .. TODO is deduplication actually a hard requirement in CS v2? @@ -493,7 +493,7 @@ Room events are split into two categories: :Message events: These are events which describe transient "once-off" activity in a room: - typically communication such as sending an instant messaage or setting up a + typically communication such as sending an instant message or setting up a VoIP call. These used to be called 'non-state' events. This specification outlines several events, all with the event type prefix @@ -971,7 +971,7 @@ Registering for a user account is done using the request:: POST $V2PREFIX/register This API endpoint uses the User-Interactive Authentication API. -This API endoint does not require an access token. +This API endpoint does not require an access token. The body of the POST request is a JSON object containing: @@ -1059,7 +1059,7 @@ The third party identifier credentials object comprises: id_server The colon-separated hostname and port of the Identity Server used to - authenticate the third party identifer. If the port is the default, it and the + authenticate the third party identifier. If the port is the default, it and the colon should be omitted. sid The session ID given by the Identity Server From ad26b7f8cb89b2a32e009bdd1d061ea6b7887b9d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 10:03:58 +0100 Subject: [PATCH 075/223] Fix a subtle rendering bug when catting spec sections Throw in gendoc.py if a spec section doesn't end with \n\n There needs to be TWO new lines at the end of each spec section else the title of the next section merges into the last paragraph of the earlier section. This happens without rst2html producing a warning, and results in the section heading of a file disappearing(!) --- scripts/gendoc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed1727261..e871055e7 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -18,7 +18,14 @@ def glob_spec_to(out_file_name): with open(out_file_name, "wb") as outfile: for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: - outfile.write(infile.read()) + section = infile.read() + # we need TWO new lines else the next file's title gets merged + # the last paragraph *WITHOUT RST PRODUCING A WARNING* + if not section[-2:] == '\n\n': + raise Exception( + "The file " + f + " does not end with 2 new lines." + ) + outfile.write(section) def rst2html(i, o): From 08defafd98727b0011dddd6c0e8e271f61d147b2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 18 Sep 2015 10:26:13 +0100 Subject: [PATCH 076/223] Print errors to stderr This means that continuserv will actually display them --- scripts/gendoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index ed1727261..0e9626af4 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -49,7 +49,7 @@ def run_through_template(input): ) except subprocess.CalledProcessError as e: with open(tmpfile, 'r') as f: - print f.read() + sys.stderr.write(f.read() + "\n") raise def prepare_env(): From ef473b4161d029b35ef193309c9b4ad23f43e4f1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:05:48 +0100 Subject: [PATCH 077/223] Start enforcing title styles This depends on the number of the file. --- scripts/gendoc.py | 55 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index e871055e7..6bf7c1a62 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -4,6 +4,7 @@ from docutils.core import publish_file import fileinput import glob import os +import re import shutil import subprocess import sys @@ -14,17 +15,55 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css"] } -def glob_spec_to(out_file_name): +title_style_matchers = { + "=": re.compile("^=+$"), + "-": re.compile("^-+$") +} +TOP_LEVEL = "=" +SECOND_LEVEL = "-" + + +def check_valid_section(filename, section): + # we need TWO new lines else the next file's title gets merged + # the last paragraph *WITHOUT RST PRODUCING A WARNING* + if not section[-2:] == '\n\n': + raise Exception( + "The file " + filename + " does not end with 2 new lines." + ) + + # Enforce some rules to reduce the risk of having mismatched title + # styles. + title_line = section.split("\n")[1] + if title_line != (len(title_line) * title_line[0]): + raise Exception( + "The file " + filename + " doesn't have a title style line on line 2" + ) + + # anything marked as x0_ is the start of a new top-level section + if re.match("^[0-9]+0_", filename): + if not title_style_matchers[TOP_LEVEL].match(title_line): + raise Exception( + "The file " + filename + " is a top-level section because it matches " + + "the filename format x0_something.rst but has the wrong title " + + "style: expected '" + TOP_LEVEL + "' but got '" + + title_line[0] + "'" + ) + # anything marked as xx_ is the start of a sub-section + elif re.match("^[0-9]+_", filename): + if not title_style_matchers[SECOND_LEVEL].match(title_line): + raise Exception( + "The file " + filename + " is a 2nd-level section because it matches " + + "the filename format xx_something.rst but has the wrong title " + + "style: expected '" + SECOND_LEVEL + "' but got '" + + title_line[0] + "'" + ) + +def cat_spec_sections_to(out_file_name): with open(out_file_name, "wb") as outfile: for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: section = infile.read() - # we need TWO new lines else the next file's title gets merged - # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == '\n\n': - raise Exception( - "The file " + f + " does not end with 2 new lines." - ) + check_valid_section(f.split("/")[-1], section) outfile.write(section) @@ -74,7 +113,7 @@ def cleanup_env(): def main(): prepare_env() - glob_spec_to("tmp/full_spec.rst") + cat_spec_sections_to("tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") From 9ed39665c8f70d9c704738d07a39877267e0a4da Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:09:04 +0100 Subject: [PATCH 078/223] PR feedback --- specification/00_basis.rst | 2 +- specification/45_typing_notifications.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/00_basis.rst b/specification/00_basis.rst index 384d583b9..a5196e990 100644 --- a/specification/00_basis.rst +++ b/specification/00_basis.rst @@ -444,7 +444,7 @@ The ``error`` string will be a human-readable error message, usually a sentence explaining what went wrong. The ``errcode`` string will be a unique string which can be used to handle an error message e.g. ``M_FORBIDDEN``. These error codes should have their namespace first in ALL CAPS, followed by a single _ to -ease separating the namespace from the error code.. For example, if there was a +ease separating the namespace from the error code. For example, if there was a custom namespace ``com.mydomain.here``, and a ``FORBIDDEN`` code, the error code should look like ``COM.MYDOMAIN.HERE_FORBIDDEN``. There may be additional keys depending on the diff --git a/specification/45_typing_notifications.rst b/specification/45_typing_notifications.rst index 89f06b48e..ff00165d6 100644 --- a/specification/45_typing_notifications.rst +++ b/specification/45_typing_notifications.rst @@ -11,9 +11,9 @@ To set "I am typing for the next N msec":: # timeout is in milliseconds; suggested no more than 20 or 30 seconds This should be re-sent by the client to continue informing the server the user -is still typing; I suggest a safety margin of 5 seconds before the expected -timeout runs out. Just keep declaring a new timeout, it will replace the old -one. +is still typing; a safety margin of 5 seconds before the expected +timeout runs out is recommended. Just keep declaring a new timeout, it will +replace the old one. To set "I am no longer typing":: From 06e46c1899755d9ae46b66a8b316b4c41b664328 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:20:34 +0100 Subject: [PATCH 079/223] Fix on a better numbering format --- scripts/gendoc.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 6bf7c1a62..1348679a7 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -21,9 +21,16 @@ title_style_matchers = { } TOP_LEVEL = "=" SECOND_LEVEL = "-" +FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}_.*\.rst$") def check_valid_section(filename, section): + if not re.match(FILE_FORMAT_MATCHER, filename): + raise Exception( + "The filename of " + filename +" does not match the expected format " + + "of '##_##_words-go-here.rst'" + ) + # we need TWO new lines else the next file's title gets merged # the last paragraph *WITHOUT RST PRODUCING A WARNING* if not section[-2:] == '\n\n': @@ -39,21 +46,21 @@ def check_valid_section(filename, section): "The file " + filename + " doesn't have a title style line on line 2" ) - # anything marked as x0_ is the start of a new top-level section - if re.match("^[0-9]+0_", filename): + # anything marked as xx_00_ is the start of a new top-level section + if re.match("^[0-9]+_00_", filename): if not title_style_matchers[TOP_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a top-level section because it matches " + - "the filename format x0_something.rst but has the wrong title " + + "the filename format ##_00_something.rst but has the wrong title " + "style: expected '" + TOP_LEVEL + "' but got '" + title_line[0] + "'" ) - # anything marked as xx_ is the start of a sub-section - elif re.match("^[0-9]+_", filename): + # anything marked as xx_x0_ is the start of a sub-section + elif re.match("^[0-9]+_0[0-9]{1}_", filename): if not title_style_matchers[SECOND_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format xx_something.rst but has the wrong title " + + "the filename format ##_#0_something.rst but has the wrong title " + "style: expected '" + SECOND_LEVEL + "' but got '" + title_line[0] + "'" ) From 7f81501762cb700615de837b7501c6bde9deda11 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 11:39:44 +0100 Subject: [PATCH 080/223] Allow relative references to schema to work in python and node. Rename "schema/v1/core" to "schema/v1/core-event-schema". Add self-referential symlinks to schema/v1/core-event-schema The python json schema libraries expect that relative references are relative to the file they are in. The node json schema libraries expect that relateive references are relative to the first file loaded. To support both kinds we reference the core event schema using "core-event-schema/event.json". We then symlink the core-event-schema directory to both the location of the file refering to "event.json" so that it will work in python and to the location of the top level file so that it will work in node. --- api/client-server/v1/core | 1 - api/client-server/v1/core-event-schema | 1 + api/client-server/v1/presence.yaml | 2 +- api/client-server/v1/sync.yaml | 10 +++++----- api/client-server/v1/{events => v1-event-schema} | 0 .../schema/v1/core-event-schema/core-event-schema | 1 + .../schema/v1/{core => core-event-schema}/event.json | 0 .../msgtype_infos/image_info.json | 0 .../v1/{core => core-event-schema}/room_event.json | 2 +- .../v1/{core => core-event-schema}/state_event.json | 2 +- event-schemas/schema/v1/m.call.answer | 2 +- event-schemas/schema/v1/m.call.candidates | 2 +- event-schemas/schema/v1/m.call.hangup | 2 +- event-schemas/schema/v1/m.call.invite | 2 +- event-schemas/schema/v1/m.room.aliases | 2 +- event-schemas/schema/v1/m.room.canonical_alias | 2 +- event-schemas/schema/v1/m.room.create | 2 +- event-schemas/schema/v1/m.room.history_visibility | 2 +- event-schemas/schema/v1/m.room.join_rules | 2 +- event-schemas/schema/v1/m.room.member | 2 +- event-schemas/schema/v1/m.room.message | 2 +- event-schemas/schema/v1/m.room.message#m.audio | 2 +- event-schemas/schema/v1/m.room.message#m.emote | 2 +- event-schemas/schema/v1/m.room.message#m.file | 4 ++-- event-schemas/schema/v1/m.room.message#m.image | 4 ++-- event-schemas/schema/v1/m.room.message#m.location | 4 ++-- event-schemas/schema/v1/m.room.message#m.notice | 2 +- event-schemas/schema/v1/m.room.message#m.text | 2 +- event-schemas/schema/v1/m.room.message#m.video | 4 ++-- event-schemas/schema/v1/m.room.message.feedback | 2 +- event-schemas/schema/v1/m.room.name | 2 +- event-schemas/schema/v1/m.room.power_levels | 2 +- event-schemas/schema/v1/m.room.redaction | 2 +- event-schemas/schema/v1/m.room.topic | 2 +- event-schemas/schema/v1/v1-event-schema | 1 + templating/matrix_templates/units.py | 2 +- 36 files changed, 40 insertions(+), 38 deletions(-) delete mode 120000 api/client-server/v1/core create mode 120000 api/client-server/v1/core-event-schema rename api/client-server/v1/{events => v1-event-schema} (100%) create mode 120000 event-schemas/schema/v1/core-event-schema/core-event-schema rename event-schemas/schema/v1/{core => core-event-schema}/event.json (100%) rename event-schemas/schema/v1/{core => core-event-schema}/msgtype_infos/image_info.json (100%) rename event-schemas/schema/v1/{core => core-event-schema}/room_event.json (93%) rename event-schemas/schema/v1/{core => core-event-schema}/state_event.json (93%) create mode 120000 event-schemas/schema/v1/v1-event-schema diff --git a/api/client-server/v1/core b/api/client-server/v1/core deleted file mode 120000 index 9fc0cd873..000000000 --- a/api/client-server/v1/core +++ /dev/null @@ -1 +0,0 @@ -events/core \ No newline at end of file diff --git a/api/client-server/v1/core-event-schema b/api/client-server/v1/core-event-schema new file mode 120000 index 000000000..045aecb02 --- /dev/null +++ b/api/client-server/v1/core-event-schema @@ -0,0 +1 @@ +v1-event-schema/core-event-schema \ No newline at end of file diff --git a/api/client-server/v1/presence.yaml b/api/client-server/v1/presence.yaml index 472a2d7f1..717e9c473 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/v1/presence.yaml @@ -205,4 +205,4 @@ paths: type: object title: PresenceEvent allOf: - - "$ref": "events/core/event.json" + - "$ref": "core-event-schema/event.json" diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index b95254911..27c7073c4 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -82,7 +82,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "events/core/room_event.json" + - "$ref": "core-event-schema/room_event.json" 400: description: "Bad pagination ``from`` parameter." "/initialSync": @@ -253,7 +253,7 @@ paths: type: object title: Event allOf: - - "$ref": "events/core/event.json" + - "$ref": "core-event-schema/event.json" rooms: type: array items: @@ -294,7 +294,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "events/core/room_event.json" + - "$ref": "core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -307,7 +307,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "events/core/state_event.json" + - "$ref": "core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -350,6 +350,6 @@ paths: } schema: allOf: - - "$ref": "events/core/event.json" + - "$ref": "core-event-schema/event.json" 404: description: The event was not found or you do not have permission to read this event. diff --git a/api/client-server/v1/events b/api/client-server/v1/v1-event-schema similarity index 100% rename from api/client-server/v1/events rename to api/client-server/v1/v1-event-schema diff --git a/event-schemas/schema/v1/core-event-schema/core-event-schema b/event-schemas/schema/v1/core-event-schema/core-event-schema new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/event-schemas/schema/v1/core-event-schema/core-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/event-schemas/schema/v1/core/event.json b/event-schemas/schema/v1/core-event-schema/event.json similarity index 100% rename from event-schemas/schema/v1/core/event.json rename to event-schemas/schema/v1/core-event-schema/event.json diff --git a/event-schemas/schema/v1/core/msgtype_infos/image_info.json b/event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json similarity index 100% rename from event-schemas/schema/v1/core/msgtype_infos/image_info.json rename to event-schemas/schema/v1/core-event-schema/msgtype_infos/image_info.json diff --git a/event-schemas/schema/v1/core/room_event.json b/event-schemas/schema/v1/core-event-schema/room_event.json similarity index 93% rename from event-schemas/schema/v1/core/room_event.json rename to event-schemas/schema/v1/core-event-schema/room_event.json index 7ac66b309..d5413f8a0 100644 --- a/event-schemas/schema/v1/core/room_event.json +++ b/event-schemas/schema/v1/core-event-schema/room_event.json @@ -3,7 +3,7 @@ "title": "Room Event", "description": "In addition to the Event fields, Room Events MUST have the following additional field.", "allOf":[{ - "$ref": "core/event.json" + "$ref": "core-event-schema/event.json" }], "properties": { "room_id": { diff --git a/event-schemas/schema/v1/core/state_event.json b/event-schemas/schema/v1/core-event-schema/state_event.json similarity index 93% rename from event-schemas/schema/v1/core/state_event.json rename to event-schemas/schema/v1/core-event-schema/state_event.json index 60de94130..f70118f49 100644 --- a/event-schemas/schema/v1/core/state_event.json +++ b/event-schemas/schema/v1/core-event-schema/state_event.json @@ -3,7 +3,7 @@ "title": "State Event", "description": "In addition to the Room Event fields, State Events have the following additional fields.", "allOf":[{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "state_key": { diff --git a/event-schemas/schema/v1/m.call.answer b/event-schemas/schema/v1/m.call.answer index abdf4a9b3..f8980a8f6 100644 --- a/event-schemas/schema/v1/m.call.answer +++ b/event-schemas/schema/v1/m.call.answer @@ -3,7 +3,7 @@ "type": "object", "description": "This event is sent by the callee when they wish to answer the call.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.candidates b/event-schemas/schema/v1/m.call.candidates index 052ead0b9..ad84fe86f 100644 --- a/event-schemas/schema/v1/m.call.candidates +++ b/event-schemas/schema/v1/m.call.candidates @@ -3,7 +3,7 @@ "type": "object", "description": "This event is sent by callers after sending an invite and by the callee after answering. Its purpose is to give the other party additional ICE candidates to try using to communicate.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.hangup b/event-schemas/schema/v1/m.call.hangup index 383952d8d..ded16a0d8 100644 --- a/event-schemas/schema/v1/m.call.hangup +++ b/event-schemas/schema/v1/m.call.hangup @@ -3,7 +3,7 @@ "type": "object", "description": "Sent by either party to signal their termination of the call. This can be sent either once the call has has been established or before to abort the call.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.call.invite b/event-schemas/schema/v1/m.call.invite index a852ff437..bfeb3b7d6 100644 --- a/event-schemas/schema/v1/m.call.invite +++ b/event-schemas/schema/v1/m.call.invite @@ -3,7 +3,7 @@ "type": "object", "description": "This event is sent by the caller when they wish to establish a call.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.aliases b/event-schemas/schema/v1/m.room.aliases index 79fd259b2..d1ad0fc76 100644 --- a/event-schemas/schema/v1/m.room.aliases +++ b/event-schemas/schema/v1/m.room.aliases @@ -4,7 +4,7 @@ "title": "Informs the room about what room aliases it has been given.", "description": "This event is sent by a homeserver directly to inform of changes to the list of aliases it knows about for that room. The ``state_key`` for this event is set to the homeserver which owns the room alias. The entire set of known aliases for the room is the union of all the ``m.room.aliases`` events, one for each homeserver. Clients **should** check the validity of any room alias given in this list before presenting it to the user as trusted fact. The lists given by this event should be considered simply as advice on which aliases might exist, for which the client can perform the lookup to confirm whether it receives the correct room ID.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.canonical_alias b/event-schemas/schema/v1/m.room.canonical_alias index 17caa3210..6d56e27eb 100644 --- a/event-schemas/schema/v1/m.room.canonical_alias +++ b/event-schemas/schema/v1/m.room.canonical_alias @@ -4,7 +4,7 @@ "title": "Informs the room as to which alias is the canonical one.", "description": "This event is used to inform the room about which alias should be considered the canonical one. This could be for display purposes or as suggestion to users which alias to use to advertise the room.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index ecc2ed632..bd9231f44 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -4,7 +4,7 @@ "title": "The first event in the room.", "description": "This is the first event in a room and cannot be changed. It acts as the root of all other events.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.history_visibility b/event-schemas/schema/v1/m.room.history_visibility index 48b0a5c45..a35fe7796 100644 --- a/event-schemas/schema/v1/m.room.history_visibility +++ b/event-schemas/schema/v1/m.room.history_visibility @@ -4,7 +4,7 @@ "title": "Controls visibility of history.", "description": "This event controls whether a member of a room can see the events that happened in a room from before they joined.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.join_rules b/event-schemas/schema/v1/m.room.join_rules index 567247eb2..8c70dcafe 100644 --- a/event-schemas/schema/v1/m.room.join_rules +++ b/event-schemas/schema/v1/m.room.join_rules @@ -4,7 +4,7 @@ "title": "Describes how users are allowed to join the room.", "description": "A room may be ``public`` meaning anyone can join the room without any prior action. Alternatively, it can be ``invite`` meaning that a user who wishes to join the room must first receive an invite to the room from someone already inside of the room. Currently, ``knock`` and ``private`` are reserved keywords which are not implemented.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 67e10d51e..a8a442bc6 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -4,7 +4,7 @@ "title": "The current membership state of a user in the room.", "description": "Adjusts the membership state for a user in a room. It is preferable to use the membership APIs (``/rooms//invite`` etc) when performing membership actions rather than adjusting the state directly as there are a restricted set of valid transformations. For example, user A cannot force user B to join a room, and trying to force this state change directly will fail.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message b/event-schemas/schema/v1/m.room.message index 61b6256f7..59edb6764 100644 --- a/event-schemas/schema/v1/m.room.message +++ b/event-schemas/schema/v1/m.room.message @@ -4,7 +4,7 @@ "title": "Message", "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.audio b/event-schemas/schema/v1/m.room.message#m.audio index b236e5b0c..41587c1d0 100644 --- a/event-schemas/schema/v1/m.room.message#m.audio +++ b/event-schemas/schema/v1/m.room.message#m.audio @@ -4,7 +4,7 @@ "title": "AudioMessage", "description": "This message represents a single audio clip.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.emote b/event-schemas/schema/v1/m.room.message#m.emote index 9f17180c9..e6bf5e680 100644 --- a/event-schemas/schema/v1/m.room.message#m.emote +++ b/event-schemas/schema/v1/m.room.message#m.emote @@ -4,7 +4,7 @@ "title": "EmoteMessage", "description": "This message is similar to ``m.text`` except that the sender is 'performing' the action contained in the ``body`` key, similar to ``/me`` in IRC. This message should be prefixed by the name of the sender. This message could also be represented in a different colour to distinguish it from regular ``m.text`` messages.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.file b/event-schemas/schema/v1/m.room.message#m.file index d8c63b338..ed9966942 100644 --- a/event-schemas/schema/v1/m.room.message#m.file +++ b/event-schemas/schema/v1/m.room.message#m.file @@ -4,7 +4,7 @@ "title": "FileMessage", "description": "This message represents a generic file.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -50,7 +50,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core/msgtype_infos/image_info.json" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.image b/event-schemas/schema/v1/m.room.message#m.image index ca22654fb..8baee04e4 100644 --- a/event-schemas/schema/v1/m.room.message#m.image +++ b/event-schemas/schema/v1/m.room.message#m.image @@ -4,7 +4,7 @@ "title": "ImageMessage", "description": "This message represents a single image and an optional thumbnail.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -31,7 +31,7 @@ "title": "ImageInfo", "description": "Metadata about the image referred to in ``thumbnail_url``.", "allOf": [{ - "$ref": "core/msgtype_infos/image_info.json" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] }, "info": { diff --git a/event-schemas/schema/v1/m.room.message#m.location b/event-schemas/schema/v1/m.room.message#m.location index 919c33256..9e59a0a91 100644 --- a/event-schemas/schema/v1/m.room.message#m.location +++ b/event-schemas/schema/v1/m.room.message#m.location @@ -4,7 +4,7 @@ "title": "LocationMessage", "description": "This message represents a real-world location.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -30,7 +30,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core/msgtype_infos/image_info.json" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } }, diff --git a/event-schemas/schema/v1/m.room.message#m.notice b/event-schemas/schema/v1/m.room.message#m.notice index e6ce6f894..5fd217f80 100644 --- a/event-schemas/schema/v1/m.room.message#m.notice +++ b/event-schemas/schema/v1/m.room.message#m.notice @@ -4,7 +4,7 @@ "title": "NoticeMessage", "description": "A m.notice message should be considered similar to a plain m.text message except that clients should visually distinguish it in some way. It is intended to be used by automated clients, such as bots, bridges, and other entities, rather than humans. Additionally, such automated agents which watch a room for messages and respond to them ought to ignore m.notice messages. This helps to prevent infinite-loop situations where two automated clients continuously exchange messages, as each responds to the other.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.text b/event-schemas/schema/v1/m.room.message#m.text index ceadbcd0f..e4b5ab117 100644 --- a/event-schemas/schema/v1/m.room.message#m.text +++ b/event-schemas/schema/v1/m.room.message#m.text @@ -4,7 +4,7 @@ "title": "TextMessage", "description": "This message is the most basic message and is used to represent text.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.message#m.video b/event-schemas/schema/v1/m.room.message#m.video index 667832ffb..2de0f26a7 100644 --- a/event-schemas/schema/v1/m.room.message#m.video +++ b/event-schemas/schema/v1/m.room.message#m.video @@ -4,7 +4,7 @@ "title": "VideoMessage", "description": "This message represents a single video clip.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { @@ -55,7 +55,7 @@ "type": "object", "title": "ImageInfo", "allOf": [{ - "$ref": "core/msgtype_infos/image_info.json" + "$ref": "core-event-schema/msgtype_infos/image_info.json" }] } } diff --git a/event-schemas/schema/v1/m.room.message.feedback b/event-schemas/schema/v1/m.room.message.feedback index b662e9e6f..2dc63b8a0 100644 --- a/event-schemas/schema/v1/m.room.message.feedback +++ b/event-schemas/schema/v1/m.room.message.feedback @@ -4,7 +4,7 @@ "title": "MessageFeedback", "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.name b/event-schemas/schema/v1/m.room.name index 5565147c0..0e212a234 100644 --- a/event-schemas/schema/v1/m.room.name +++ b/event-schemas/schema/v1/m.room.name @@ -4,7 +4,7 @@ "description": "A room has an opaque room ID which is not human-friendly to read. A room alias is human-friendly, but not all rooms have room aliases. The room name is a human-friendly string designed to be displayed to the end-user. The room name is not unique, as multiple rooms can have the same room name set. The room name can also be set when creating a room using ``/createRoom`` with the ``name`` key.", "type": "object", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.power_levels b/event-schemas/schema/v1/m.room.power_levels index 95850fcb9..a682a214d 100644 --- a/event-schemas/schema/v1/m.room.power_levels +++ b/event-schemas/schema/v1/m.room.power_levels @@ -4,7 +4,7 @@ "title": "Defines the power levels (privileges) of users in the room.", "description": "This event specifies the minimum level a user must have in order to perform a certain action. It also specifies the levels of each user in the room. If a ``user_id`` is in the ``users`` list, then that ``user_id`` has the associated power level. Otherwise they have the default level ``users_default``. If ``users_default`` is not supplied, it is assumed to be 0. The level required to send a certain event is governed by ``events``, ``state_default`` and ``events_default``. If an event type is specified in ``events``, then the user must have at least the level specified in order to send that event. If the event type is not supplied, it defaults to ``events_default`` for Message Events and ``state_default`` for State Events.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.redaction b/event-schemas/schema/v1/m.room.redaction index b095b17d7..5168e1334 100644 --- a/event-schemas/schema/v1/m.room.redaction +++ b/event-schemas/schema/v1/m.room.redaction @@ -4,7 +4,7 @@ "title": "Redaction", "description": "Events can be redacted by either room or server admins. Redacting an event means that all keys not required by the protocol are stripped off, allowing admins to remove offensive or illegal content that may have been attached to any event. This cannot be undone, allowing server owners to physically delete the offending data. There is also a concept of a moderator hiding a message event, which can be undone, but cannot be applied to state events. The event that has been redacted is specified in the ``redacts`` event level key.", "allOf": [{ - "$ref": "core/room_event.json" + "$ref": "core-event-schema/room_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/m.room.topic b/event-schemas/schema/v1/m.room.topic index 32319c383..f88f56ca8 100644 --- a/event-schemas/schema/v1/m.room.topic +++ b/event-schemas/schema/v1/m.room.topic @@ -4,7 +4,7 @@ "title": "Topic", "description": "A topic is a short message detailing what is currently being discussed in the room. It can also be used as a way to display extra information about the room, which may not be suitable for the room name. The room topic can also be set when creating a room using ``/createRoom`` with the ``topic`` key.", "allOf": [{ - "$ref": "core/state_event.json" + "$ref": "core-event-schema/state_event.json" }], "properties": { "content": { diff --git a/event-schemas/schema/v1/v1-event-schema b/event-schemas/schema/v1/v1-event-schema new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/event-schemas/schema/v1/v1-event-schema @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 4b8127aea..8e803d3ba 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -281,7 +281,7 @@ class MatrixUnits(Units): return apis def load_common_event_fields(self): - path = "../event-schemas/schema/v1/core" + path = "../event-schemas/schema/v1/core-event-schema" event_types = {} for (root, dirs, files) in os.walk(path): From 1f6b12b3e828696a588fed9a0028e59991d28b94 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 11:43:53 +0100 Subject: [PATCH 081/223] Modify and enforce the file format/structure used Convert the file format to be of the form ##_##_something.rst where the first ## is the top-level section number and the second ## is the second-level section number, e.g. 07_01_push_cs_api.rst means Section 7.1 - This is now enforced in gendoc.py along with the title line style that should be used (= for top-level, - for 2nd level) which will give helpful suggestions if you trip up. This feels much more intuitive now looking in /specification --- scripts/gendoc.py | 12 +++++++----- specification/{00_basis.rst => 00_00_intro.rst} | 0 ..._server_api.rst => 01_00_client_server_api.rst} | 0 ...ications.rst => 01_01_typing_notifications.rst} | 8 ++++---- .../{46_receipts.rst => 01_02_receipts.rst} | 8 ++++---- ...visibility.rst => 01_03_history_visibility.rst} | 2 +- specification/{20_events.rst => 02_00_events.rst} | 0 .../{22_voip_events.rst => 02_01_voip_events.rst} | 0 ...1_event_signing.rst => 02_02_event_signing.rst} | 14 +++++++------- ...e_api.rst => 03_00_application_service_api.rst} | 0 ..._server_api.rst => 04_00_server_server_api.rst} | 0 ...repository.rst => 05_00_content_repository.rst} | 0 ...ryption.rst => 06_00_end_to_end_encryption.rst} | 0 ...2_push_overview.rst => 07_00_push_overview.rst} | 0 .../{43_push_cs_api.rst => 07_01_push_cs_api.rst} | 0 ..._push_gw_api.rst => 07_02_push_push_gw_api.rst} | 0 ...n_core_apis.rst => 08_00_address_book_repo.rst} | 0 ...tity_servers.rst => 09_00_identity_servers.rst} | 0 .../{50_appendices.rst => 10_00_appendices.rst} | 0 19 files changed, 23 insertions(+), 21 deletions(-) rename specification/{00_basis.rst => 00_00_intro.rst} (100%) rename specification/{10_client_server_api.rst => 01_00_client_server_api.rst} (100%) rename specification/{45_typing_notifications.rst => 01_01_typing_notifications.rst} (96%) rename specification/{46_receipts.rst => 01_02_receipts.rst} (94%) rename specification/{47_history_visibility.rst => 01_03_history_visibility.rst} (97%) rename specification/{20_events.rst => 02_00_events.rst} (100%) rename specification/{22_voip_events.rst => 02_01_voip_events.rst} (100%) rename specification/{31_event_signing.rst => 02_02_event_signing.rst} (99%) rename specification/{25_application_service_api.rst => 03_00_application_service_api.rst} (100%) rename specification/{30_server_server_api.rst => 04_00_server_server_api.rst} (100%) rename specification/{40_content_repository.rst => 05_00_content_repository.rst} (100%) rename specification/{41_end_to_end_encryption.rst => 06_00_end_to_end_encryption.rst} (100%) rename specification/{42_push_overview.rst => 07_00_push_overview.rst} (100%) rename specification/{43_push_cs_api.rst => 07_01_push_cs_api.rst} (100%) rename specification/{44_push_push_gw_api.rst => 07_02_push_push_gw_api.rst} (100%) rename specification/{49_other_non_core_apis.rst => 08_00_address_book_repo.rst} (100%) rename specification/{60_identity_servers.rst => 09_00_identity_servers.rst} (100%) rename specification/{50_appendices.rst => 10_00_appendices.rst} (100%) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 1348679a7..c97995e86 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -21,7 +21,7 @@ title_style_matchers = { } TOP_LEVEL = "=" SECOND_LEVEL = "-" -FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}_.*\.rst$") +FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") def check_valid_section(filename, section): @@ -55,14 +55,16 @@ def check_valid_section(filename, section): "style: expected '" + TOP_LEVEL + "' but got '" + title_line[0] + "'" ) - # anything marked as xx_x0_ is the start of a sub-section - elif re.match("^[0-9]+_0[0-9]{1}_", filename): + # anything marked as xx_xx_ is the start of a sub-section + elif re.match("^[0-9]+_[0-9]{2}_", filename): if not title_style_matchers[SECOND_LEVEL].match(title_line): raise Exception( "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format ##_#0_something.rst but has the wrong title " + + "the filename format ##_##_something.rst but has the wrong title " + "style: expected '" + SECOND_LEVEL + "' but got '" + - title_line[0] + "'" + title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " + + "then use the form '##_##b_something.rst' which will not apply this " + + "check." ) def cat_spec_sections_to(out_file_name): diff --git a/specification/00_basis.rst b/specification/00_00_intro.rst similarity index 100% rename from specification/00_basis.rst rename to specification/00_00_intro.rst diff --git a/specification/10_client_server_api.rst b/specification/01_00_client_server_api.rst similarity index 100% rename from specification/10_client_server_api.rst rename to specification/01_00_client_server_api.rst diff --git a/specification/45_typing_notifications.rst b/specification/01_01_typing_notifications.rst similarity index 96% rename from specification/45_typing_notifications.rst rename to specification/01_01_typing_notifications.rst index ff00165d6..25b714ab4 100644 --- a/specification/45_typing_notifications.rst +++ b/specification/01_01_typing_notifications.rst @@ -1,8 +1,8 @@ Typing Notifications -==================== +-------------------- Client APIs ------------ +~~~~~~~~~~~ To set "I am typing for the next N msec":: @@ -21,7 +21,7 @@ To set "I am no longer typing":: Content: { "typing": false } Client Events -------------- +~~~~~~~~~~~~~ All room members will receive an event on the event stream:: @@ -39,7 +39,7 @@ users who are not currently typing, as that list gets big quickly. The client should mark as not typing, any user ID who is not in that list. Server APIs ------------ +~~~~~~~~~~~ Servers will emit EDUs in the following form:: diff --git a/specification/46_receipts.rst b/specification/01_02_receipts.rst similarity index 94% rename from specification/46_receipts.rst rename to specification/01_02_receipts.rst index a0a90c16e..e2f83eea8 100644 --- a/specification/46_receipts.rst +++ b/specification/01_02_receipts.rst @@ -1,5 +1,5 @@ Receipts -======== +-------- Receipts are used to publish which events in a room the user or their devices have interacted with. For example, which events the user has read. For @@ -7,7 +7,7 @@ efficiency this is done as "up to" markers, i.e. marking a particular event as, say, ``read`` indicates the user has read all events *up to* that event. Client-Server API ------------------ +~~~~~~~~~~~~~~~~~ Clients will receive receipts in the following format:: @@ -58,7 +58,7 @@ other users. The server will automatically set the ``ts`` field. Server-Server API ------------------ +~~~~~~~~~~~~~~~~~ Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -73,5 +73,5 @@ format of the EDUs are:: ... } -These are always sent as deltas to previously sent reciepts. +These are always sent as deltas to previously sent receipts. diff --git a/specification/47_history_visibility.rst b/specification/01_03_history_visibility.rst similarity index 97% rename from specification/47_history_visibility.rst rename to specification/01_03_history_visibility.rst index b1630f1ef..01c2e419a 100644 --- a/specification/47_history_visibility.rst +++ b/specification/01_03_history_visibility.rst @@ -1,5 +1,5 @@ Room History Visibility -======================= +----------------------- Whether a member of a room can see the events that happened in a room from before they joined the room is controlled by the ``history_visibility`` key diff --git a/specification/20_events.rst b/specification/02_00_events.rst similarity index 100% rename from specification/20_events.rst rename to specification/02_00_events.rst diff --git a/specification/22_voip_events.rst b/specification/02_01_voip_events.rst similarity index 100% rename from specification/22_voip_events.rst rename to specification/02_01_voip_events.rst diff --git a/specification/31_event_signing.rst b/specification/02_02_event_signing.rst similarity index 99% rename from specification/31_event_signing.rst rename to specification/02_02_event_signing.rst index f3a22d1d8..bde58f0cd 100644 --- a/specification/31_event_signing.rst +++ b/specification/02_02_event_signing.rst @@ -1,8 +1,8 @@ Signing Events -============== +-------------- Canonical JSON --------------- +~~~~~~~~~~~~~~ Matrix events are represented using JSON objects. If we want to sign JSON events we need to encode the JSON as a binary string. Unfortunately the same @@ -38,7 +38,7 @@ using this representation. ).encode("UTF-8") Grammar -~~~~~~~ ++++++++ Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing insignificant whitespace, fractions, exponents and redundant character escapes @@ -69,14 +69,14 @@ insignificant whitespace, fractions, exponents and redundant character escapes / %x75.30.30.31 (%x30-39 / %x61-66) ; u001X Signing JSON ------------- +~~~~~~~~~~~~ We can now sign a JSON object by encoding it as a sequence of bytes, computing the signature for that sequence and then adding the signature to the original JSON object. Signing Details -~~~~~~~~~~~~~~~ ++++++++++++++++ JSON is signed by encoding the JSON object without ``signatures`` or keys grouped as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the @@ -133,7 +133,7 @@ and additional signatures. return json_object Checking for a Signature -~~~~~~~~~~~~~~~~~~~~~~~~ +++++++++++++++++++++++++ To check if an entity has signed a JSON object a server does the following @@ -151,7 +151,7 @@ To check if an entity has signed a JSON object a server does the following the check fails. Otherwise the check succeeds. Signing Events --------------- +~~~~~~~~~~~~~~ Signing events is a more complicated process since servers can choose to redact non-essential parts of an event. Before signing the event it is encoded as diff --git a/specification/25_application_service_api.rst b/specification/03_00_application_service_api.rst similarity index 100% rename from specification/25_application_service_api.rst rename to specification/03_00_application_service_api.rst diff --git a/specification/30_server_server_api.rst b/specification/04_00_server_server_api.rst similarity index 100% rename from specification/30_server_server_api.rst rename to specification/04_00_server_server_api.rst diff --git a/specification/40_content_repository.rst b/specification/05_00_content_repository.rst similarity index 100% rename from specification/40_content_repository.rst rename to specification/05_00_content_repository.rst diff --git a/specification/41_end_to_end_encryption.rst b/specification/06_00_end_to_end_encryption.rst similarity index 100% rename from specification/41_end_to_end_encryption.rst rename to specification/06_00_end_to_end_encryption.rst diff --git a/specification/42_push_overview.rst b/specification/07_00_push_overview.rst similarity index 100% rename from specification/42_push_overview.rst rename to specification/07_00_push_overview.rst diff --git a/specification/43_push_cs_api.rst b/specification/07_01_push_cs_api.rst similarity index 100% rename from specification/43_push_cs_api.rst rename to specification/07_01_push_cs_api.rst diff --git a/specification/44_push_push_gw_api.rst b/specification/07_02_push_push_gw_api.rst similarity index 100% rename from specification/44_push_push_gw_api.rst rename to specification/07_02_push_push_gw_api.rst diff --git a/specification/49_other_non_core_apis.rst b/specification/08_00_address_book_repo.rst similarity index 100% rename from specification/49_other_non_core_apis.rst rename to specification/08_00_address_book_repo.rst diff --git a/specification/60_identity_servers.rst b/specification/09_00_identity_servers.rst similarity index 100% rename from specification/60_identity_servers.rst rename to specification/09_00_identity_servers.rst diff --git a/specification/50_appendices.rst b/specification/10_00_appendices.rst similarity index 100% rename from specification/50_appendices.rst rename to specification/10_00_appendices.rst From 63f08bace61b01d629bcdc3beaaf8870c7bfd993 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 14:40:48 +0100 Subject: [PATCH 082/223] Fix the examples in the swagger API documentation to be valid JSON --- api/client-server/v1/presence.yaml | 6 +++--- api/client-server/v1/sync.yaml | 2 +- templating/matrix_templates/units.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/v1/presence.yaml b/api/client-server/v1/presence.yaml index 717e9c473..5684398b0 100644 --- a/api/client-server/v1/presence.yaml +++ b/api/client-server/v1/presence.yaml @@ -101,7 +101,7 @@ paths: The length of time in milliseconds since an action was performed by this user. status_msg: - type: string + type: [string, "null"] description: The state message for this user if one was set. 404: description: |- @@ -185,7 +185,7 @@ paths: "last_active_ago": 395, "presence": "offline", "user_id": "@alice:matrix.org" - } + }, "type": "m.presence" }, { @@ -195,7 +195,7 @@ paths: "last_active_ago": 16874, "presence": "online", "user_id": "@marisa:matrix.org" - } + }, "type": "m.presence" } ] diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 27c7073c4..833c425ab 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -343,7 +343,7 @@ paths: "body": "Hello world!", "msgtype": "m.text" }, - "room_id:" "!wfgy43Sg4a:matrix.org", + "room_id:": "!wfgy43Sg4a:matrix.org", "user_id": "@bob:matrix.org", "event_id": "$asfDuShaf7Gafaw:matrix.org", "type": "m.room.message" diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 8e803d3ba..3083a51ea 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -87,6 +87,8 @@ def get_json_schema_object_fields(obj, enforce_title=False): desc += ( " Must be '%s'." % props[key_name]["enum"][0] ) + if isinstance(value_type, list): + value_type = " or ".join(value_type) fields["rows"].append({ "key": key_name, From f99a38ce726e5d04a9be182bd42f43daf73a3364 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 14:43:57 +0100 Subject: [PATCH 083/223] Update the hard-coded paths in templating units.py. Replace the hard code paths with global variables. --- templating/matrix_templates/units.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 4b8127aea..701b68e24 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -8,6 +8,16 @@ import subprocess import urllib import yaml +V1_CLIENT_API = "../api/client-server/v1" +V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" +V1_EVENT_SCHEMA = "../event-schemas/schema/v1" +CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" +CHANGELOG = "../CHANGELOG.rst" + +ROOM_EVENT = "core-event-schema/room_event.json" +STATE_EVENT = "core-event-schema/state_event.json" + + def get_json_schema_object_fields(obj, enforce_title=False): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -266,7 +276,7 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - path = "../api/client-server/v1" + path = V1_CLIENT_API apis = {} for filename in os.listdir(path): if not filename.endswith(".yaml"): @@ -281,7 +291,7 @@ class MatrixUnits(Units): return apis def load_common_event_fields(self): - path = "../event-schemas/schema/v1/core" + path = CORE_EVENT_SCHEMA event_types = {} for (root, dirs, files) in os.walk(path): @@ -320,7 +330,7 @@ class MatrixUnits(Units): return event_types def load_event_examples(self): - path = "../event-schemas/examples/v1" + path = V1_EVENT_EXAMPLES examples = {} for filename in os.listdir(path): if not filename.startswith("m."): @@ -332,7 +342,7 @@ class MatrixUnits(Units): return examples def load_event_schemas(self): - path = "../event-schemas/schema/v1" + path = V1_EVENT_SCHEMA schemata = {} for filename in os.listdir(path): @@ -361,8 +371,8 @@ class MatrixUnits(Units): # add typeof base_defs = { - "core#/definitions/room_event": "Message Event", - "core#/definitions/state_event": "State Event" + ROOM_EVENT: "Message Event", + STATE_EVENT: "State Event" } if type(json_schema.get("allOf")) == list: schema["typeof"] = base_defs.get( @@ -413,7 +423,7 @@ class MatrixUnits(Units): return schemata def load_spec_meta(self): - path = "../CHANGELOG.rst" + path = CHANGELOG title_part = None version = None changelog_lines = [] From 427e4c8b1b23ab875b24acad5605848abf0d8ee9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 15:46:37 +0100 Subject: [PATCH 084/223] Log port being listened on --- scripts/continuserv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index e7757c06d..dc7705a85 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -56,7 +56,7 @@ func main() { go doPopulate(ch, dir) go watchFS(ch, w) - + fmt.Printf("Listening on port %d\n", *port) http.HandleFunc("/", serve) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From ba4998a62b28fcd13c9085ae58996d1c857b27c5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 17 Sep 2015 15:49:01 +0100 Subject: [PATCH 085/223] Apparently tabs are A Thing --- scripts/continuserv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index dc7705a85..658ae0fb6 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -56,7 +56,7 @@ func main() { go doPopulate(ch, dir) go watchFS(ch, w) - fmt.Printf("Listening on port %d\n", *port) + fmt.Printf("Listening on port %d\n", *port) http.HandleFunc("/", serve) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From cf8abdafd43ad67802b20896d53ab977d6d4beee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 18 Sep 2015 15:42:49 +0100 Subject: [PATCH 086/223] Fix PR comments --- scripts/gendoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index c97995e86..e2f51419f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -27,13 +27,13 @@ FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") def check_valid_section(filename, section): if not re.match(FILE_FORMAT_MATCHER, filename): raise Exception( - "The filename of " + filename +" does not match the expected format " + + "The filename of " + filename + " does not match the expected format " + "of '##_##_words-go-here.rst'" ) # we need TWO new lines else the next file's title gets merged # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == '\n\n': + if not section[-2:] == "\n\n": raise Exception( "The file " + filename + " does not end with 2 new lines." ) @@ -72,7 +72,7 @@ def cat_spec_sections_to(out_file_name): for f in sorted(glob.glob("../specification/*.rst")): with open(f, "rb") as infile: section = infile.read() - check_valid_section(f.split("/")[-1], section) + check_valid_section(os.path.basename(f), section) outfile.write(section) From 299a4356d49e9b694c1fb6340cda2669e8e30377 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 16:10:21 +0100 Subject: [PATCH 087/223] Add script to check that the example responses in the swagger matches the examples. --- api/check_examples.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 api/check_examples.py diff --git a/api/check_examples.py b/api/check_examples.py new file mode 100755 index 000000000..cec8ac09c --- /dev/null +++ b/api/check_examples.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python + +import sys + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__=='__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + +import json +import os + +def check_response(filepath, request, code, response): + try: + example = json.loads( + response.get('examples', {}).get('application/json', "null") + ) + except Exception as e: + raise ValueError("Error parsing JSON example response for %r %r" % ( + request, code + ), e) + schema = response.get('schema') + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking schema for: %r %r %r" % (filepath, request, code)) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" %( + request, code + ), e) + + +def check_swagger_file(filepath): + with open(filepath) as f: + swagger = yaml.load(f) + + for path, path_api in swagger['paths'].items(): + for method, request_api in path_api.items(): + request = "%s %s" % (method.upper(), path) + try: + responses = request_api['responses'] + except KeyError: + raise ValueError("No responses for %r" % (request,)) + for code, response in responses.items(): + check_response(filepath, request, code, response) + + +if __name__=='__main__': + for path in sys.argv[1:]: + try: + check_swagger_file(path) + except Exception as e: + raise ValueError("Error checking file %r" % (path,), e) From 9896f98e2b63fd6204482bd9c7b8ba699487576d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 16:21:48 +0100 Subject: [PATCH 088/223] Search for yaml swagger files if check_examples.py is run without arguments. --- api/check_examples.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index cec8ac09c..9b2f1fe99 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -1,6 +1,9 @@ #! /usr/bin/env python import sys +import json +import os + def import_error(module, package, debian, error): sys.stderr.write(( @@ -10,7 +13,7 @@ def import_error(module, package, debian, error): "or on Debian run:\n" " sudo apt-get install python-%(debian)s\n" ) % locals()) - if __name__=='__main__': + if __name__ == '__main__': sys.exit(1) try: @@ -25,8 +28,6 @@ except ImportError as e: import_error("yaml", "PyYAML", "yaml", e) raise -import json -import os def check_response(filepath, request, code, response): try: @@ -47,7 +48,7 @@ def check_response(filepath, request, code, response): schema['id'] = fileurl jsonschema.validate(example, schema) except Exception as e: - raise ValueError("Error validating JSON schema for %r %r" %( + raise ValueError("Error validating JSON schema for %r %r" % ( request, code ), e) @@ -56,7 +57,7 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in swagger['paths'].items(): + for path, path_api in swagger.get('paths', {}).items(): for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) try: @@ -67,8 +68,15 @@ def check_swagger_file(filepath): check_response(filepath, request, code, response) -if __name__=='__main__': - for path in sys.argv[1:]: +if __name__ == '__main__': + paths = sys.argv[1:] + if not paths: + paths = [] + for (root, dirs, files) in os.walk(os.curdir): + for filename in files: + if filename.endswith(".yaml"): + paths.append(os.path.join(root, filename)) + for path in paths: try: check_swagger_file(path) except Exception as e: From f827765ba1999073bc9357695b27bcc3e7e0c30c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 16:35:27 +0100 Subject: [PATCH 089/223] Make to code to skip checking swagger responses which don't have an application/json example clearer. --- api/check_examples.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index 9b2f1fe99..00e75263c 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -30,10 +30,11 @@ except ImportError as e: def check_response(filepath, request, code, response): + example = None try: - example = json.loads( - response.get('examples', {}).get('application/json', "null") - ) + example_json = response.get('examples', {}).get('application/json') + if example_json: + example = json.loads(example_json) except Exception as e: raise ValueError("Error parsing JSON example response for %r %r" % ( request, code From 6b5b8432b3761484f3e36910575f5fd844c09400 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 17:26:10 +0100 Subject: [PATCH 090/223] Turn on code highlighting for HTTP api responses and add a code highlighting stylesheet for the specification. --- scripts/codehighlight.css | 6 ++++++ scripts/gendoc.py | 2 +- templating/matrix_templates/templates/http-api.tmpl | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 scripts/codehighlight.css diff --git a/scripts/codehighlight.css b/scripts/codehighlight.css new file mode 100644 index 000000000..5c9b0c36f --- /dev/null +++ b/scripts/codehighlight.css @@ -0,0 +1,6 @@ +pre.code .comment, code .comment { color: green } +pre.code .keyword, code .keyword { color: darkred; font-weight: bold } +pre.code .name.builtin, code .name.builtin { color: darkred; font-weight: bold } +pre.code .literal.number, code .literal.number { color: blue } +pre.code .name.tag, code .name.tag { color: darkgreen } +pre.code .literal.string, code .literal.string { color: darkblue } diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 3521efed5..a821aea7f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -12,7 +12,7 @@ import sys os.chdir(os.path.dirname(os.path.abspath(__file__))) stylesheets = { - "stylesheet_path": ["basic.css", "nature.css"] + "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] } title_style_matchers = { diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index a0b25924f..a03ffa7d2 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -62,7 +62,9 @@ Response{{"s" if endpoint.example.responses|length > 1 else "" }}: {{res["description"]}} -Example:: +Example + +.. code:: json {{res["example"] | indent_block(2)}} From 6a2c4d27fc187d6efb755705be9eca7a07dec26f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 18 Sep 2015 17:58:44 +0100 Subject: [PATCH 091/223] Update the docs for room v1 api --- api/client-server/v1/rooms.yaml | 10 +++++----- event-schemas/schema/v1/m.room.member | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index c038dba54..4902560b3 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -168,7 +168,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "events/core/state_event.json" + - "$ref": "core-event-schema/state_event.json" 403: description: You are not joined to the room. @@ -344,7 +344,7 @@ paths: type: object title: RoomEvent allOf: - - "$ref": "events/core/room_event.json" + - "$ref": "core-event-schema/room_event.json" required: ["start", "end", "chunk"] state: type: array @@ -357,7 +357,7 @@ paths: title: StateEvent type: object allOf: - - "$ref": "events/core/state_event.json" + - "$ref": "core-event-schema/state_event.json" visibility: type: string enum: ["private", "public"] @@ -414,7 +414,7 @@ paths: "state_key": "@bob:example.com", "type": "m.room.member", "user_id": "@bob:example.com" - }, + } ] } schema: @@ -426,5 +426,5 @@ paths: title: MemberEvent type: object allOf: - - "$ref": "events/m.room.member" + - "$ref": "v1-event-schema/m.room.member" diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 48f1b0347..49b9f5b83 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -19,7 +19,7 @@ "description": "The avatar URL for this user, if any. This is added by the homeserver." }, "displayname": { - "type": "string", + "type": ["string", "null"], "description": "The display name for this user, if any. This is added by the homeserver." } }, From 52640eb2053e155c7f33468c0e6c74512b062b0b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 13:02:37 +0100 Subject: [PATCH 092/223] Add a python script for checking that the examples match the event schema. Does the same checks as check.sh, but is a *lot* faster making it suitable for using as a pre-commit hook. I don't suggest replacing check.sh since it's good to check that the schema works with multiple implementations of jsonschema. --- event-schemas/check_examples.py | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 event-schemas/check_examples.py diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py new file mode 100755 index 000000000..5fc08f4ac --- /dev/null +++ b/event-schemas/check_examples.py @@ -0,0 +1,62 @@ +#! /usr/bin/env python + +import sys +import json +import os + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + + +def check_example_file(examplepath, schemapath): + with open(examplepath) as f: + example = yaml.load(f) + + with open(schemapath) as f: + schema = yaml.load(f) + + fileurl = "file://" + os.path.abspath(schemapath) + + print ("Checking schema for: %r %r" % (examplepath, schemapath)) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + try: + jsonschema.validate(example, schema) + except: + raise ValueError("Error validating JSON schema for %r %r" % ( + examplepath, schemapath + ), e) + + +def check_example_dir(exampledir, schemadir): + for root, dirs, files in os.walk(exampledir): + for filename in files: + examplepath = os.path.join(root, filename) + schemapath = examplepath.replace(exampledir, schemadir) + check_example_file(examplepath, schemapath) + + +if __name__ == '__main__': + check_example_dir("examples", "schema") From f2c952ec5855bb120814a736bf482a19e1c43fd3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:13:18 +0100 Subject: [PATCH 093/223] Reshuffle spec into sections roughly right as per PR #52 --- .../address-book-repo.rst | 0 ..._repository.rst => 02_03_content_repo.rst} | 0 ...on.rst => 02_04_end_to_end_encryption.rst} | 0 ...ons.rst => 02_05_typing_notifications.rst} | 0 ...{01_02_receipts.rst => 02_06_receipts.rst} | 0 ...ility.rst => 02_07_history_visibility.rst} | 0 ...h_overview.rst => 02_08_push_overview.rst} | 0 ...push_cs_api.rst => 02_08b_push_cs_api.rst} | 0 ...push_gw_api.rst => 02_08c_push_cs_api.rst} | 0 .../03_00_application_service_api.rst | 22 ------------------- ...servers.rst => 05_00_identity_servers.rst} | 0 ...00_appendices.rst => 06_00_appendices.rst} | 0 12 files changed, 22 deletions(-) rename specification/08_00_address_book_repo.rst => drafts/address-book-repo.rst (100%) rename specification/{05_00_content_repository.rst => 02_03_content_repo.rst} (100%) rename specification/{06_00_end_to_end_encryption.rst => 02_04_end_to_end_encryption.rst} (100%) rename specification/{01_01_typing_notifications.rst => 02_05_typing_notifications.rst} (100%) rename specification/{01_02_receipts.rst => 02_06_receipts.rst} (100%) rename specification/{01_03_history_visibility.rst => 02_07_history_visibility.rst} (100%) rename specification/{07_00_push_overview.rst => 02_08_push_overview.rst} (100%) rename specification/{07_01_push_cs_api.rst => 02_08b_push_cs_api.rst} (100%) rename specification/{07_02_push_push_gw_api.rst => 02_08c_push_cs_api.rst} (100%) rename specification/{09_00_identity_servers.rst => 05_00_identity_servers.rst} (100%) rename specification/{10_00_appendices.rst => 06_00_appendices.rst} (100%) diff --git a/specification/08_00_address_book_repo.rst b/drafts/address-book-repo.rst similarity index 100% rename from specification/08_00_address_book_repo.rst rename to drafts/address-book-repo.rst diff --git a/specification/05_00_content_repository.rst b/specification/02_03_content_repo.rst similarity index 100% rename from specification/05_00_content_repository.rst rename to specification/02_03_content_repo.rst diff --git a/specification/06_00_end_to_end_encryption.rst b/specification/02_04_end_to_end_encryption.rst similarity index 100% rename from specification/06_00_end_to_end_encryption.rst rename to specification/02_04_end_to_end_encryption.rst diff --git a/specification/01_01_typing_notifications.rst b/specification/02_05_typing_notifications.rst similarity index 100% rename from specification/01_01_typing_notifications.rst rename to specification/02_05_typing_notifications.rst diff --git a/specification/01_02_receipts.rst b/specification/02_06_receipts.rst similarity index 100% rename from specification/01_02_receipts.rst rename to specification/02_06_receipts.rst diff --git a/specification/01_03_history_visibility.rst b/specification/02_07_history_visibility.rst similarity index 100% rename from specification/01_03_history_visibility.rst rename to specification/02_07_history_visibility.rst diff --git a/specification/07_00_push_overview.rst b/specification/02_08_push_overview.rst similarity index 100% rename from specification/07_00_push_overview.rst rename to specification/02_08_push_overview.rst diff --git a/specification/07_01_push_cs_api.rst b/specification/02_08b_push_cs_api.rst similarity index 100% rename from specification/07_01_push_cs_api.rst rename to specification/02_08b_push_cs_api.rst diff --git a/specification/07_02_push_push_gw_api.rst b/specification/02_08c_push_cs_api.rst similarity index 100% rename from specification/07_02_push_push_gw_api.rst rename to specification/02_08c_push_cs_api.rst diff --git a/specification/03_00_application_service_api.rst b/specification/03_00_application_service_api.rst index 9b58c8614..2674ba443 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/03_00_application_service_api.rst @@ -400,25 +400,3 @@ in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix - -Active Application Services ----------------------------- -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - API that provides hooks into the server so that you can intercept and - manipulate events, and/or insert virtual users & rooms into the server. - -Policy Servers -============== -.. NOTE:: - This section is a work in progress. - -.. TODO-spec - We should mention them in the Architecture section at least: how they fit - into the picture. - -Enforcing policies ------------------- - diff --git a/specification/09_00_identity_servers.rst b/specification/05_00_identity_servers.rst similarity index 100% rename from specification/09_00_identity_servers.rst rename to specification/05_00_identity_servers.rst diff --git a/specification/10_00_appendices.rst b/specification/06_00_appendices.rst similarity index 100% rename from specification/10_00_appendices.rst rename to specification/06_00_appendices.rst From c77ef1a2ccca95149c4d9d9d9ee965068190fd23 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:17:31 +0100 Subject: [PATCH 094/223] Add stub files feature_profiles and modules --- specification/00_01_feature_profiles.rst | 0 specification/{02_00_events.rst => 00_02a_events.rst} | 0 .../{02_02_event_signing.rst => 00_02b_event_signing.rst} | 0 specification/02_00_modules.rst | 0 .../{02_08_push_overview.rst => 02_08a_push_overview.rst} | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 specification/00_01_feature_profiles.rst rename specification/{02_00_events.rst => 00_02a_events.rst} (100%) rename specification/{02_02_event_signing.rst => 00_02b_event_signing.rst} (100%) create mode 100644 specification/02_00_modules.rst rename specification/{02_08_push_overview.rst => 02_08a_push_overview.rst} (100%) diff --git a/specification/00_01_feature_profiles.rst b/specification/00_01_feature_profiles.rst new file mode 100644 index 000000000..e69de29bb diff --git a/specification/02_00_events.rst b/specification/00_02a_events.rst similarity index 100% rename from specification/02_00_events.rst rename to specification/00_02a_events.rst diff --git a/specification/02_02_event_signing.rst b/specification/00_02b_event_signing.rst similarity index 100% rename from specification/02_02_event_signing.rst rename to specification/00_02b_event_signing.rst diff --git a/specification/02_00_modules.rst b/specification/02_00_modules.rst new file mode 100644 index 000000000..e69de29bb diff --git a/specification/02_08_push_overview.rst b/specification/02_08a_push_overview.rst similarity index 100% rename from specification/02_08_push_overview.rst rename to specification/02_08a_push_overview.rst From f520ac9d828890193772764e902c3ae5b4b77c1d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:25:52 +0100 Subject: [PATCH 095/223] Create a modules folder to group together similar parts of the spec --- .../{02_01_voip_events.rst => modules/01_00_voip_events.rst} | 0 .../02_00_typing_notifications.rst} | 0 specification/{02_06_receipts.rst => modules/03_00_receipts.rst} | 0 .../{02_03_content_repo.rst => modules/04_00_content_repo.rst} | 0 .../05_00_end_to_end_encryption.rst} | 0 .../06_00_history_visibility.rst} | 0 .../{02_08a_push_overview.rst => modules/07_00_push_overview.rst} | 0 .../{02_08b_push_cs_api.rst => modules/07_01_push_cs_api.rst} | 0 .../07_02_push_push_gw_api.rst} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename specification/{02_01_voip_events.rst => modules/01_00_voip_events.rst} (100%) rename specification/{02_05_typing_notifications.rst => modules/02_00_typing_notifications.rst} (100%) rename specification/{02_06_receipts.rst => modules/03_00_receipts.rst} (100%) rename specification/{02_03_content_repo.rst => modules/04_00_content_repo.rst} (100%) rename specification/{02_04_end_to_end_encryption.rst => modules/05_00_end_to_end_encryption.rst} (100%) rename specification/{02_07_history_visibility.rst => modules/06_00_history_visibility.rst} (100%) rename specification/{02_08a_push_overview.rst => modules/07_00_push_overview.rst} (100%) rename specification/{02_08b_push_cs_api.rst => modules/07_01_push_cs_api.rst} (100%) rename specification/{02_08c_push_cs_api.rst => modules/07_02_push_push_gw_api.rst} (100%) diff --git a/specification/02_01_voip_events.rst b/specification/modules/01_00_voip_events.rst similarity index 100% rename from specification/02_01_voip_events.rst rename to specification/modules/01_00_voip_events.rst diff --git a/specification/02_05_typing_notifications.rst b/specification/modules/02_00_typing_notifications.rst similarity index 100% rename from specification/02_05_typing_notifications.rst rename to specification/modules/02_00_typing_notifications.rst diff --git a/specification/02_06_receipts.rst b/specification/modules/03_00_receipts.rst similarity index 100% rename from specification/02_06_receipts.rst rename to specification/modules/03_00_receipts.rst diff --git a/specification/02_03_content_repo.rst b/specification/modules/04_00_content_repo.rst similarity index 100% rename from specification/02_03_content_repo.rst rename to specification/modules/04_00_content_repo.rst diff --git a/specification/02_04_end_to_end_encryption.rst b/specification/modules/05_00_end_to_end_encryption.rst similarity index 100% rename from specification/02_04_end_to_end_encryption.rst rename to specification/modules/05_00_end_to_end_encryption.rst diff --git a/specification/02_07_history_visibility.rst b/specification/modules/06_00_history_visibility.rst similarity index 100% rename from specification/02_07_history_visibility.rst rename to specification/modules/06_00_history_visibility.rst diff --git a/specification/02_08a_push_overview.rst b/specification/modules/07_00_push_overview.rst similarity index 100% rename from specification/02_08a_push_overview.rst rename to specification/modules/07_00_push_overview.rst diff --git a/specification/02_08b_push_cs_api.rst b/specification/modules/07_01_push_cs_api.rst similarity index 100% rename from specification/02_08b_push_cs_api.rst rename to specification/modules/07_01_push_cs_api.rst diff --git a/specification/02_08c_push_cs_api.rst b/specification/modules/07_02_push_push_gw_api.rst similarity index 100% rename from specification/02_08c_push_cs_api.rst rename to specification/modules/07_02_push_push_gw_api.rst From 703f913a47afc3b035a1730e9014e89807370b3e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:31:57 +0100 Subject: [PATCH 096/223] Add a 'targets.yaml' files in /specification We're well beyond the point now where a simple `cat` of .rst files to "build" the spec is practical. We may want to slice and dice the spec in different ways to address various cross-cutting concerns. To this end, there is now a 'targets' file which contains the "build targets" for the spec, which contains the sorting order for the .rst files. For now, we just have a single target: 'main'. --- specification/targets.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 specification/targets.yaml diff --git a/specification/targets.yaml b/specification/targets.yaml new file mode 100644 index 000000000..845da99dc --- /dev/null +++ b/specification/targets.yaml @@ -0,0 +1,25 @@ +targets: + main: + files: + - 00_00_intro.rst + - 00_01_feature_profiles.rst + - 00_02a_events.rst + - 00_02b_event_signing.rst + - 01_00_client_server_api.rst + - 02_00_modules.rst + - "group:module" + - 03_00_application_service_api.rst + - 04_00_server_server_api.rst + - 05_00_identity_servers.rst + - 06_00_appendices.rst +groups: + modules: + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - modules/07_00_push_overview.rst + - modules/07_01_push_cs_api.rst + - modules/07_02_push_push_gw_api.rst From 568982ee2e5a3dbf2f76556475dfdd8fae96998d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 14:49:54 +0100 Subject: [PATCH 097/223] Represent nested deps in targets.yaml along with title styles This will allow us to programatically position .rst snippets *anywhere* which will for once and for all remove the horrid title level mismatch bugs. We require this in order to allow people to re-shuffle the spec without having to adjust the spec itself (e.g. 2 targets with different levels of nesting). --- specification/targets.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/specification/targets.yaml b/specification/targets.yaml index 845da99dc..b61e2e201 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -20,6 +20,10 @@ groups: - modules/04_00_content_repo.rst - modules/05_00_end_to_end_encryption.rst - modules/06_00_history_visibility.rst - - modules/07_00_push_overview.rst - - modules/07_01_push_cs_api.rst - - modules/07_02_push_push_gw_api.rst + - 1: modules/07_00_push_overview.rst + 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] +title_styles: + - "=" + - "-" + - "~" + - "+" From 6ba9b29b3b4ffe9d19c30302e36813b1835eb117 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 15:04:03 +0100 Subject: [PATCH 098/223] Report all the errors in schemas/check_examples, not just the first error. --- event-schemas/check_examples.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index 5fc08f4ac..8675396e3 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -3,6 +3,7 @@ import sys import json import os +import traceback def import_error(module, package, debian, error): @@ -44,19 +45,29 @@ def check_example_file(examplepath, schemapath): schema['id'] = fileurl try: jsonschema.validate(example, schema) - except: + except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( examplepath, schemapath ), e) def check_example_dir(exampledir, schemadir): + errors = [] for root, dirs, files in os.walk(exampledir): for filename in files: examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) - check_example_file(examplepath, schemapath) - + try: + check_example_file(examplepath, schemapath) + except Exception as e: + errors.append(sys.exc_info()) + for (exc_type, exc_value, exc_trace) in errors: + traceback.print_exception(exc_type, exc_value, exc_trace) + if errors: + raise ValueError("Error validating examples") if __name__ == '__main__': - check_example_dir("examples", "schema") + try: + check_example_dir("examples", "schema") + except: + sys.exit(1) From 8974b2b67bbc2acf0c482c646be6b07b230a1a15 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 15:05:10 +0100 Subject: [PATCH 099/223] Skip files that start with ".", e.g. vim swp files. --- event-schemas/check_examples.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/event-schemas/check_examples.py b/event-schemas/check_examples.py index 8675396e3..e54d3a1ca 100755 --- a/event-schemas/check_examples.py +++ b/event-schemas/check_examples.py @@ -55,6 +55,9 @@ def check_example_dir(exampledir, schemadir): errors = [] for root, dirs, files in os.walk(exampledir): for filename in files: + if filename.startswith("."): + # Skip over any vim .swp files. + continue examplepath = os.path.join(root, filename) schemapath = examplepath.replace(exampledir, schemadir) try: From 2c31731262b3258307dd7e5eb8364d71817c4749 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 15:16:02 +0100 Subject: [PATCH 100/223] Add the jenkins command to source control so that we can update it without having to fiddle with the jenkins UI. It also allow us to move files without breaking the CI since we won't be hard coding the locations of scripts in the jenkins UI. --- jenkins.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 jenkins.sh diff --git a/jenkins.sh b/jenkins.sh new file mode 100755 index 000000000..e10436449 --- /dev/null +++ b/jenkins.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -ex + +(cd event-schemas/ && ./check_examples.py) +(cd api && ./check_examples.py) +(cd scripts && ./gendoc.py) +(cd api && npm install && node validator.js -s "client-server/v1") +(cd event-schemas/ && ./check.sh) From 8590cc84b51ba81b14b0f73988879368c26e649f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 15:33:28 +0100 Subject: [PATCH 101/223] Process and extract targets in gendoc.py --- scripts/gendoc.py | 74 +++++++++++++++++++++++++++++++------- specification/targets.yaml | 10 +++--- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 3521efed5..69c39dcca 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -8,6 +8,7 @@ import re import shutil import subprocess import sys +import yaml os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -24,7 +25,7 @@ SECOND_LEVEL = "-" FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") -def check_valid_section(filename, section): +def check_valid_section_old(filename, section): if not re.match(FILE_FORMAT_MATCHER, filename): raise Exception( "The filename of " + filename + " does not match the expected format " + @@ -67,13 +68,19 @@ def check_valid_section(filename, section): "check." ) -def cat_spec_sections_to(out_file_name): - with open(out_file_name, "wb") as outfile: - for f in sorted(glob.glob("../specification/*.rst")): - with open(f, "rb") as infile: - section = infile.read() - check_valid_section(os.path.basename(f), section) - outfile.write(section) +def check_valid_section(section): + pass + + +def get_rst(file_info, target): + pass + +def build_spec(target, out_filename): + with open(out_filename, "wb") as outfile: + for file_info in target["files"]: + section = get_rst(file_info, target) + check_valid_section(section) + outfile.write(section) def rst2html(i, o): @@ -88,6 +95,7 @@ def rst2html(i, o): settings_overrides=stylesheets ) + def run_through_template(input): tmpfile = './tmp/output' try: @@ -107,6 +115,43 @@ def run_through_template(input): sys.stderr.write(f.read() + "\n") raise + +def get_build_target(targets_listing, target_name): + build_target = { + "title_styles": [], + "files": [] + } + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + build_target["title_styles"] = all_targets["title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + resolved_files = [] + for f in target["files"]: + if isinstance(f, basestring) and f.startswith("group:"): + # copy across the group of files specified + group_name = f[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not isinstance(group, list): + raise Exception( + "Tried to find group '" + group_name + "' but either " + + "it doesn't exist or it isn't a list of files." + ) + resolved_files.extend(group) + else: + resolved_files.append(f) + build_target["files"] = resolved_files + return build_target + + def prepare_env(): try: os.makedirs("./gen") @@ -116,13 +161,17 @@ def prepare_env(): os.makedirs("./tmp") except OSError: pass - + + def cleanup_env(): shutil.rmtree("./tmp") -def main(): + +def main(target_name): prepare_env() - cat_spec_sections_to("tmp/full_spec.rst") + target = get_build_target("../specification/targets.yaml", target_name) + print target + build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") @@ -131,6 +180,7 @@ def main(): if "--nodelete" not in sys.argv: cleanup_env() + if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: # we accept almost no args, so they don't know what they're doing! @@ -145,4 +195,4 @@ if __name__ == '__main__': print "Requirements:" print " - This script requires Jinja2 and rst2html (docutils)." sys.exit(0) - main() + main("main") diff --git a/specification/targets.yaml b/specification/targets.yaml index b61e2e201..c4828ef17 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,18 +1,18 @@ targets: - main: - files: + main: # arbitrary name to identify this build target + files: # the sort order of files to cat - 00_00_intro.rst - 00_01_feature_profiles.rst - 00_02a_events.rst - 00_02b_event_signing.rst - 01_00_client_server_api.rst - 02_00_modules.rst - - "group:module" + - "group:modules" # reference a group of files - 03_00_application_service_api.rst - 04_00_server_server_api.rst - 05_00_identity_servers.rst - 06_00_appendices.rst -groups: +groups: # reusable blobs of files when prefixed with 'group:' modules: - modules/01_00_voip_events.rst - modules/02_00_typing_notifications.rst @@ -20,6 +20,7 @@ groups: - modules/04_00_content_repo.rst - modules/05_00_end_to_end_encryption.rst - modules/06_00_history_visibility.rst + # Mark a nested file dependency - 1: modules/07_00_push_overview.rst 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] title_styles: @@ -27,3 +28,4 @@ title_styles: - "-" - "~" - "+" + From 65ce95249f99fb67e61377e5b35734522a6d7ba3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 16:54:30 +0100 Subject: [PATCH 102/223] cat the spec according to the build target. Remove old checks as they are now obsolete since we don't care about the filename --- scripts/gendoc.py | 121 ++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 69c39dcca..22aaad11e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -16,70 +16,74 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css"] } -title_style_matchers = { - "=": re.compile("^=+$"), - "-": re.compile("^-+$") -} -TOP_LEVEL = "=" -SECOND_LEVEL = "-" -FILE_FORMAT_MATCHER = re.compile("^[0-9]+_[0-9]{2}[a-z]*_.*\.rst$") - - -def check_valid_section_old(filename, section): - if not re.match(FILE_FORMAT_MATCHER, filename): - raise Exception( - "The filename of " + filename + " does not match the expected format " + - "of '##_##_words-go-here.rst'" - ) - - # we need TWO new lines else the next file's title gets merged - # the last paragraph *WITHOUT RST PRODUCING A WARNING* - if not section[-2:] == "\n\n": - raise Exception( - "The file " + filename + " does not end with 2 new lines." - ) - - # Enforce some rules to reduce the risk of having mismatched title - # styles. - title_line = section.split("\n")[1] - if title_line != (len(title_line) * title_line[0]): - raise Exception( - "The file " + filename + " doesn't have a title style line on line 2" - ) - - # anything marked as xx_00_ is the start of a new top-level section - if re.match("^[0-9]+_00_", filename): - if not title_style_matchers[TOP_LEVEL].match(title_line): - raise Exception( - "The file " + filename + " is a top-level section because it matches " + - "the filename format ##_00_something.rst but has the wrong title " + - "style: expected '" + TOP_LEVEL + "' but got '" + - title_line[0] + "'" - ) - # anything marked as xx_xx_ is the start of a sub-section - elif re.match("^[0-9]+_[0-9]{2}_", filename): - if not title_style_matchers[SECOND_LEVEL].match(title_line): - raise Exception( - "The file " + filename + " is a 2nd-level section because it matches " + - "the filename format ##_##_something.rst but has the wrong title " + - "style: expected '" + SECOND_LEVEL + "' but got '" + - title_line[0] + "' - If this is meant to be a 3rd/4th/5th-level section " + - "then use the form '##_##b_something.rst' which will not apply this " + - "check." - ) - -def check_valid_section(section): - pass +def _list_get(l, index, default=None): + try: + return l[index] + except IndexError: + return default + + +def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): + rst_lines = [] + title_chars = "".join(title_styles) + title_regex = re.compile("^[" + re.escape(title_chars) + "]+$") + + curr_title_level = title_level + for i, line in enumerate(file_stream, 1): + if title_regex.match(line): + line_title_level = title_styles.index(line[0]) + # Allowed to go 1 deeper or any number shallower + if curr_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line[0], title_styles, + title_styles[curr_title_level]) + ) + curr_title_level = line_title_level + rst_lines.append(line) + else: + rst_lines.append(line) + return "".join(rst_lines) + + + +def get_rst(file_info, title_level, title_styles, spec_dir): + # string are file paths to RST blobs + if isinstance(file_info, basestring): + with open(spec_dir + file_info, "r") as f: + return load_with_adjusted_titles(file_info, f, title_level, title_styles) + # dicts look like {1: filepath, 2: filepath} where the key is the title level + elif isinstance(file_info, dict): + levels = sorted(file_info.keys()) + rst = [] + for l in levels: + rst.append(get_rst(file_info[l], l, title_styles, spec_dir)) + return "".join(rst) + # lists are multiple file paths e.g. [filepath, filepath] + elif isinstance(file_info, list): + rst = [] + for f in file_info: + rst.append(get_rst(f, title_level, title_styles, spec_dir)) + return "".join(rst) + raise Exception( + "The following 'file' entry in this target isn't a string, list or dict. " + + "It really really should be. Entry: %s" % (file_info,) + ) -def get_rst(file_info, target): - pass def build_spec(target, out_filename): with open(out_filename, "wb") as outfile: for file_info in target["files"]: - section = get_rst(file_info, target) - check_valid_section(section) + section = get_rst( + file_info=file_info, + title_level=0, + title_styles=target["title_styles"], + spec_dir="../specification/" + ) outfile.write(section) @@ -170,7 +174,6 @@ def cleanup_env(): def main(target_name): prepare_env() target = get_build_target("../specification/targets.yaml", target_name) - print target build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") From cb41adee707a04591c93726f5002855febf2543f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 17:10:23 +0100 Subject: [PATCH 103/223] Fix the swagger host to be "localhost:8008" so that it can be used in a "Try it now" setup against localhost --- api/client-server/v1/rooms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 4902560b3..4494d3577 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -2,7 +2,7 @@ swagger: '2.0' info: title: "Matrix Client-Server v1 Rooms API" version: "1.0.0" -host: example.com:8008 +host: localhost:8008 schemes: - https - http From 615a9575cbad5401997e89c292cc741e0bf5ff23 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 17:12:29 +0100 Subject: [PATCH 104/223] SPEC-216: Clarify when the room getters will return 403 --- api/client-server/v1/rooms.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 4494d3577..3bd23c4a1 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -57,7 +57,9 @@ paths: 404: description: The room has no state with the given type or key. 403: - description: You are not joined to the room. + description: > + You aren't a member of the room and weren't previously a + member of the room. "/rooms/{roomId}/state": get: @@ -170,7 +172,9 @@ paths: allOf: - "$ref": "core-event-schema/state_event.json" 403: - description: You are not joined to the room. + description: > + You aren't a member of the room and weren't previously a + member of the room. "/rooms/{roomId}/initialSync": get: @@ -365,6 +369,10 @@ paths: Whether this room is visible to the ``/publicRooms`` API or not." required: ["room_id", "membership"] + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. "/rooms/{roomId}/members": get: @@ -427,4 +435,8 @@ paths: type: object allOf: - "$ref": "v1-event-schema/m.room.member" + 403: + description: > + You aren't a member of the room and weren't previously a + member of the room. From ba6c7d267ca4b5265d2eb1ba19933fe6f790090a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 17:17:40 +0100 Subject: [PATCH 105/223] SPEC-216: Document the behaviour of the room getters when the user has left the room --- api/client-server/v1/rooms.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 3bd23c4a1..7550c5fe5 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -24,7 +24,8 @@ paths: description: |- Looks up the contents of a state event in a room. If the user is joined to the room then the state is taken from the current - state of the room. + state of the room. If the user has left the room then the state is + taken from the state of the room when they left. security: - accessToken: [] parameters: @@ -165,7 +166,9 @@ paths: title: RoomState description: |- If the user is a member of the room this will be the - current state of the room as a list of events. + current state of the room as a list of events. If the user + has left the room then this will be the state of the room + when they left as a list of events. items: title: StateEvent type: object @@ -378,7 +381,7 @@ paths: get: summary: Get the m.room.member events for the room. description: - Get the list of members of the room. + Get the list of members for this room. parameters: - in: path type: string From 067363c629bcf140e925980d20bcf8b264407cac Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 21 Sep 2015 17:21:08 +0100 Subject: [PATCH 106/223] Get the desired title levels right; print out the actual level used to stdout --- scripts/gendoc.py | 16 ++++++++----- specification/modules/00_modules_intro.rst | 0 specification/targets.yaml | 26 +++++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 specification/modules/00_modules_intro.rst diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2e303ea2a..f7bde1628 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -50,13 +50,13 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): return "".join(rst_lines) - def get_rst(file_info, title_level, title_styles, spec_dir): # string are file paths to RST blobs if isinstance(file_info, basestring): + print "%s %s" % (">" * (1 + title_level), file_info) with open(spec_dir + file_info, "r") as f: return load_with_adjusted_titles(file_info, f, title_level, title_styles) - # dicts look like {1: filepath, 2: filepath} where the key is the title level + # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): levels = sorted(file_info.keys()) rst = [] @@ -144,12 +144,15 @@ def get_build_target(targets_listing, target_name): # copy across the group of files specified group_name = f[len("group:"):] group = all_targets.get("groups", {}).get(group_name) - if not isinstance(group, list): + if not group: raise Exception( - "Tried to find group '" + group_name + "' but either " + - "it doesn't exist or it isn't a list of files." + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." ) - resolved_files.extend(group) + if isinstance(group, list): + resolved_files.extend(group) + else: + resolved_files.append(group) else: resolved_files.append(f) build_target["files"] = resolved_files @@ -173,6 +176,7 @@ def cleanup_env(): def main(target_name): prepare_env() + print "Building spec [target=%s]" % target_name target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/full_spec.rst") run_through_template("tmp/full_spec.rst") diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/00_modules_intro.rst new file mode 100644 index 000000000..e69de29bb diff --git a/specification/targets.yaml b/specification/targets.yaml index c4828ef17..e492ad3c4 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,18 +14,14 @@ targets: - 06_00_appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - # Mark a nested file dependency - - 1: modules/07_00_push_overview.rst - 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] -title_styles: - - "=" - - "-" - - "~" - - "+" - + 0: modules/00_modules_intro.rst + 1: + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - 1: modules/07_00_push_overview.rst # Mark a nested file dependency + 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] +title_styles: ["=", "-", "~", "+"] From f60190086a41579d35a7d42caaaee7e69fe27b00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 17:30:10 +0100 Subject: [PATCH 107/223] Describe the behaviour of /rooms/{roomId}/member when the user has left the room --- api/client-server/v1/rooms.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index 7550c5fe5..e02c75c64 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -391,7 +391,10 @@ paths: x-example: "!room:example.com" responses: 200: - description: The members of the room. + description: |- + A list of members of the room. If you are joined to the room then + this will be the current members of the room. If you have left te + room then this will be the members of the room when you left. examples: application/json: |- { From 98d91d0c2b3b098618bc91f70079a1a9c64bb546 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 17:31:35 +0100 Subject: [PATCH 108/223] Make the example room id more "random" so that people are less likely to think that it is supposed to be human readable --- api/client-server/v1/rooms.yaml | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/api/client-server/v1/rooms.yaml b/api/client-server/v1/rooms.yaml index e02c75c64..9300d7d10 100644 --- a/api/client-server/v1/rooms.yaml +++ b/api/client-server/v1/rooms.yaml @@ -34,7 +34,7 @@ paths: name: roomId description: The room to look up the state in. required: true - x-example: "!room:example.com" + x-example: "!636q39766251:example.com" - in: path type: string name: eventType @@ -75,7 +75,7 @@ paths: name: roomId description: The room to look up the state for. required: true - x-example: "!room:example.com" + x-example: "!636q39766251:example.com" responses: 200: description: The current state of the room @@ -89,7 +89,7 @@ paths: }, "event_id": "$14259997323TLwtb:example.com", "origin_server_ts": 1425999732392, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", "user_id": "@alice:example.com" @@ -104,7 +104,7 @@ paths: "event_id": "$1426600438280zExKY:example.com", "membership": "join", "origin_server_ts": 1426600438277, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", "user_id": "@alice:example.com" @@ -116,7 +116,7 @@ paths: }, "event_id": "$14259997320KhbwJ:example.com", "origin_server_ts": 1425999732089, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", "user_id": "@alice:example.com" @@ -131,7 +131,7 @@ paths: "event_id": "$1431525430134MxlLX:example.com", "origin_server_ts": 1431525430569, "replaces_state": "$142652023736BSXcM:example.com", - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", "user_id": "@bob:example.com" @@ -155,7 +155,7 @@ paths: }, "event_id": "$14259997322mqfaq:example.com", "origin_server_ts": 1425999732285, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", "user_id": "@alice:example.com" @@ -192,7 +192,7 @@ paths: name: roomId description: The room to get the data. required: true - x-example: "!room:example.com" + x-example: "!636q39766251:example.com" responses: 200: description: The current state of the room @@ -210,7 +210,7 @@ paths: }, "event_id": "$14328044851tzTJS:example.com", "origin_server_ts": 1432804485886, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "type": "m.room.message", "user_id": "@alice:example.com" }, @@ -222,7 +222,7 @@ paths: }, "event_id": "$14328044872spjFg:example.com", "origin_server_ts": 1432804487480, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "type": "m.room.message", "user_id": "@bob:example.com" } @@ -230,7 +230,7 @@ paths: "end": "s3456_9_0", "start": "t44-3453_9_0" }, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state": [ { "age": 7148266897, @@ -239,7 +239,7 @@ paths: }, "event_id": "$14259997323TLwtb:example.com", "origin_server_ts": 1425999732392, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.join_rules", "user_id": "@alice:example.com" @@ -254,7 +254,7 @@ paths: "event_id": "$1426600438280zExKY:example.com", "membership": "join", "origin_server_ts": 1426600438277, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", "user_id": "@alice:example.com" @@ -266,7 +266,7 @@ paths: }, "event_id": "$14259997320KhbwJ:example.com", "origin_server_ts": 1425999732089, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.create", "user_id": "@alice:example.com" @@ -281,7 +281,7 @@ paths: "event_id": "$1431525430134MxlLX:example.com", "origin_server_ts": 1431525430569, "replaces_state": "$142652023736BSXcM:example.com", - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", "user_id": "@bob:example.com" @@ -305,7 +305,7 @@ paths: }, "event_id": "$14259997322mqfaq:example.com", "origin_server_ts": 1425999732285, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "", "type": "m.room.power_levels", "user_id": "@alice:example.com" @@ -388,7 +388,7 @@ paths: name: roomId description: The room to get the member events for. required: true - x-example: "!room:example.com" + x-example: "!636q39766251:example.com" responses: 200: description: |- @@ -409,7 +409,7 @@ paths: "event_id": "$1426600438280zExKY:example.com", "membership": "join", "origin_server_ts": 1426600438277, - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@alice:example.com", "type": "m.room.member", "user_id": "@alice:example.com" @@ -424,7 +424,7 @@ paths: "event_id": "$1431525430134MxlLX:example.com", "origin_server_ts": 1431525430569, "replaces_state": "$142652023736BSXcM:example.com", - "room_id": "!room:example.com", + "room_id": "!636q39766251:example.com", "state_key": "@bob:example.com", "type": "m.room.member", "user_id": "@bob:example.com" From f71763b0d3b6d95ed7839f5c9735d21aa59d5d71 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 13:08:15 +0100 Subject: [PATCH 109/223] Implement relative title styles Templates don't know at what level they will be inserted. Previously, we hard-coded the title style which is not compatible with the build target system. Define a set of styles which will be replaced by the gendoc script when it encounters them: '<' : Make this title a sub-heading '/' : Make this title a heading at the same level '>' : Make this title a super-heading The build target system is now basically complete and functioning. --- scripts/gendoc.py | 234 +++++++++++++++--- specification/00_01_feature_profiles.rst | 5 + specification/02_00_modules.rst | 5 + .../03_00_application_service_api.rst | 1 + specification/04_00_server_server_api.rst | 4 +- specification/modules/00_modules_intro.rst | 5 + specification/targets.yaml | 41 +-- templating/matrix_templates/sections.py | 35 +-- .../matrix_templates/templates/msgtypes.tmpl | 2 +- templating/matrix_templates/units.py | 7 + 10 files changed, 268 insertions(+), 71 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index f7bde1628..308d148f7 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -17,57 +17,133 @@ stylesheets = { } -def _list_get(l, index, default=None): - try: - return l[index] - except IndexError: - return default - - +""" +Read a RST file and replace titles with a different title level if required. +Args: + filename: The name of the file being read (for debugging) + file_stream: The open file stream to read from. + title_level: The integer which determines the offset to *start* from. + title_styles: An array of characters detailing the right title styles to use + e.g. ["=", "-", "~", "+"] +Returns: + string: The file contents with titles adjusted. +Example: + Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file + when read line-by-line encounters the titles "===", "---", "---", "===", "---". + This function will bump every title encountered down a sub-heading e.g. + "=" to "-" and "-" to "~" because title_level = 1, so the output would be + "---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level. +""" def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): rst_lines = [] title_chars = "".join(title_styles) - title_regex = re.compile("^[" + re.escape(title_chars) + "]+$") + title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$") - curr_title_level = title_level + prev_line_title_level = 0 # We expect the file to start with '=' titles + file_offset = None + prev_non_title_line = None for i, line in enumerate(file_stream, 1): - if title_regex.match(line): - line_title_level = title_styles.index(line[0]) - # Allowed to go 1 deeper or any number shallower - if curr_title_level - line_title_level < -1: - raise Exception( - ("File '%s' line '%s' has a title " + - "style '%s' which doesn't match one of the " + - "allowed title styles of %s because the " + - "title level before this line was '%s'") % - (filename, (i + 1), line[0], title_styles, - title_styles[curr_title_level]) - ) - curr_title_level = line_title_level + # ignore anything which isn't a title (e.g. '===============') + if not title_regex.match(line): rst_lines.append(line) - else: + prev_non_title_line = line + continue + # The title underline must match at a minimum the length of the title + if len(prev_non_title_line) > len(line): rst_lines.append(line) + prev_non_title_line = line + continue + + line_title_style = line[0] + line_title_level = title_styles.index(line_title_style) + + # Not all files will start with "===" and we should be flexible enough + # to allow that. The first title we encounter sets the "file offset" + # which is added to the title_level desired. + if file_offset is None: + file_offset = line_title_level + if file_offset != 0: + print (" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0]) + + # Sanity checks: Make sure that this file is obeying the title levels + # specified and bail if it isn't. + # The file is allowed to go 1 deeper or any number shallower + if prev_line_title_level - line_title_level < -1: + raise Exception( + ("File '%s' line '%s' has a title " + + "style '%s' which doesn't match one of the " + + "allowed title styles of %s because the " + + "title level before this line was '%s'") % + (filename, (i + 1), line_title_style, title_styles, + title_styles[prev_line_title_level]) + ) + prev_line_title_level = line_title_level + + adjusted_level = ( + title_level + line_title_level - file_offset + ) + + # Sanity check: Make sure we can bump down the title and we aren't at the + # lowest level already + if adjusted_level >= len(title_styles): + raise Exception( + ("Files '%s' line '%s' has a sub-title level too low and it " + + "cannot be adjusted to fit. You can add another level to the " + + "'title_styles' key in targets.yaml to fix this.") % + (filename, (i + 1)) + ) + + if adjusted_level == line_title_level: + # no changes required + rst_lines.append(line) + continue + + # Adjusting line levels + # print ( + # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + # (filename, line_title_style, + # title_styles[adjusted_level], + # file_offset, title_level) + # ) + rst_lines.append(line.replace( + line_title_style, + title_styles[adjusted_level] + )) + return "".join(rst_lines) -def get_rst(file_info, title_level, title_styles, spec_dir): +def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): print "%s %s" % (">" * (1 + title_level), file_info) with open(spec_dir + file_info, "r") as f: - return load_with_adjusted_titles(file_info, f, title_level, title_styles) + rst = None + if adjust_titles: + rst = load_with_adjusted_titles( + file_info, f, title_level, title_styles + ) + else: + rst = f.read() + if rst[-2:] != "\n\n": + raise Exception( + ("File %s should end with TWO new-line characters to ensure " + + "file concatenation works correctly.") % (file_info,) + ) + return rst # dicts look like {0: filepath, 1: filepath} where the key is the title level elif isinstance(file_info, dict): levels = sorted(file_info.keys()) rst = [] for l in levels: - rst.append(get_rst(file_info[l], l, title_styles, spec_dir)) + rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles)) return "".join(rst) # lists are multiple file paths e.g. [filepath, filepath] elif isinstance(file_info, list): rst = [] for f in file_info: - rst.append(get_rst(f, title_level, title_styles, spec_dir)) + rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles)) return "".join(rst) raise Exception( "The following 'file' entry in this target isn't a string, list or dict. " + @@ -82,11 +158,74 @@ def build_spec(target, out_filename): file_info=file_info, title_level=0, title_styles=target["title_styles"], - spec_dir="../specification/" + spec_dir="../specification/", + adjust_titles=True ) outfile.write(section) +""" +Replaces relative title styles with actual title styles. + +The templating system has no idea what the right title style is when it produces +RST because it depends on the build target. As a result, it uses relative title +styles defined in targets.yaml to say "down a level, up a level, same level". + +This function replaces these relative titles with actual title styles from the +array in targets.yaml. +""" +def fix_relative_titles(target, filename, out_filename): + title_styles = target["title_styles"] # ["=", "-", "~", "+"] + relative_title_chars = [ # ["<", "/", ">"] + target["relative_title_styles"]["subtitle"], + target["relative_title_styles"]["sametitle"], + target["relative_title_styles"]["supertitle"] + ] + relative_title_matcher = re.compile( + "^[" + re.escape("".join(relative_title_chars)) + "]{3,}$" + ) + title_matcher = re.compile( + "^[" + re.escape("".join(title_styles)) + "]{3,}$" + ) + current_title_style = None + with open(filename, "r") as infile: + with open(out_filename, "w") as outfile: + for line in infile.readlines(): + if not relative_title_matcher.match(line): + if title_matcher.match(line): + current_title_style = line[0] + outfile.write(line) + continue + line_char = line[0] + replacement_char = None + current_title_level = title_styles.index(current_title_style) + if line_char == target["relative_title_styles"]["subtitle"]: + if (current_title_level + 1) == len(title_styles): + raise Exception( + "Encountered sub-title line style but we can't go " + + "any lower." + ) + replacement_char = title_styles[current_title_level + 1] + elif line_char == target["relative_title_styles"]["sametitle"]: + replacement_char = title_styles[current_title_level] + elif line_char == target["relative_title_styles"]["supertitle"]: + if (current_title_level - 1) < 0: + raise Exception( + "Encountered super-title line style but we can't go " + + "any higher." + ) + replacement_char = title_styles[current_title_level - 1] + else: + raise Exception( + "Unknown relative line char %s" % (line_char,) + ) + + outfile.write( + line.replace(line_char, replacement_char) + ) + + + def rst2html(i, o): with open(i, "r") as in_file: with open(o, "w") as out_file: @@ -123,11 +262,13 @@ def run_through_template(input): def get_build_target(targets_listing, target_name): build_target = { "title_styles": [], + "relative_title_styles": {}, "files": [] } with open(targets_listing, "r") as targ_file: all_targets = yaml.load(targ_file.read()) build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] target = all_targets["targets"].get(target_name) if not target: raise Exception( @@ -138,17 +279,30 @@ def get_build_target(targets_listing, target_name): raise Exception( "Found target but 'files' key is not a list." ) + + def get_group(group_id): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: + raise Exception( + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." + ) + return group + resolved_files = [] for f in target["files"]: + group = None if isinstance(f, basestring) and f.startswith("group:"): - # copy across the group of files specified - group_name = f[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." - ) + group = get_group(f) + elif isinstance(f, dict): + for (k, v) in f.iteritems(): + if isinstance(v, basestring) and v.startswith("group:"): + f[k] = get_group(v) + resolved_files.append(f) + continue + + if group: if isinstance(group, list): resolved_files.extend(group) else: @@ -178,8 +332,12 @@ def main(target_name): prepare_env() print "Building spec [target=%s]" % target_name target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/full_spec.rst") - run_through_template("tmp/full_spec.rst") + build_spec(target=target, out_filename="tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst") + fix_relative_titles( + target=target, filename="tmp/templated_spec.rst", + out_filename="tmp/full_spec.rst" + ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") diff --git a/specification/00_01_feature_profiles.rst b/specification/00_01_feature_profiles.rst index e69de29bb..155e51c58 100644 --- a/specification/00_01_feature_profiles.rst +++ b/specification/00_01_feature_profiles.rst @@ -0,0 +1,5 @@ +Feature Profiles +================ + +Feature profiles blurb goes here. + diff --git a/specification/02_00_modules.rst b/specification/02_00_modules.rst index e69de29bb..ab35fe9f1 100644 --- a/specification/02_00_modules.rst +++ b/specification/02_00_modules.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules intro here. + diff --git a/specification/03_00_application_service_api.rst b/specification/03_00_application_service_api.rst index 2674ba443..e982390b4 100644 --- a/specification/03_00_application_service_api.rst +++ b/specification/03_00_application_service_api.rst @@ -400,3 +400,4 @@ in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. For instance, this could contain the message-ID for emails/nntp posts, or a link to a blog comment when gatewaying blog comment traffic in & out of matrix + diff --git a/specification/04_00_server_server_api.rst b/specification/04_00_server_server_api.rst index f5cadabff..8d1f8898f 100644 --- a/specification/04_00_server_server_api.rst +++ b/specification/04_00_server_server_api.rst @@ -92,7 +92,7 @@ server by querying other servers. .. _Perspectives Project: http://perspectives-project.org/ Publishing Keys -_______________ +^^^^^^^^^^^^^^^ Home servers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of @@ -178,7 +178,7 @@ events sent by that server can still be checked. } Querying Keys Through Another Server -____________________________________ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys for another server. This API can be used to GET at list of JSON objects for a diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/00_modules_intro.rst index e69de29bb..fdfcbbdc8 100644 --- a/specification/modules/00_modules_intro.rst +++ b/specification/modules/00_modules_intro.rst @@ -0,0 +1,5 @@ +Modules +======= + +Modules blurb goes here. + diff --git a/specification/targets.yaml b/specification/targets.yaml index e492ad3c4..7bd3d882d 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,26 +2,37 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - 00_00_intro.rst - - 00_01_feature_profiles.rst - - 00_02a_events.rst - - 00_02b_event_signing.rst + - { 1: 00_01_feature_profiles.rst } + - { 1: 00_02a_events.rst } + - { 1: 00_02b_event_signing.rst } - 01_00_client_server_api.rst - 02_00_modules.rst - - "group:modules" # reference a group of files + - { 1: "group:modules" } # reference a group of files - 03_00_application_service_api.rst - 04_00_server_server_api.rst - 05_00_identity_servers.rst - 06_00_appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - 0: modules/00_modules_intro.rst - 1: - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - - 1: modules/07_00_push_overview.rst # Mark a nested file dependency - 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] -title_styles: ["=", "-", "~", "+"] + - modules/00_modules_intro.rst + - modules/01_00_voip_events.rst + - modules/02_00_typing_notifications.rst + - modules/03_00_receipts.rst + - modules/04_00_content_repo.rst + - modules/05_00_end_to_end_encryption.rst + - modules/06_00_history_visibility.rst + - modules/07_00_push_overview.rst + - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] } + +title_styles: ["=", "-", "~", "+", "^"] + +# The templating system doesn't know the right title style to use when generating +# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title +# at the same level (0)", or "make a title one above (+1)". The gendoc script +# will inspect this file and replace these relative styles with actual title +# styles. The templating system will also inspect this file to know which symbols +# to inject. +relative_title_styles: + subtitle: "<" + sametitle: "/" + supertitle: ">" diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 729157bb9..6072222ac 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -23,10 +23,13 @@ class MatrixSections(Sections): spec_meta = self.units.get("spec_meta") return spec_meta["changelog"] - def _render_events(self, filterFn, sortFn, title_kind="~"): + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] for event_name in sortFn(schemas): if not filterFn(event_name): @@ -34,14 +37,16 @@ class MatrixSections(Sections): sections.append(template.render( example=examples[event_name], event=schemas[event_name], - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) - def _render_http_api_group(self, group, sortFnOrPathList=None, - title_kind="-"): + def _render_http_api_group(self, group, sortFnOrPathList=None): template = self.env.get_template("http-api.tmpl") http_api = self.units.get("swagger_apis")[group]["__meta"] + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] endpoints = [] if sortFnOrPathList: @@ -67,15 +72,14 @@ class MatrixSections(Sections): for endpoint in endpoints: sections.append(template.render( endpoint=endpoint, - title_kind=title_kind + title_kind=subtitle_title_char )) return "\n\n".join(sections) def render_profile_http_api(self): return self._render_http_api_group( "profile", - sortFnOrPathList=["displayname", "avatar_url"], - title_kind="~" + sortFnOrPathList=["displayname", "avatar_url"] ) def render_sync_http_api(self): @@ -86,20 +90,17 @@ class MatrixSections(Sections): def render_presence_http_api(self): return self._render_http_api_group( "presence", - sortFnOrPathList=["status"], - title_kind="~" + sortFnOrPathList=["status"] ) def render_membership_http_api(self): return self._render_http_api_group( - "membership", - title_kind="~" + "membership" ) def render_login_http_api(self): return self._render_http_api_group( - "login", - title_kind="~" + "login" ) def render_room_events(self): @@ -114,6 +115,9 @@ class MatrixSections(Sections): template = self.env.get_template("msgtypes.tmpl") examples = self.units.get("event_examples") schemas = self.units.get("event_schemas") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] sections = [] msgtype_order = [ "m.room.message#m.text", "m.room.message#m.emote", @@ -129,7 +133,8 @@ class MatrixSections(Sections): continue sections.append(template.render( example=examples[event_name], - event=schemas[event_name] + event=schemas[event_name], + title_kind=subtitle_title_char )) return "\n\n".join(sections) @@ -150,7 +155,7 @@ class MatrixSections(Sections): def render_presence_events(self): def filterFn(eventType): return eventType.startswith("m.presence") - return self._render_events(filterFn, sorted, title_kind="+") + return self._render_events(filterFn, sorted) def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") diff --git a/templating/matrix_templates/templates/msgtypes.tmpl b/templating/matrix_templates/templates/msgtypes.tmpl index 29e86160d..f78624517 100644 --- a/templating/matrix_templates/templates/msgtypes.tmpl +++ b/templating/matrix_templates/templates/msgtypes.tmpl @@ -1,5 +1,5 @@ ``{{event.msgtype}}`` -{{(4 + event.msgtype | length) * '+'}} +{{(4 + event.msgtype | length) * title_kind}} {{event.desc | wrap(80)}} {% for table in event.content_fields -%} {{"``"+table.title+"``" if table.title else "" }} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 1a9d981ae..0096bbfa1 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" +TARGETS = "../specification/targets.yaml" ROOM_EVENT = "core-event-schema/room_event.json" STATE_EVENT = "core-event-schema/state_event.json" @@ -466,6 +467,12 @@ class MatrixUnits(Units): "changelog": "".join(changelog_lines) } + + def load_spec_targets(self): + with open(TARGETS, "r") as f: + return yaml.load(f.read()) + + def load_git_version(self): null = open(os.devnull, 'w') cwd = os.path.dirname(os.path.abspath(__file__)) From e8cdfcbba28223055a182e06c1ee1a4f04cc3738 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 14:01:55 +0100 Subject: [PATCH 110/223] Rename spec files to not have numbers everywhere; update targets.yaml --- ..._event_signing.rst => 0-event_signing.rst} | 0 .../{00_02a_events.rst => 0-events.rst} | 0 ...re_profiles.rst => 0-feature_profiles.rst} | 0 .../{00_00_intro.rst => 0-intro.rst} | 0 ...server_api.rst => 1-client_server_api.rst} | 0 .../{02_00_modules.rst => 2-modules.rst} | 0 ..._api.rst => 3-application_service_api.rst} | 0 ...server_api.rst => 4-server_server_api.rst} | 0 ...ity_servers.rst => 5-identity_servers.rst} | 0 ...{06_00_appendices.rst => 6-appendices.rst} | 0 ...4_00_content_repo.rst => content_repo.rst} | 0 ...cryption.rst => end_to_end_encryption.rst} | 0 ..._visibility.rst => history_visibility.rst} | 0 .../{00_modules_intro.rst => intro.rst} | 0 ...{07_01_push_cs_api.rst => push_cs_api.rst} | 0 ...00_push_overview.rst => push_overview.rst} | 0 ...h_push_gw_api.rst => push_push_gw_api.rst} | 0 .../{03_00_receipts.rst => receipts.rst} | 0 ...fications.rst => typing_notifications.rst} | 0 ...{01_00_voip_events.rst => voip_events.rst} | 0 specification/targets.yaml | 38 +++++++++---------- 21 files changed, 19 insertions(+), 19 deletions(-) rename specification/{00_02b_event_signing.rst => 0-event_signing.rst} (100%) rename specification/{00_02a_events.rst => 0-events.rst} (100%) rename specification/{00_01_feature_profiles.rst => 0-feature_profiles.rst} (100%) rename specification/{00_00_intro.rst => 0-intro.rst} (100%) rename specification/{01_00_client_server_api.rst => 1-client_server_api.rst} (100%) rename specification/{02_00_modules.rst => 2-modules.rst} (100%) rename specification/{03_00_application_service_api.rst => 3-application_service_api.rst} (100%) rename specification/{04_00_server_server_api.rst => 4-server_server_api.rst} (100%) rename specification/{05_00_identity_servers.rst => 5-identity_servers.rst} (100%) rename specification/{06_00_appendices.rst => 6-appendices.rst} (100%) rename specification/modules/{04_00_content_repo.rst => content_repo.rst} (100%) rename specification/modules/{05_00_end_to_end_encryption.rst => end_to_end_encryption.rst} (100%) rename specification/modules/{06_00_history_visibility.rst => history_visibility.rst} (100%) rename specification/modules/{00_modules_intro.rst => intro.rst} (100%) rename specification/modules/{07_01_push_cs_api.rst => push_cs_api.rst} (100%) rename specification/modules/{07_00_push_overview.rst => push_overview.rst} (100%) rename specification/modules/{07_02_push_push_gw_api.rst => push_push_gw_api.rst} (100%) rename specification/modules/{03_00_receipts.rst => receipts.rst} (100%) rename specification/modules/{02_00_typing_notifications.rst => typing_notifications.rst} (100%) rename specification/modules/{01_00_voip_events.rst => voip_events.rst} (100%) diff --git a/specification/00_02b_event_signing.rst b/specification/0-event_signing.rst similarity index 100% rename from specification/00_02b_event_signing.rst rename to specification/0-event_signing.rst diff --git a/specification/00_02a_events.rst b/specification/0-events.rst similarity index 100% rename from specification/00_02a_events.rst rename to specification/0-events.rst diff --git a/specification/00_01_feature_profiles.rst b/specification/0-feature_profiles.rst similarity index 100% rename from specification/00_01_feature_profiles.rst rename to specification/0-feature_profiles.rst diff --git a/specification/00_00_intro.rst b/specification/0-intro.rst similarity index 100% rename from specification/00_00_intro.rst rename to specification/0-intro.rst diff --git a/specification/01_00_client_server_api.rst b/specification/1-client_server_api.rst similarity index 100% rename from specification/01_00_client_server_api.rst rename to specification/1-client_server_api.rst diff --git a/specification/02_00_modules.rst b/specification/2-modules.rst similarity index 100% rename from specification/02_00_modules.rst rename to specification/2-modules.rst diff --git a/specification/03_00_application_service_api.rst b/specification/3-application_service_api.rst similarity index 100% rename from specification/03_00_application_service_api.rst rename to specification/3-application_service_api.rst diff --git a/specification/04_00_server_server_api.rst b/specification/4-server_server_api.rst similarity index 100% rename from specification/04_00_server_server_api.rst rename to specification/4-server_server_api.rst diff --git a/specification/05_00_identity_servers.rst b/specification/5-identity_servers.rst similarity index 100% rename from specification/05_00_identity_servers.rst rename to specification/5-identity_servers.rst diff --git a/specification/06_00_appendices.rst b/specification/6-appendices.rst similarity index 100% rename from specification/06_00_appendices.rst rename to specification/6-appendices.rst diff --git a/specification/modules/04_00_content_repo.rst b/specification/modules/content_repo.rst similarity index 100% rename from specification/modules/04_00_content_repo.rst rename to specification/modules/content_repo.rst diff --git a/specification/modules/05_00_end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst similarity index 100% rename from specification/modules/05_00_end_to_end_encryption.rst rename to specification/modules/end_to_end_encryption.rst diff --git a/specification/modules/06_00_history_visibility.rst b/specification/modules/history_visibility.rst similarity index 100% rename from specification/modules/06_00_history_visibility.rst rename to specification/modules/history_visibility.rst diff --git a/specification/modules/00_modules_intro.rst b/specification/modules/intro.rst similarity index 100% rename from specification/modules/00_modules_intro.rst rename to specification/modules/intro.rst diff --git a/specification/modules/07_01_push_cs_api.rst b/specification/modules/push_cs_api.rst similarity index 100% rename from specification/modules/07_01_push_cs_api.rst rename to specification/modules/push_cs_api.rst diff --git a/specification/modules/07_00_push_overview.rst b/specification/modules/push_overview.rst similarity index 100% rename from specification/modules/07_00_push_overview.rst rename to specification/modules/push_overview.rst diff --git a/specification/modules/07_02_push_push_gw_api.rst b/specification/modules/push_push_gw_api.rst similarity index 100% rename from specification/modules/07_02_push_push_gw_api.rst rename to specification/modules/push_push_gw_api.rst diff --git a/specification/modules/03_00_receipts.rst b/specification/modules/receipts.rst similarity index 100% rename from specification/modules/03_00_receipts.rst rename to specification/modules/receipts.rst diff --git a/specification/modules/02_00_typing_notifications.rst b/specification/modules/typing_notifications.rst similarity index 100% rename from specification/modules/02_00_typing_notifications.rst rename to specification/modules/typing_notifications.rst diff --git a/specification/modules/01_00_voip_events.rst b/specification/modules/voip_events.rst similarity index 100% rename from specification/modules/01_00_voip_events.rst rename to specification/modules/voip_events.rst diff --git a/specification/targets.yaml b/specification/targets.yaml index 7bd3d882d..df61e7a14 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,28 +1,28 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - - 00_00_intro.rst - - { 1: 00_01_feature_profiles.rst } - - { 1: 00_02a_events.rst } - - { 1: 00_02b_event_signing.rst } - - 01_00_client_server_api.rst - - 02_00_modules.rst + - 0-intro.rst + - { 1: 0-feature_profiles.rst } + - { 1: 0-events.rst } + - { 1: 0-event_signing.rst } + - 1-client_server_api.rst + - 2-modules.rst - { 1: "group:modules" } # reference a group of files - - 03_00_application_service_api.rst - - 04_00_server_server_api.rst - - 05_00_identity_servers.rst - - 06_00_appendices.rst + - 3-application_service_api.rst + - 4-server_server_api.rst + - 5-identity_servers.rst + - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/00_modules_intro.rst - - modules/01_00_voip_events.rst - - modules/02_00_typing_notifications.rst - - modules/03_00_receipts.rst - - modules/04_00_content_repo.rst - - modules/05_00_end_to_end_encryption.rst - - modules/06_00_history_visibility.rst - - modules/07_00_push_overview.rst - - { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] } + - modules/intro.rst + - modules/voip_events.rst + - modules/typing_notifications.rst + - modules/receipts.rst + - modules/content_repo.rst + - modules/end_to_end_encryption.rst + - modules/history_visibility.rst + - modules/push_overview.rst + - { 2: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } title_styles: ["=", "-", "~", "+", "^"] From 0c0ac3e814c763e271d7a0c9b5c9b5e824b2f2f3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 14:07:23 +0100 Subject: [PATCH 111/223] Fix common event fields template to use subtitle char --- specification/modules/intro.rst | 5 ----- specification/targets.yaml | 1 - templating/matrix_templates/sections.py | 7 ++++++- .../matrix_templates/templates/common-event-fields.tmpl | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 specification/modules/intro.rst diff --git a/specification/modules/intro.rst b/specification/modules/intro.rst deleted file mode 100644 index fdfcbbdc8..000000000 --- a/specification/modules/intro.rst +++ /dev/null @@ -1,5 +0,0 @@ -Modules -======= - -Modules blurb goes here. - diff --git a/specification/targets.yaml b/specification/targets.yaml index df61e7a14..96527daee 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,7 +14,6 @@ targets: - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - - modules/intro.rst - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 6072222ac..78f011b92 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -160,7 +160,12 @@ class MatrixSections(Sections): def _render_ce_type(self, type): template = self.env.get_template("common-event-fields.tmpl") ce_types = self.units.get("common_event_fields") - return template.render(common_event=ce_types[type]) + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + return template.render( + common_event=ce_types[type], title_kind=subtitle_title_char + ) def render_common_event_fields(self): return self._render_ce_type("event") diff --git a/templating/matrix_templates/templates/common-event-fields.tmpl b/templating/matrix_templates/templates/common-event-fields.tmpl index 2a3f7ec5d..3f16be3da 100644 --- a/templating/matrix_templates/templates/common-event-fields.tmpl +++ b/templating/matrix_templates/templates/common-event-fields.tmpl @@ -1,5 +1,5 @@ {{common_event.title}} Fields -{{(7 + common_event.title | length) * '-'}} +{{(7 + common_event.title | length) * title_kind}} {{common_event.desc | wrap(80)}} From 056b5eba22e94b613c50b6255b23152553bced2e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:01:22 +0100 Subject: [PATCH 112/223] Partially handle representing top-level array responses If an HTTP API returned a top-level array response, the templating system would fail to create a table for it. This is now partially fixed by pulling out the type of the elements (no recursion is done to populate nested tables) --- templating/matrix_templates/units.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 84c96ce34..975f7c804 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -247,7 +247,8 @@ class MatrixUnits(Units): "rows": [{ "key": good_response["schema"].get("name", ""), "type": res_type, - "desc": res.get("description", "") + "desc": res.get("description", ""), + "req_str": "" }] }) elif res_type and Units.prop(good_response, "schema/properties"): @@ -257,6 +258,24 @@ class MatrixUnits(Units): for table in res_tables: if "no-table" not in table: endpoint["res_tables"].append(table) + elif res_type and Units.prop(good_response, "schema/items"): + # response is an array: + # FIXME: Doesn't recurse at all. + schema = good_response["schema"] + array_type = Units.prop(schema, "items/type") + if Units.prop(schema, "items/allOf"): + array_type = ( + Units.prop(schema, "items/title") + ) + endpoint["res_tables"].append({ + "title": schema.get("title", ""), + "rows": [{ + "key": "N/A", + "type": ("[%s]" % array_type), + "desc": schema.get("description", ""), + "req_str": "" + }] + }) endpoints.append(endpoint) From 388aeefac000d89e9b775e1fb7905b8e8f178ae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:07:19 +0100 Subject: [PATCH 113/223] Remove obsolete key --- templating/matrix_templates/sections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index 15dca9b13..cacaa06a8 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -105,8 +105,7 @@ class MatrixSections(Sections): def render_rooms_http_api(self): return self._render_http_api_group( - "rooms", - title_kind="+" + "rooms" ) def render_room_events(self): From b21859836d1b6ca0df6f4072c393211706eae9a8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:11:16 +0100 Subject: [PATCH 114/223] Print stdout of build.py in verbose mode --- scripts/gendoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 308d148f7..6c95e33ce 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -243,9 +243,9 @@ def run_through_template(input): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: - subprocess.check_output( + print subprocess.check_output( [ - 'python', 'build.py', + 'python', 'build.py', "-v", "-i", "matrix_templates", "-o", "../scripts/tmp", "../scripts/"+input From 16693a644aead93c67ad1f10d3afb1d7fc0169a5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 16:59:49 +0100 Subject: [PATCH 115/223] Various review fixes --- scripts/gendoc.py | 105 ++++++++++++++++----------- specification/0-feature_profiles.rst | 2 - specification/2-modules.rst | 2 - 3 files changed, 62 insertions(+), 47 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 6c95e33ce..26ee5a483 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -118,7 +118,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): print "%s %s" % (">" * (1 + title_level), file_info) - with open(spec_dir + file_info, "r") as f: + with open(os.path.join(spec_dir, file_info), "r") as f: rst = None if adjust_titles: rst = load_with_adjusted_titles( @@ -175,8 +175,8 @@ This function replaces these relative titles with actual title styles from the array in targets.yaml. """ def fix_relative_titles(target, filename, out_filename): - title_styles = target["title_styles"] # ["=", "-", "~", "+"] - relative_title_chars = [ # ["<", "/", ">"] + title_styles = target["title_styles"] + relative_title_chars = [ target["relative_title_styles"]["subtitle"], target["relative_title_styles"]["sametitle"], target["relative_title_styles"]["supertitle"] @@ -259,6 +259,17 @@ def run_through_template(input): raise +""" +Extract and resolve groups for the given target in the given targets listing. +Args: + targets_listing (str): The path to a YAML file containing a list of targets + target_name (str): The name of the target to extract from the listings. +Returns: + dict: Containing "filees" (a list of file paths), "relative_title_styles" + (a dict of relative style keyword to title character) and "title_styles" + (a list of characters which represent the global title style to follow, + with the top section title first, the second section second, and so on.) +""" def get_build_target(targets_listing, target_name): build_target = { "title_styles": [], @@ -267,49 +278,57 @@ def get_build_target(targets_listing, target_name): } with open(targets_listing, "r") as targ_file: all_targets = yaml.load(targ_file.read()) - build_target["title_styles"] = all_targets["title_styles"] - build_target["relative_title_styles"] = all_targets["relative_title_styles"] - target = all_targets["targets"].get(target_name) - if not target: - raise Exception( - "No target by the name '" + target_name + "' exists in '" + - targets_listing + "'." - ) - if not isinstance(target.get("files"), list): + + build_target["title_styles"] = all_targets["title_styles"] + build_target["relative_title_styles"] = all_targets["relative_title_styles"] + target = all_targets["targets"].get(target_name) + if not target: + raise Exception( + "No target by the name '" + target_name + "' exists in '" + + targets_listing + "'." + ) + if not isinstance(target.get("files"), list): + raise Exception( + "Found target but 'files' key is not a list." + ) + + def get_group(group_id): + group_name = group_id[len("group:"):] + group = all_targets.get("groups", {}).get(group_name) + if not group: raise Exception( - "Found target but 'files' key is not a list." + "Tried to find group '" + group_name + "' but it " + + "doesn't exist." ) - - def get_group(group_id): - group_name = group_id[len("group:"):] - group = all_targets.get("groups", {}).get(group_name) - if not group: - raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." - ) - return group - - resolved_files = [] - for f in target["files"]: - group = None - if isinstance(f, basestring) and f.startswith("group:"): - group = get_group(f) - elif isinstance(f, dict): - for (k, v) in f.iteritems(): - if isinstance(v, basestring) and v.startswith("group:"): - f[k] = get_group(v) - resolved_files.append(f) - continue - - if group: - if isinstance(group, list): - resolved_files.extend(group) - else: - resolved_files.append(group) + return group + + resolved_files = [] + for file_entry in target["files"]: + # file_entry is a group id + if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + group = get_group(file_entry) + # The group may be resolved to a list of file entries, in which case + # we want to extend the array to insert each of them rather than + # insert the entire list as a single element (which is what append does) + if isinstance(group, list): + resolved_files.extend(group) else: - resolved_files.append(f) - build_target["files"] = resolved_files + resolved_files.append(group) + # file_entry is a dict which has more file entries as values + elif isinstance(file_entry, dict): + resolved_entry = {} + for (k, v) in file_entry.iteritems(): + if isinstance(v, basestring) and v.startswith("group:"): + resolved_entry[k] = get_group(v) + else: + # map across without editing (e.g. normal file path) + resolved_entry[k] = v + resolved_files.append(resolved_entry) + continue + # file_entry is just a plain ol' file path + else: + resolved_files.append(file_entry) + build_target["files"] = resolved_files return build_target diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 155e51c58..234e14db0 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -1,5 +1,3 @@ Feature Profiles ================ -Feature profiles blurb goes here. - diff --git a/specification/2-modules.rst b/specification/2-modules.rst index ab35fe9f1..0aad77e1c 100644 --- a/specification/2-modules.rst +++ b/specification/2-modules.rst @@ -1,5 +1,3 @@ Modules ======= -Modules intro here. - From b49cb57fe596eb942759f67a054906386854867a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 22 Sep 2015 17:32:31 +0100 Subject: [PATCH 116/223] Move events sections to CS API --- specification/targets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/targets.yaml b/specification/targets.yaml index 96527daee..92ad9bb4c 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -3,9 +3,9 @@ targets: files: # the sort order of files to cat - 0-intro.rst - { 1: 0-feature_profiles.rst } + - 1-client_server_api.rst - { 1: 0-events.rst } - { 1: 0-event_signing.rst } - - 1-client_server_api.rst - 2-modules.rst - { 1: "group:modules" } # reference a group of files - 3-application_service_api.rst From 5b134119bd53200f00da3b52557ff88b739f8ad5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 09:59:54 +0100 Subject: [PATCH 117/223] Add presence module; fix relative title bug If a relative title appeared after an HTTP API table, it would insert the wrong level because it thought that part of the table was a title. --- specification/1-client_server_api.rst | 6 -- specification/modules/presence.rst | 60 +++++++++++++++++++ specification/targets.yaml | 1 + .../matrix_templates/templates/http-api.tmpl | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 specification/modules/presence.rst diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index ad39dafc0..93e3cb907 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1054,12 +1054,6 @@ medium address The textual address of the 3pid, eg. the email address -Presence --------- -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them - -{{presence_http_api}} Profiles -------- diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst new file mode 100644 index 000000000..6f6448797 --- /dev/null +++ b/specification/modules/presence.rst @@ -0,0 +1,60 @@ +Presence Events +=============== + +{{presence_events}} + +Each user has the concept of presence information. This encodes the +"availability" of that user, suitable for display on other user's clients. +This is transmitted as an ``m.presence`` event and is one of the few events +which are sent *outside the context of a room*. The basic piece of presence +information is represented by the ``presence`` key, which is an enum of one +of the following: + + - ``online`` : The default state when the user is connected to an event + stream. + - ``unavailable`` : The user is not reachable at this time. + - ``offline`` : The user is not connected to an event stream. + - ``free_for_chat`` : The user is generally willing to receive messages + moreso than default. + - ``hidden`` : Behaves as offline, but allows the user to see the client + state anyway and generally interact with client features. (Not yet + implemented in synapse). + +In addition, the server maintains a timestamp of the last time it saw a +pro-active event from the user; either sending a message to a room, or +changing presence state from a lower to a higher level of availability +(thus: changing state from ``unavailable`` to ``online`` counts as a +proactive event, whereas in the other direction it will not). This timestamp +is presented via a key called ``last_active_ago``, which gives the relative +number of milliseconds since the message is generated/emitted that the user +was last seen active. + +Presence HTTP API +----------------- +.. TODO-spec + - Define how users receive presence invites, and how they accept/decline them + +{{presence_http_api}} + + +Events on Change of Profile Information +--------------------------------------- +Because the profile displayname and avatar information are likely to be used in +many places of a client's display, changes to these fields cause an automatic +propagation event to occur, informing likely-interested parties of the new +values. This change is conveyed using two separate mechanisms: + + - a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. + - a ``m.presence`` presence status update is sent, again containing the new values of the + ``displayname`` and ``avatar_url`` keys, in addition to the required + ``presence`` key containing the current presence state of the user. + +Both of these should be done automatically by the home server when a user +successfully changes their displayname or avatar URL fields. + +Additionally, when home servers emit room membership events for their own +users, they should include the displayname and avatar URL fields in these +events so that clients already have these details to hand, and do not have to +perform extra roundtrips to query it. + diff --git a/specification/targets.yaml b/specification/targets.yaml index 92ad9bb4c..454f23b97 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -17,6 +17,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst + - modules/presence.rst - modules/content_repo.rst - modules/end_to_end_encryption.rst - modules/history_visibility.rst diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index a03ffa7d2..eb3f3e64e 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -18,7 +18,7 @@ Request format: ================== ================= =========================================== {% for loc in endpoint.req_param_by_loc -%} *{{loc}} parameters* --------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- {% for param in endpoint.req_param_by_loc[loc] -%} {{param.key}}{{param.type|indent(19-param.key|length)}}{{param.desc|indent(18-param.type|length)|wrap(43)|indent_block(37)}} {% endfor -%} From 51153462971d3c6ffd4fae4672504e857532c5c2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 10:48:49 +0100 Subject: [PATCH 118/223] Add instant_messaging module; modify batesian section rules Previously, all `m.room.*` events were wodged into `{{room_events}}` which isn't great when you want to pull specific ones out. Batesian had a 1:1 mapping of `render_foo()` to a section `{{foo}}`, and having to constantly add functions for new types is a PITA. Batesian now supports returning a `dict` instead of a section `string` where the keys are the `{{foo}}` and the value is what will be inserted. Also add conflicting section key checks to avoid multiple definitions of the same `{{foo}}`. Define dicts for event schemata and swagger HTTP APIs. Using this new feature, split out the instant messaging stuff from the events section, and replace `{{room_events}}` with a list of specific events e.g. `{{m_room_member_event}}`. --- specification/0-events.rst | 82 ++++----------------- specification/modules/instant_messaging.rst | 27 +++++++ specification/modules/presence.rst | 11 ++- specification/targets.yaml | 1 + templating/batesian/sections.py | 38 ++++++++-- templating/matrix_templates/sections.py | 58 +++++++-------- 6 files changed, 111 insertions(+), 106 deletions(-) create mode 100644 specification/modules/instant_messaging.rst diff --git a/specification/0-events.rst b/specification/0-events.rst index ce36b040a..a50d01fae 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -20,71 +20,19 @@ Room Events This specification outlines several standard event types, all of which are prefixed with ``m.`` -{{room_events}} - -m.room.message msgtypes -~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO-spec - How a client should handle unknown message types. - - -Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type -of message being sent. Each type has their own required and optional keys, as -outlined below. - -{{msgtype_events}} - -Presence Events -~~~~~~~~~~~~~~~ - -{{presence_events}} - -Each user has the concept of presence information. This encodes the -"availability" of that user, suitable for display on other user's clients. -This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. The basic piece of presence -information is represented by the ``presence`` key, which is an enum of one -of the following: - - - ``online`` : The default state when the user is connected to an event - stream. - - ``unavailable`` : The user is not reachable at this time. - - ``offline`` : The user is not connected to an event stream. - - ``free_for_chat`` : The user is generally willing to receive messages - moreso than default. - - ``hidden`` : Behaves as offline, but allows the user to see the client - state anyway and generally interact with client features. (Not yet - implemented in synapse). - -In addition, the server maintains a timestamp of the last time it saw a -pro-active event from the user; either sending a message to a room, or -changing presence state from a lower to a higher level of availability -(thus: changing state from ``unavailable`` to ``online`` counts as a -proactive event, whereas in the other direction it will not). This timestamp -is presented via a key called ``last_active_ago``, which gives the relative -number of milliseconds since the message is generated/emitted that the user -was last seen active. - - -Events on Change of Profile Information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because the profile displayname and avatar information are likely to be used in -many places of a client's display, changes to these fields cause an automatic -propagation event to occur, informing likely-interested parties of the new -values. This change is conveyed using two separate mechanisms: - - - a ``m.room.member`` event is sent to every room the user is a member of, - to update the ``displayname`` and ``avatar_url``. - - a ``m.presence`` presence status update is sent, again containing the new values of the - ``displayname`` and ``avatar_url`` keys, in addition to the required - ``presence`` key containing the current presence state of the user. - -Both of these should be done automatically by the home server when a user -successfully changes their displayname or avatar URL fields. - -Additionally, when home servers emit room membership events for their own -users, they should include the displayname and avatar URL fields in these -events so that clients already have these details to hand, and do not have to -perform extra roundtrips to query it. +{{m_room_aliases_event}} + +{{m_room_canonical_aliases_event}} + +{{m_room_create_event}} + +{{m_room_history_visibility_event}} + +{{m_room_join_rules_event}} + +{{m_room_member_event}} + +{{m_room_power_levels_event}} + +{{m_room_redaction_event}} diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst new file mode 100644 index 000000000..7f582ca4b --- /dev/null +++ b/specification/modules/instant_messaging.rst @@ -0,0 +1,27 @@ +Instant Messaging +================= + +Events +------ + +{{m_room_message_event}} + +{{m_room_message_feedback_event}} + +{{m_room_name_event}} + +{{m_room_topic_event}} + +m.room.message msgtypes +----------------------- + +.. TODO-spec + How a client should handle unknown message types. + + +Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type +of message being sent. Each type has their own required and optional keys, as +outlined below. + +{{msgtype_events}} + diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 6f6448797..ddd2adff2 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,7 +1,5 @@ -Presence Events -=============== - -{{presence_events}} +Presence +======== Each user has the concept of presence information. This encodes the "availability" of that user, suitable for display on other user's clients. @@ -29,6 +27,11 @@ is presented via a key called ``last_active_ago``, which gives the relative number of milliseconds since the message is generated/emitted that the user was last seen active. +Events +------ + +{{presence_events}} + Presence HTTP API ----------------- .. TODO-spec diff --git a/specification/targets.yaml b/specification/targets.yaml index 454f23b97..e33bc7851 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -14,6 +14,7 @@ targets: - 6-appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: + - modules/instant_messaging.rst - modules/voip_events.rst - modules/typing_notifications.rst - modules/receipts.rst diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 11c34fb3a..318493898 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -27,12 +27,38 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if not isinstance(section, basestring): + if isinstance(section, basestring): + if section_key in section_dict: + raise Exception( + ("%s : Section %s already exists. It must have been " + + "generated dynamically. Check which render_ methods " + + "return a dict.") % + (func_name, section_key) + ) + section_dict[section_key] = section + self.log( + " Generated. Snippet => %s" % section[:60].replace("\n","") + ) + elif isinstance(section, dict): + self.log(" Generated multiple sections:") + for (k, v) in section.iteritems(): + if not isinstance(k, basestring) or not isinstance(v, basestring): + raise Exception( + ("Method %s returned multiple sections as a dict but " + + "expected the dict elements to be strings but they aren't.") % + (func_name, ) + ) + if k in section_dict: + raise Exception( + "%s tried to produce section %s which already exists." % + (func_name, k) + ) + section_dict[k] = v + self.log( + " %s => %s" % (k, v[:60].replace("\n","")) + ) + else: raise Exception( - "Section function '%s' didn't return a string!" % func_name + "Section function '%s' didn't return a string/dict!" % func_name ) - section_dict[section_key] = section - self.log( - " Generated. Snippet => %s" % section[:60].replace("\n","") - ) return section_dict \ No newline at end of file diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py index cacaa06a8..e75a75af8 100644 --- a/templating/matrix_templates/sections.py +++ b/templating/matrix_templates/sections.py @@ -76,37 +76,36 @@ class MatrixSections(Sections): )) return "\n\n".join(sections) - def render_profile_http_api(self): - return self._render_http_api_group( - "profile", - sortFnOrPathList=["displayname", "avatar_url"] - ) - - def render_sync_http_api(self): - return self._render_http_api_group( - "sync" - ) - - def render_presence_http_api(self): - return self._render_http_api_group( - "presence", - sortFnOrPathList=["status"] - ) - - def render_membership_http_api(self): - return self._render_http_api_group( - "membership" - ) - def render_login_http_api(self): - return self._render_http_api_group( - "login" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_http_apis(self): + # map all swagger_apis to the form $GROUP_http_api + swagger_groups = self.units.get("swagger_apis").keys() + renders = {} + for group in swagger_groups: + sortFnOrPathList = None + if group == "presence": + sortFnOrPathList = ["status"] + elif group == "profile": + sortFnOrPathList=["displayname", "avatar_url"] + renders[group + "_http_api"] = self._render_http_api_group( + group, sortFnOrPathList + ) + return renders - def render_rooms_http_api(self): - return self._render_http_api_group( - "rooms" - ) + # Special function: Returning a dict will specify multiple sections where + # the key is the section name and the value is the value of the section + def render_group_events(self): + # map all event schemata to the form $EVENTTYPE_event with s/./_/g + # e.g. m_room_topic_event + schemas = self.units.get("event_schemas") + renders = {} + for event_type in schemas: + renders[event_type.replace(".", "_") + "_event"] = self._render_events( + lambda x: x == event_type, sorted + ) + return renders def render_room_events(self): def filterFn(eventType): @@ -180,3 +179,4 @@ class MatrixSections(Sections): def render_common_state_event_fields(self): return self._render_ce_type("state_event") + From 29bae1579017c05d8ff5f5cf415459cebf0bac12 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 11:30:07 +0100 Subject: [PATCH 119/223] Fix typo --- specification/0-events.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/0-events.rst b/specification/0-events.rst index a50d01fae..16948462c 100644 --- a/specification/0-events.rst +++ b/specification/0-events.rst @@ -22,7 +22,7 @@ prefixed with ``m.`` {{m_room_aliases_event}} -{{m_room_canonical_aliases_event}} +{{m_room_canonical_alias_event}} {{m_room_create_event}} From 1da64db302f07019e410e607775c9efa0702d100 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 14:29:37 +0100 Subject: [PATCH 120/223] Use relative depths for groups instead of absolute ones This means the group can be agnostic to how deeply nested it is, improving reusability of groups. --- scripts/gendoc.py | 32 ++++++++++++++++++++++++-------- specification/targets.yaml | 3 ++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 26ee5a483..364b8e65e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from docutils.core import publish_file +import copy import fileinput import glob import os @@ -292,21 +293,32 @@ def get_build_target(targets_listing, target_name): "Found target but 'files' key is not a list." ) - def get_group(group_id): + def get_group(group_id, depth): group_name = group_id[len("group:"):] group = all_targets.get("groups", {}).get(group_name) if not group: raise Exception( - "Tried to find group '" + group_name + "' but it " + - "doesn't exist." + "Tried to find group '%s' but it doesn't exist." % group_name ) + if not isinstance(group, list): + raise Exception( + "Expected group '%s' to be a list but it isn't." % group_name + ) + # deep copy so changes to depths don't contaminate multiple uses of this group + group = copy.deepcopy(group) + # swap relative depths for absolute ones + for i, entry in enumerate(group): + if isinstance(entry, dict): + group[i] = { + (rel_depth + depth): v for (rel_depth, v) in entry.items() + } return group resolved_files = [] for file_entry in target["files"]: # file_entry is a group id if isinstance(file_entry, basestring) and file_entry.startswith("group:"): - group = get_group(file_entry) + group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than # insert the entire list as a single element (which is what append does) @@ -317,12 +329,16 @@ def get_build_target(targets_listing, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (k, v) in file_entry.iteritems(): - if isinstance(v, basestring) and v.startswith("group:"): - resolved_entry[k] = get_group(v) + for (depth, entry) in file_entry.iteritems(): + if not isinstance(entry, basestring): + raise Exception( + "Double-nested depths are not supported. Entry: %s" % (file_entry,) + ) + if entry.startswith("group:"): + resolved_entry[depth] = get_group(entry, depth) else: # map across without editing (e.g. normal file path) - resolved_entry[k] = v + resolved_entry[depth] = entry resolved_files.append(resolved_entry) continue # file_entry is just a plain ol' file path diff --git a/specification/targets.yaml b/specification/targets.yaml index e33bc7851..62585c698 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -23,7 +23,8 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/end_to_end_encryption.rst - modules/history_visibility.rst - modules/push_overview.rst - - { 2: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } + # relative depth + - { 1: [modules/push_cs_api.rst , modules/push_push_gw_api.rst] } title_styles: ["=", "-", "~", "+", "^"] From 064a2c91721a285cf03dcd0505e3c01a8af39256 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 14:59:57 +0100 Subject: [PATCH 121/223] Use argparse and log functions for gendoc.py gendoc.py has become more complex such that we actually want to pass things to it like `--verbose`, `--nodelete`, `--target`, so use `argparse` to do this like we have `build.py`. Pass through `-v` flags to `build.py`. --- scripts/gendoc.py | 91 +++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 364b8e65e..d2b0d75ef 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,5 +1,6 @@ #! /usr/bin/env python +from argparse import ArgumentParser from docutils.core import publish_file import copy import fileinput @@ -17,6 +18,7 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] } +VERBOSE = False """ Read a RST file and replace titles with a different title level if required. @@ -64,8 +66,8 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): if file_offset is None: file_offset = line_title_level if file_offset != 0: - print (" WARNING: %s starts with a title style of '%s' but '%s' " + - "is preferable.") % (filename, line_title_style, title_styles[0]) + logv((" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0])) # Sanity checks: Make sure that this file is obeying the title levels # specified and bail if it isn't. @@ -101,12 +103,11 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): continue # Adjusting line levels - # print ( - # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % - # (filename, line_title_style, - # title_styles[adjusted_level], - # file_offset, title_level) - # ) + logv( + "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + (filename, line_title_style, title_styles[adjusted_level], + file_offset, title_level) + ) rst_lines.append(line.replace( line_title_style, title_styles[adjusted_level] @@ -118,7 +119,7 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): - print "%s %s" % (">" * (1 + title_level), file_info) + log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None if adjust_titles: @@ -244,15 +245,21 @@ def run_through_template(input): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: - print subprocess.check_output( - [ - 'python', 'build.py', "-v", - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ], - stderr=out, - cwd="../templating", + args = [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ] + if VERBOSE: + args.insert(2, "-v") + log(" ==== build.py output ==== ") + log( + subprocess.check_output( + args, + stderr=out, + cwd="../templating" + ) ) except subprocess.CalledProcessError as e: with open(tmpfile, 'r') as f: @@ -347,6 +354,13 @@ def get_build_target(targets_listing, target_name): build_target["files"] = resolved_files return build_target +def log(line): + print "gendoc: %s" % line + +def logv(line): + if VERBOSE: + print "gendoc:V: %s" % line + def prepare_env(): try: @@ -363,9 +377,9 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name): +def main(target_name, keep_intermediates): prepare_env() - print "Building spec [target=%s]" % target_name + log("Building spec [target=%s]" % target_name) target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/templated_spec.rst") run_through_template("tmp/templated_spec.rst") @@ -377,22 +391,29 @@ def main(target_name): run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") - if "--nodelete" not in sys.argv: + if not keep_intermediates: cleanup_env() if __name__ == '__main__': - if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: - # we accept almost no args, so they don't know what they're doing! - print "gendoc.py - Generate the Matrix specification as HTML." - print "Usage:" - print " python gendoc.py [--nodelete]" - print "" - print "The specification can then be found in the gen/ folder." - print ("If --nodelete was specified, intermediate files will be " - "present in the tmp/ folder.") - print "" - print "Requirements:" - print " - This script requires Jinja2 and rst2html (docutils)." - sys.exit(0) - main("main") + parser = ArgumentParser( + "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." + ) + parser.add_argument( + "--nodelete", "-n", action="store_true", + help="Do not delete intermediate files. They will be found in tmp/" + ) + parser.add_argument( + "--target", "-t", default="main", + help="Specify the build target to build from specification/targets.yaml" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Turn on verbose mode." + ) + args = parser.parse_args() + if not args.target: + parser.print_help() + sys.exit(1) + VERBOSE = args.verbose + main(args.target, args.nodelete) From f1adad5fb3b2c6f03d1b635880726a661ba7bb03 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 15:10:55 +0100 Subject: [PATCH 122/223] Add more logging with file prefixes This makes the handoff between gendoc and batesian clearer in the logs. --- scripts/gendoc.py | 11 +++++------ templating/batesian/sections.py | 2 +- templating/batesian/units.py | 2 +- templating/build.py | 14 ++++++++------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index d2b0d75ef..611d0d105 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -253,13 +253,12 @@ def run_through_template(input): ] if VERBOSE: args.insert(2, "-v") + log("EXEC: %s" % " ".join(args)) log(" ==== build.py output ==== ") - log( - subprocess.check_output( - args, - stderr=out, - cwd="../templating" - ) + print subprocess.check_output( + args, + stderr=out, + cwd="../templating" ) except subprocess.CalledProcessError as e: with open(tmpfile, 'r') as f: diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 318493898..c18c9a5fa 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -16,7 +16,7 @@ class Sections(object): def log(self, text): if self.debug: - print text + print "batesian:sections: %s" % text def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/batesian/units.py b/templating/batesian/units.py index c20a2d1f2..993f253dc 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,7 @@ class Units(object): def log(self, text): if self.debug: - print text + print "batesian:units: %s" % text def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/build.py b/templating/build.py index ac3d2491a..013248f4e 100755 --- a/templating/build.py +++ b/templating/build.py @@ -52,8 +52,8 @@ def create_from_template(template, sections): def check_unaccessed(name, store): unaccessed_keys = store.get_unaccessed_set() if len(unaccessed_keys) > 0: - print "Found %s unused %s keys." % (len(unaccessed_keys), name) - print unaccessed_keys + log("Found %s unused %s keys." % (len(unaccessed_keys), name)) + log(unaccessed_keys) def main(input_module, file_stream=None, out_dir=None, verbose=False): if out_dir and not os.path.exists(out_dir): @@ -121,17 +121,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return # check the input files and substitute in sections where required - print "Parsing input template: %s" % file_stream.name + log("Parsing input template: %s" % file_stream.name) temp = Template(file_stream.read()) - print "Creating output for: %s" % file_stream.name + log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: f.write(output) - print "Output file for: %s" % file_stream.name + log("Output file for: %s" % file_stream.name) check_unaccessed("units", units) +def log(line): + print "batesian: %s" % line if __name__ == '__main__': parser = ArgumentParser( @@ -175,7 +177,7 @@ if __name__ == '__main__': sys.exit(0) if not args.file: - print "No file supplied." + log("No file supplied.") parser.print_help() sys.exit(1) From 6afdfc0771265ded3b7e0752c7ee438b09f076cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 15:36:13 +0100 Subject: [PATCH 123/223] Add more logging and make logging context clearer This is now actually useful if you want to debug why your swagger YAML isn't producing a table you think it should be. --- scripts/gendoc.py | 8 ++++---- templating/batesian/units.py | 6 +++++- templating/matrix_templates/units.py | 28 +++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 611d0d105..1655d6f0a 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -241,7 +241,7 @@ def rst2html(i, o): ) -def run_through_template(input): +def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: @@ -251,7 +251,7 @@ def run_through_template(input): "-o", "../scripts/tmp", "../scripts/"+input ] - if VERBOSE: + if set_verbose: args.insert(2, "-v") log("EXEC: %s" % " ".join(args)) log(" ==== build.py output ==== ") @@ -381,13 +381,13 @@ def main(target_name, keep_intermediates): log("Building spec [target=%s]" % target_name) target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst", VERBOSE) fix_relative_titles( target=target, filename="tmp/templated_spec.rst", out_filename="tmp/full_spec.rst" ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: diff --git a/templating/batesian/units.py b/templating/batesian/units.py index 993f253dc..144cf245a 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,11 @@ class Units(object): def log(self, text): if self.debug: - print "batesian:units: %s" % text + func_name = "" + trace = inspect.stack() + if len(trace) > 1 and len(trace[1]) > 2: + func_name = trace[1][3] + ":" + print "batesian:units:%s %s" % (func_name, text) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index a5c6a8153..50fa784e6 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -1,4 +1,12 @@ -"""Contains all the units for the spec.""" +""" +Contains all the units for the spec. + +This file loads swagger and JSON schema files and parses out the useful bits +and returns them as Units for use in Batesian. + +For the actual conversion of data -> RST (including templates), see the sections +file instead. +""" from batesian.units import Units import inspect import json @@ -134,7 +142,7 @@ class MatrixUnits(Units): "good_response": "" } } - self.log(".o.O.o. Endpoint: %s %s" % (method, path)) + self.log(" ------- Endpoint: %s %s ------- " % (method, path)) for param in single_api.get("parameters", []): # description desc = param.get("description", "") @@ -183,6 +191,9 @@ class MatrixUnits(Units): "desc": json_body[key]["description"] }) # endfor[param] + for row in endpoint["req_params"]: + self.log("Request parameter: %s" % row) + # group params by location to ease templating endpoint["req_param_by_loc"] = { # path: [...], query: [...], body: [...] @@ -240,6 +251,7 @@ class MatrixUnits(Units): # add response params if this API has any. if good_response: + self.log("Found a 200 response for this API") res_type = Units.prop(good_response, "schema/type") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that @@ -278,6 +290,16 @@ class MatrixUnits(Units): }] }) + for response_table in endpoint["res_tables"]: + self.log("Response: %s" % response_table["title"]) + for r in response_table["rows"]: + self.log("Row: %s" % r) + if len(endpoint["res_tables"]) == 0: + self.log( + "This API appears to have no response table. Are you " + + "sure this API returns no parameters?" + ) + endpoints.append(endpoint) aliases = single_api.get("x-alias", None) @@ -475,7 +497,7 @@ class MatrixUnits(Units): if re.match("^v[0-9\.]+$", word): version = word[1:] # strip the 'v' - self.log("Version: %s Title part: %s Changelog lines: %s" % ( + self.log("Version: %s Title part: %s Changelog line count: %s" % ( version, title_part, len(changelog_lines) )) if not version or len(changelog_lines) == 0: From 385b6c4759515b8b79151984e28932968ba9dc0c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 13:37:32 +0100 Subject: [PATCH 124/223] Only validate a file if it ends with ".yaml". Otherwise we try to validate vim .swp files. --- api/validator.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/validator.js b/api/validator.js index 6eef652f3..3b89a5a36 100644 --- a/api/validator.js +++ b/api/validator.js @@ -44,7 +44,8 @@ if (isDir) { process.exit(1); } files.forEach(function(f) { - if (f.indexOf(".yaml") > 0) { + var suffix = ".yaml"; + if (f.indexOf(suffix, f.length - suffix.length) > 0) { parser.parse(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); From 2f039a114201887f7b2da8b4f8a334bcd9f8f515 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 15:39:17 +0100 Subject: [PATCH 125/223] Add swagger docs for the v2 filter POST API --- api/client-server/v2_alpha/filter.yaml | 188 +++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 api/client-server/v2_alpha/filter.yaml diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml new file mode 100644 index 000000000..99dc0ebcf --- /dev/null +++ b/api/client-server/v2_alpha/filter.yaml @@ -0,0 +1,188 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 filter API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +definitions: + EventFilter: + title: + type: object + properties: + types: + type: array + description: |- + A list of event types to include. + If this list is absent then all event types are included. + items: + type: string + not_types: + type: array + description: |- + A list of event types to exclude. + If this list is absent then no event types are excluded. + items: + type: string + senders: + type: array + description: |- + A list of senders IDs to include. + If this list is absent then all senders are included. + items: + type: string + not_senders: + type: array + description: |- + A list of sender IDs to exclude + If this list is absent then no senders are excluded. + items: + type: string + rooms: + type: array + description: |- + A list of room IDs to include. + If this list is absent then all rooms are included. + items: + type: string + not_rooms: + type: array + description: |- + A list of room IDs to exclude + If this list is absent then no rooms are excluded. + items: + type: string + SyncFilter: + room: + type: object + properties: + state: + description: + The state events to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + events: + description: + The message and state update events to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + ephemeral: + description: |- + The events that aren't recorded in the permenant history, e.g. + typing and receipts, to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + public_user_data: + description: |- + The public user data, e.g. profile and presence, to include. + allOf: + - $ref: "#/definitions/EventFilter" + private_user_data: + description: |- + Events that are private to a user but shared amoungst their devices, + e.g. notification settings, to include. + allOf: + - $ref: "#/definitions/EventFilter" + event_format: + description: |- + The format to use for events. "client" will return the events in a + format suitable for clients. "federation" will return the raw event + as receieved over federation. The default is "client". + type: string + event_fields: + type: array + description: |- + List of event fields to include. If this list is absent then all fields + are included. The entries may include "." charaters to indicate + sub-fields. So ["content.body"] will include the "body" field of the + "content" object. A server may include more fields than were requested. + items: + string +paths: + "/user/{userId}/filter": + post: + summary: Upload a new filter. + description: |- + Uploads a new filter definition to the homeserver. + Returns a filter ID that may be used in /sync requests to + retrict which events are returned to the client. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: + The id of the user uploading the filter. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: body + name: filter + required: true + description: The filter to upload. + schema: + type: object + allOf: + - $ref: "#/definitions/SyncFilter" + example: + type: object + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"], + }, + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + responses: + 200: + description: The filter was created. + examples: + application/json: |- + { + "filter_id": "66696p746572" + } + schema: + type: object + properties: + filter_id: + type: string + description: + The ID of the filter that was created. + + + From 883105eae69d51e8672f63b9aa48946b6ae5347b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:25:03 +0100 Subject: [PATCH 126/223] Document the v2 filter GET API --- .../v2_alpha/definitions/definitions | 1 + .../v2_alpha/definitions/event_filter.json | 53 ++++++ .../v2_alpha/definitions/sync_filter.json | 48 ++++++ api/client-server/v2_alpha/filter.yaml | 158 +++++++----------- 4 files changed, 159 insertions(+), 101 deletions(-) create mode 120000 api/client-server/v2_alpha/definitions/definitions create mode 100644 api/client-server/v2_alpha/definitions/event_filter.json create mode 100644 api/client-server/v2_alpha/definitions/sync_filter.json diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/api/client-server/v2_alpha/definitions/definitions @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json new file mode 100644 index 000000000..b8ab1352e --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "properties": { + "types": { + "type": "array", + "description": + "A list of event types to include. If this list is absent then all event types are included.", + "items": { + "type": "string" + } + }, + "not_types": { + "type": "array", + "description": + "A list of event types to exclude. If this list is absent then no event types are excluded.", + "items": { + "type": "string" + } + }, + "senders": { + "type": "array", + "description": + "A list of senders IDs to include. If this list is absent then all senders are included.", + "items": { + "type": "string" + } + }, + "not_senders": { + "type": "array", + "description": + "A list of sender IDs to exclude. If this list is absent then no senders are excluded.", + "items": { + "type": "string" + } + }, + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json new file mode 100644 index 000000000..c9c47646d --- /dev/null +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -0,0 +1,48 @@ +{ + "type": "object", + "properties": { + "room": { + "type": "object", + "properties": { + "state": { + "description": + "The state events to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "events": { + "description": + "The message and state update events to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "ephemeral": { + "description": + "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + } + } + }, + "public_user_data": { + "description": + "The public user data, e.g. profile and presence, to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "private_user_data": { + "description": + "Events that are private to a user but shared amoungst their devices, e.g. notification settings, to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "event_format": { + "description": + "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", + "type": "string" + }, + "event_fields": { + "type": "array", + "description": + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A server may include more fields than were requested.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 99dc0ebcf..115ef2308 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -16,103 +16,10 @@ securityDefinitions: description: The user_id or application service access_token name: access_token in: query -definitions: - EventFilter: - title: - type: object - properties: - types: - type: array - description: |- - A list of event types to include. - If this list is absent then all event types are included. - items: - type: string - not_types: - type: array - description: |- - A list of event types to exclude. - If this list is absent then no event types are excluded. - items: - type: string - senders: - type: array - description: |- - A list of senders IDs to include. - If this list is absent then all senders are included. - items: - type: string - not_senders: - type: array - description: |- - A list of sender IDs to exclude - If this list is absent then no senders are excluded. - items: - type: string - rooms: - type: array - description: |- - A list of room IDs to include. - If this list is absent then all rooms are included. - items: - type: string - not_rooms: - type: array - description: |- - A list of room IDs to exclude - If this list is absent then no rooms are excluded. - items: - type: string - SyncFilter: - room: - type: object - properties: - state: - description: - The state events to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - events: - description: - The message and state update events to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - ephemeral: - description: |- - The events that aren't recorded in the permenant history, e.g. - typing and receipts, to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - public_user_data: - description: |- - The public user data, e.g. profile and presence, to include. - allOf: - - $ref: "#/definitions/EventFilter" - private_user_data: - description: |- - Events that are private to a user but shared amoungst their devices, - e.g. notification settings, to include. - allOf: - - $ref: "#/definitions/EventFilter" - event_format: - description: |- - The format to use for events. "client" will return the events in a - format suitable for clients. "federation" will return the raw event - as receieved over federation. The default is "client". - type: string - event_fields: - type: array - description: |- - List of event fields to include. If this list is absent then all fields - are included. The entries may include "." charaters to indicate - sub-fields. So ["content.body"] will include the "body" field of the - "content" object. A server may include more fields than were requested. - items: - string paths: - "/user/{userId}/filter": - post: - summary: Upload a new filter. + "/user/{userId}/filter": + post: + summary: Upload a new filter. description: |- Uploads a new filter definition to the homeserver. Returns a filter ID that may be used in /sync requests to @@ -135,7 +42,7 @@ paths: schema: type: object allOf: - - $ref: "#/definitions/SyncFilter" + - $ref: "definitions/sync_filter.json" example: type: object example: |- @@ -181,8 +88,57 @@ paths: properties: filter_id: type: string - description: + description: |- The ID of the filter that was created. - - - + "/user/{userId}/filter/{filterId}": + get: + summary: Download a filter + parameters: + - in: path + name: userId + type: string + description: |- + The user ID to download a filter for. + x-example: "@alice:example.com" + - in: path + name: filterId + description: |- + The filter ID to download. + x-example: "66696p746572" + responses: + 200: + examples: + application/json: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" From 69298b96121182e44a47a29eab3c69de5e91f37a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:29:51 +0100 Subject: [PATCH 127/223] Check "v2_alpha" in jenkins --- api/client-server/v2_alpha/filter.yaml | 5 +++++ jenkins.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 115ef2308..f53ef2ba1 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -100,13 +100,18 @@ paths: description: |- The user ID to download a filter for. x-example: "@alice:example.com" + required: true - in: path name: filterId + type: string description: |- The filter ID to download. x-example: "66696p746572" + required: true responses: 200: + description: |- + "The filter defintion" examples: application/json: |- { diff --git a/jenkins.sh b/jenkins.sh index e10436449..0936de9da 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,5 +5,5 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) (cd scripts && ./gendoc.py) -(cd api && npm install && node validator.js -s "client-server/v1") +(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) From a31a446661262872ba668c2c3192c5cf0a180387 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:48:00 +0100 Subject: [PATCH 128/223] Fix the POST example for the v2 filter API --- api/client-server/v2_alpha/filter.yaml | 58 +++++++++++++------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index f53ef2ba1..5296f11ac 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -43,38 +43,36 @@ paths: type: object allOf: - $ref: "definitions/sync_filter.json" - example: - type: object - example: |- - { - "room": { - "state": { - "types": ["m.room.*"], - "not_rooms": ["!726s6s6q:example.com"], - }, - "events": { - "types": ["m.room.message"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - }, - "emphemeral": { - "types": ["m.receipt", "m.typing"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - } - }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] }, - "server_data": { - "types": [] + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] }, - "event_format": "client", - "event_fields": ["type", "content", "sender"] - } + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } responses: 200: description: The filter was created. From f34722485f5984d5bbf3bc26f50bd54949730295 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:50:27 +0100 Subject: [PATCH 129/223] Check the request example JSON matches the schema --- api/check_examples.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index 00e75263c..a0cd0658b 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -29,6 +29,33 @@ except ImportError as e: raise +def check_parameter(filepath, request, parameter): + schema = parameter.get("schema") + example = None + try: + example_json = schema.get('example') + if example_json: + example = json.loads(example_json) + except Exception as e: + raise ValueError("Error parsing JSON example request for %r" % ( + request + ), e) + fileurl = "file://" + os.path.abspath(filepath) + if example and schema: + try: + print ("Checking request schema for: %r %r" % ( + filepath, request + )) + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = fileurl + jsonschema.validate(example, schema) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + + def check_response(filepath, request, code, response): example = None try: @@ -43,7 +70,9 @@ def check_response(filepath, request, code, response): fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: - print ("Checking schema for: %r %r %r" % (filepath, request, code)) + print ("Checking response schema for: %r %r %r" % ( + filepath, request, code + )) # Setting the 'id' tells jsonschema where the file is so that it # can correctly resolve relative $ref references in the schema schema['id'] = fileurl @@ -59,8 +88,13 @@ def check_swagger_file(filepath): swagger = yaml.load(f) for path, path_api in swagger.get('paths', {}).items(): + for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) + for parameter in request_api.get('parameters', ()): + if parameter['in'] == 'body': + check_parameter(filepath, request, parameter) + try: responses = request_api['responses'] except KeyError: From 078dd0165ffe045229b0efee92ec8f425cfd28df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 11:58:47 +0100 Subject: [PATCH 130/223] Update the room creation API spec to include new keys: 'preset' and 'initial_state' --- specification/1-client_server_api.rst | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 93e3cb907..e3dbde5a3 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -750,10 +750,41 @@ options which can be set when creating a room: This will tell the server to invite everyone in the list to the newly created room. +``preset`` + Type: + String + Optional: + Yes + Value: + ``private_chat`` or ``public_chat`` + Description: + Convenience parameter for setting various default state events based on a + preset. + + Two presets are defined: + + - ``private_chat``: Sets the ``join_rules`` to ``invite`` and + ``history_visibility`` to ``shared`` + - ``public_chat``: Sets the ``join_rules`` to ``public`` and + ``history_visibility`` to ``shared`` + +``initial_state`` + Type: + List + Optional: + Yes + Value: + A list of state events to set in the new room. + Description: + Allows the user to override the default state events set in the new room. + + The expected format of the state events are an object with ``type``, + ``state_key`` and ``content`` keys set. + Example:: { - "visibility": "public", + "preset": "public_chat", "room_alias_name": "thepub", "name": "The Grand Duke Pub", "topic": "All about happy hour" From 6c89e6ea6717d9cd07afed7906add62ab82c0431 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:03:46 +0100 Subject: [PATCH 131/223] Wrap refresh_token in `s --- api/client-server/v1/login.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml index 0852db6b8..a4fd8b959 100644 --- a/api/client-server/v1/login.yaml +++ b/api/client-server/v1/login.yaml @@ -66,7 +66,7 @@ paths: description: |- An access token for the account. This access token can then be used to authorize other requests. - The access token may expire at some point, and if so, it SHOULD come with a refresh_token. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. There is no specific error message to indicate that a request has failed because an access token has expired; instead, if a client has reason to believe its access token is valid, and it receives an auth error, they should attempt to @@ -132,7 +132,7 @@ paths: description: |- An access token for the account. This access token can then be used to authorize other requests. - The access token may expire at some point, and if so, it SHOULD come with a refresh_token. + The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``. refresh_token: type: string description: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the TODO Linkify /tokenrefresh API endpoint. From fc87f4cdb03cca548ee7ec499949c1d52d5104a3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:10:15 +0100 Subject: [PATCH 132/223] Remove unused keys --- api/client-server/v1/login.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml index a4fd8b959..4cb564b7c 100644 --- a/api/client-server/v1/login.yaml +++ b/api/client-server/v1/login.yaml @@ -29,7 +29,6 @@ paths: parameters: - in: body name: body - required: true schema: type: object example: |- @@ -100,7 +99,6 @@ paths: parameters: - in: body name: body - required: true schema: type: object example: |- From 5c4398c181bbf9da4aada6d4f8a3bf5b0d49c877 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:10:49 +0100 Subject: [PATCH 133/223] Remove superfluous comma --- api/client-server/v1/login.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml index 4cb564b7c..e415e7980 100644 --- a/api/client-server/v1/login.yaml +++ b/api/client-server/v1/login.yaml @@ -114,7 +114,7 @@ paths: 200: description: |- The refresh token was accepted, and a new access token has been issued. - The passed refresh token is no longer valid, and cannot be used. + The passed refresh token is no longer valid and cannot be used. A new refresh token may have been returned. examples: application/json: |- From 6c1491b3bafb3a806d8fdf4ed59eacdf4df79d06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:17:11 +0100 Subject: [PATCH 134/223] Respond to some review comments --- api/client-server/v1/login.yaml | 3 ++- drafts/macaroons_caveats.rst | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/login.yaml b/api/client-server/v1/login.yaml index e415e7980..3d415c293 100644 --- a/api/client-server/v1/login.yaml +++ b/api/client-server/v1/login.yaml @@ -115,7 +115,8 @@ paths: description: |- The refresh token was accepted, and a new access token has been issued. The passed refresh token is no longer valid and cannot be used. - A new refresh token may have been returned. + A new refresh token will have been returned unless some policy does + not allow the user to continue to renew their session. examples: application/json: |- { diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index 791d217ad..c4b6b6a4d 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -1,7 +1,9 @@ Macaroon Caveats ================ -Macaroons (http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. +`Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. + +.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. From f5d436bd808ac9a1f20d816dccceac63a9202546 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:18:09 +0100 Subject: [PATCH 135/223] Remove extraneous ) --- drafts/macaroons_caveats.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drafts/macaroons_caveats.rst b/drafts/macaroons_caveats.rst index c4b6b6a4d..93622c3d4 100644 --- a/drafts/macaroons_caveats.rst +++ b/drafts/macaroons_caveats.rst @@ -3,7 +3,7 @@ Macaroon Caveats `Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them. -.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf) +.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand. From cd6f15f62740075c2e8d1914fe77f3596046b166 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 25 Sep 2015 13:34:24 +0100 Subject: [PATCH 136/223] Remove obsolete comment --- api/client-server/v1/membership.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/client-server/v1/membership.yaml b/api/client-server/v1/membership.yaml index 04922656d..41f2febc5 100644 --- a/api/client-server/v1/membership.yaml +++ b/api/client-server/v1/membership.yaml @@ -69,7 +69,6 @@ paths: "/rooms/{roomId}/invite": post: summary: Invite a user to participate in a particular room. - # It's a crying shame that I don't know how to force line breaks. description: |- This API invites a user to participate in a particular room. They do not start participating in the room until they actually join the From d7d9f406a67419306d992bf906f01232aad59e47 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 14:21:50 +0100 Subject: [PATCH 137/223] Bundle some state into invites --- api/client-server/v1/sync.yaml | 5 +++++ event-schemas/schema/v1/m.room.member | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index 833c425ab..c7f781877 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -267,6 +267,11 @@ paths: type: string description: "The user's membership state in this room." enum: ["invite", "join", "leave", "ban"] + invite: + type: object + description: "The invite event if `membership` is `invite`" + allOf: + - "$ref": "room.room.member" messages: type: object title: PaginationChunk diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 49b9f5b83..32302b85f 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -32,6 +32,25 @@ "type": { "type": "string", "enum": ["m.room.member"] + }, + "invite_room_state": { + "type": "array", + "description": "A subset of the state of the room at the time of the invite, if ``membership`` is ``invite``", + "items": { + "type": "object", + "title": "StateEvent", + "properties": { + "type": { + "type": "string" + }, + "state_key": { + "type": "string" + }, + "content": { + "type": "object" + } + } + } } } } From 37ccddb3087283067617a94d83dc968489bae20f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 14:25:07 +0100 Subject: [PATCH 138/223] Typo --- api/client-server/v1/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index c7f781877..d82ee9e4e 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -271,7 +271,7 @@ paths: type: object description: "The invite event if `membership` is `invite`" allOf: - - "$ref": "room.room.member" + - "$ref": "m.room.member" messages: type: object title: PaginationChunk From 18dc7784df09c3ad6c3f4d844b87d046538bd5a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 14:34:06 +0100 Subject: [PATCH 139/223] Mention precedence --- specification/1-client_server_api.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index e3dbde5a3..c5453931a 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -781,6 +781,9 @@ options which can be set when creating a room: The expected format of the state events are an object with ``type``, ``state_key`` and ``content`` keys set. + Takes precedence over events set by ``presets``, but gets overriden by + ``name`` and ``topic`` keys. + Example:: { From c6375ed3d1743eb931e37d12b082c91a2b20d7b4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 25 Sep 2015 15:09:15 +0100 Subject: [PATCH 140/223] Flesh out feature profiles section Add table detailing the profiles. Add anchors to link through to each module following a well-defined format (rather than the name of the module section). Allow UTF-8 in the spec. --- specification/0-feature_profiles.rst | 35 +++++++++++++++++++ specification/modules/content_repo.rst | 2 ++ .../modules/end_to_end_encryption.rst | 2 ++ specification/modules/instant_messaging.rst | 2 ++ specification/modules/presence.rst | 2 ++ specification/modules/push_overview.rst | 2 ++ specification/modules/receipts.rst | 2 ++ .../modules/typing_notifications.rst | 2 ++ specification/modules/voip_events.rst | 3 ++ templating/build.py | 4 +-- 10 files changed, 54 insertions(+), 2 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 234e14db0..dcf7f6426 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -1,3 +1,38 @@ Feature Profiles ================ +Matrix supports many different kinds of clients: from embedded IoT devices to +desktop clients. Not all clients can provide the same feature sets as other +clients e.g. due to lack of physical hardware such as not having a screen. +Clients can fall into one of several profiles and each profile contains a set +of features that the client MUST support. This section details a set of +"feature profiles". Clients are expected to implement a profile in its entirety +in order for it to be classified as that profile. + +Summary +------- + +============================ ===== =========== ======== ========= ===== ===== + Module / Profile Web Embed-Web Mobile Desktop CLI IoT +============================ ===== =========== ======== ========= ===== ===== + `End-to-End Encryption`_ YES YES YES YES + `Instant Messaging`_ YES YES YES YES YES YES + `Presence`_ YES YES YES YES + `Push Notifications`_ YES + `Receipts`_ YES YES YES YES + `Typing Notifications`_ YES YES YES YES + `VoIP`_ YES YES YES + `Content Repository`_ YES YES YES YES +============================ ===== =========== ======== ========= ===== ===== + +*Please see each module for more details on what clients need to implement.* + +.. _End-to-End Encryption: `module:e2e`_ +.. _Instant Messaging: `module:im`_ +.. _Presence: `module:presence`_ +.. _Push Notifications: `module:push`_ +.. _Receipts: `module:receipts`_ +.. _Typing Notifications: `module:typing`_ +.. _VoIP: `module:voip`_ +.. _Content Repository: `module:content`_ + diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 2c45ced74..c2cf3505b 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -1,6 +1,8 @@ Content repository ================== +.. _module:content: + HTTP API -------- diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index 023881527..e3a526136 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -1,6 +1,8 @@ End-to-End Encryption ===================== +.. _module:e2e: + .. TODO-doc - Why is this needed. - Overview of process diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst index 7f582ca4b..43a06aa13 100644 --- a/specification/modules/instant_messaging.rst +++ b/specification/modules/instant_messaging.rst @@ -1,6 +1,8 @@ Instant Messaging ================= +.. _module:im: + Events ------ diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index ddd2adff2..cb71107f3 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,5 +1,7 @@ Presence ======== + +.. _module:presence: Each user has the concept of presence information. This encodes the "availability" of that user, suitable for display on other user's clients. diff --git a/specification/modules/push_overview.rst b/specification/modules/push_overview.rst index 972a8eea1..46028283a 100644 --- a/specification/modules/push_overview.rst +++ b/specification/modules/push_overview.rst @@ -1,6 +1,8 @@ Push Notifications ================== +.. _module:push: + Overview -------- diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index e2f83eea8..9787682f0 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -1,6 +1,8 @@ Receipts -------- +.. _module:receipts: + Receipts are used to publish which events in a room the user or their devices have interacted with. For example, which events the user has read. For efficiency this is done as "up to" markers, i.e. marking a particular event diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index 25b714ab4..614f6af72 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,6 +1,8 @@ Typing Notifications -------------------- +.. _module:typing: + Client APIs ~~~~~~~~~~~ diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index a54682379..60d610788 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -1,5 +1,8 @@ Voice over IP ------------- + +.. _module:voip: + Matrix can also be used to set up VoIP calls. This is part of the core specification, although is at a relatively early stage. Voice (and video) over Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like diff --git a/templating/build.py b/templating/build.py index 013248f4e..6f465607b 100755 --- a/templating/build.py +++ b/templating/build.py @@ -122,13 +122,13 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): # check the input files and substitute in sections where required log("Parsing input template: %s" % file_stream.name) - temp = Template(file_stream.read()) + temp = Template(file_stream.read().decode("utf-8")) log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: - f.write(output) + f.write(output.encode("utf-8")) log("Output file for: %s" % file_stream.name) check_unaccessed("units", units) From 9fac152d32cea2dff81ed5094bb47dcb793ac640 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 25 Sep 2015 15:26:58 +0100 Subject: [PATCH 141/223] Explain what the clients are and the column reference --- specification/0-feature_profiles.rst | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index dcf7f6426..4961c2163 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -36,3 +36,46 @@ Summary .. _VoIP: `module:voip`_ .. _Content Repository: `module:content`_ +Clients +------- + +Stand-alone web client (``Web``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a web page which heavily uses Matrix for communication. Single-page web +apps would be classified as a stand-alone web client, as would multi-page web +apps which use Matrix on nearly every page. + +Embedded web client (``EmbedWeb``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a Matrix client which is embedded in another website, e.g. using +iframes. These embedded clients are typically for a single purpose +related to the website in question, and are not intended to be fully-fledged +communication apps. + +Mobile client (``Mobile``) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a Matrix client specifically designed for consumption on mobile devices. +This is typically a mobile app but need not be so provided the feature set can +be reached (e.g. if a mobile site could display push notifications it could be +classified as a mobile client). + +Desktop client (``Desktop``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a native application which can run in its own environment outside a +browser. + +Command Line Interface client (``CLI``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a client which is used via a text-based terminal. + +Internet of Things client (``IoT``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a client which is typically running on an embedded device such as a +kettle, fridge or car. + From 510553ee009049616288a262aec3a01705e14429 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 25 Sep 2015 15:29:33 +0100 Subject: [PATCH 142/223] Remove smurf suffixes. Add anchor for feature profiles. --- specification/0-feature_profiles.rst | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 4961c2163..ab9771625 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -1,6 +1,8 @@ Feature Profiles ================ +.. sect:feature-profiles: + Matrix supports many different kinds of clients: from embedded IoT devices to desktop clients. Not all clients can provide the same feature sets as other clients e.g. due to lack of physical hardware such as not having a screen. @@ -39,42 +41,42 @@ Summary Clients ------- -Stand-alone web client (``Web``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Stand-alone web (``Web``) +~~~~~~~~~~~~~~~~~~~~~~~~~ This is a web page which heavily uses Matrix for communication. Single-page web apps would be classified as a stand-alone web client, as would multi-page web apps which use Matrix on nearly every page. -Embedded web client (``EmbedWeb``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Embedded web (``EmbedWeb``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a Matrix client which is embedded in another website, e.g. using iframes. These embedded clients are typically for a single purpose related to the website in question, and are not intended to be fully-fledged communication apps. -Mobile client (``Mobile``) -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mobile (``Mobile``) +~~~~~~~~~~~~~~~~~~~ This is a Matrix client specifically designed for consumption on mobile devices. This is typically a mobile app but need not be so provided the feature set can be reached (e.g. if a mobile site could display push notifications it could be classified as a mobile client). -Desktop client (``Desktop``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Desktop (``Desktop``) +~~~~~~~~~~~~~~~~~~~~~ This is a native application which can run in its own environment outside a browser. -Command Line Interface client (``CLI``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Command Line Interface (``CLI``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a client which is used via a text-based terminal. -Internet of Things client (``IoT``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Internet of Things (``IoT``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a client which is typically running on an embedded device such as a kettle, fridge or car. From 9c882ea5e4df6dd68fa77e08235ded9e1f8a6e83 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 25 Sep 2015 16:24:31 +0100 Subject: [PATCH 143/223] Detect unused {{ vars }} and whine loudly. --- templating/build.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/templating/build.py b/templating/build.py index 013248f4e..ee3d79dec 100755 --- a/templating/build.py +++ b/templating/build.py @@ -38,7 +38,7 @@ Processing """ from batesian import AccessKeyStore -from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template +from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta from argparse import ArgumentParser, FileType import importlib import json @@ -122,7 +122,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): # check the input files and substitute in sections where required log("Parsing input template: %s" % file_stream.name) - temp = Template(file_stream.read()) + temp_str = file_stream.read() + # do sanity checking on the template to make sure they aren't reffing things + # which will never be replaced with a section. + ast = env.parse(temp_str) + template_vars = meta.find_undeclared_variables(ast) + unused_vars = [var for var in template_vars if var not in sections] + if len(unused_vars) > 0: + raise Exception( + "You have {{ variables }} which are not found in sections: %s" % + (unused_vars,) + ) + # process the template + temp = Template(temp_str) log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( From e1f73f523318951d2303e2d538b26fb3478abeda Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 18:09:17 +0100 Subject: [PATCH 144/223] Add a limit to filters --- api/client-server/v2_alpha/definitions/event_filter.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index b8ab1352e..76d8be64e 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -1,6 +1,11 @@ { "type": "object", "properties": { + "limit": { + "type": "integer", + "description": + "The maximum number of events to return." + }, "types": { "type": "array", "description": From a0068e1adaa028d29bc00d5ad62dd5a5b6371b1b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 18:10:28 +0100 Subject: [PATCH 145/223] Draft documention for the v2 sync api --- .../v2_alpha/definitions/event_batch.json | 12 ++ .../v2_alpha/definitions/timeline_batch.json | 14 ++ api/client-server/v2_alpha/sync.yaml | 167 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 api/client-server/v2_alpha/definitions/event_batch.json create mode 100644 api/client-server/v2_alpha/definitions/timeline_batch.json create mode 100644 api/client-server/v2_alpha/sync.yaml diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json new file mode 100644 index 000000000..c7cdbfb30 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of indicies into an events array", + "items": { + "type": "integer" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json new file mode 100644 index 000000000..dc62724bb --- /dev/null +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [{"$ref":"definitions/event_batch.json"}], + "properties": { + "limited": { + "type": "boolean", + "description": "Whether there are more events on the server" + }, + "prev_batch": { + "type": "string", + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + } + } +} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml new file mode 100644 index 000000000..c857d2729 --- /dev/null +++ b/api/client-server/v2_alpha/sync.yaml @@ -0,0 +1,167 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 sync API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/sync": + get: + summary: Synchronise the client's state and receive new messages. + description: |- + Synchronise the client's state with the latest state on the server. + Client's use this API when they first log in to get an initial snapshot + of the state on the server, and then continue to call this API to get + incremental deltas to the state, and to receive new messages. + security: + - accessToken: [] + parameters: + - in: query + name: filter + type: string + description: |- + The ID of a filter created using the filter API. + - in: query + name: since + type: string + description: |- + A point in time to continue a sync from. + - in: query + name: timeout + type: integer + description: |- + The maximum time to poll in milliseconds before returning this + request. + responses: + 200: + description: + The initial snapshot or delta for the client to use to update their + state. + schema: + type: object + properties: + next_batch: + type: string + description: |- + The batch token to supply in the ``since`` param of the next + ``/sync`` request. + events: + type: array + description: |- + A list of event objects that are referred to by index in the + rest of the ``sync`` response. + items: + type: object + rooms: + type: array + description: |- + A list of rooms that the client needs to update. + items: + type: object + properties: + room_id: + type: string + description: |- + The ID of the room. + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + public_user_data: + description: |- + The updates to publically visible user data. + allOf: + - $ref: "definitions/event_batch.json" + private_user_data: + description: |- + Updates to the data which is private to the user but shared + amoungst their devices. + allOf: + - $ref: "definitions/event_batch.json" + examples: + application/json: |- + { + "next_batch": "s72595_4483_1934", + "events": [ + { + "sender": "@bob:example.com", + "type": "com.example.weird.setting", + "content": {"setting1": true, "setting2": false} + }, + { + "sender": "@alice:example.com", + "type": "m.profile.display_name", + "content": {"display_name": "Alice"} + }, + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + }, + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + }, + { + "sender": "@alice:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + { + "sender": "@bob:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + { + "sender": "@alice:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } + ], + "private_user_data": {"events": [0]}, + "public_user_data": {"events": [1, 2]}, + "rooms": [{ + "room_id": "!726s6s6q:example.com", + "ephemeral": {"events": [3]}, + "state": {"events": [4, 5]}, + "timeline": { + "events": [5,6], + "limited": true, + "prev_batch": "t34-23535_0_0" + } + }] + } From de07586ab786d8cc8431dfc1b23030e9a77d6b57 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 19:34:55 +0100 Subject: [PATCH 146/223] Rename 'events' to 'timeline' in the sync filter --- api/client-server/v2_alpha/definitions/sync_filter.json | 2 +- api/client-server/v2_alpha/filter.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index c9c47646d..bfc8b51d6 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -9,7 +9,7 @@ "The state events to include for rooms.", "allOf": [{"$ref": "definitions/event_filter.json"}] }, - "events": { + "timeline": { "description": "The message and state update events to include for rooms.", "allOf": [{"$ref": "definitions/event_filter.json"}] diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 5296f11ac..3e564f4b5 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -50,7 +50,7 @@ paths: "types": ["m.room.*"], "not_rooms": ["!726s6s6q:example.com"] }, - "events": { + "timeline": { "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -118,7 +118,7 @@ paths: "types": ["m.room.*"], "not_rooms": ["!726s6s6q:example.com"] }, - "events": { + "timeline": { "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From c3b3b2df63ed8964d72517d0092b298988009d3f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 19:37:41 +0100 Subject: [PATCH 147/223] Add "set_presence" in the sync parameters --- api/client-server/v2_alpha/sync.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c857d2729..5018080e9 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -38,6 +38,11 @@ paths: type: string description: |- A point in time to continue a sync from. + - in: query + name: set_presence + type: string + description: |- + Set the presence status of this client. - in: query name: timeout type: integer From 8c22b715cabdb283de39a0f79bdef71847366323 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 09:29:07 +0100 Subject: [PATCH 148/223] Add title --- api/client-server/v1/sync.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index d82ee9e4e..dbfa39b80 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -269,6 +269,7 @@ paths: enum: ["invite", "join", "leave", "ban"] invite: type: object + title: "InviteEvent" description: "The invite event if `membership` is `invite`" allOf: - "$ref": "m.room.member" From 317c2f20d31345a656c553dc239563cf9f2de78d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 09:29:35 +0100 Subject: [PATCH 149/223] Draft login token spec --- specification/1-client_server_api.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 93e3cb907..7f9673379 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -197,6 +197,7 @@ This specification defines the following login types: - ``m.login.recaptcha`` - ``m.login.oauth2`` - ``m.login.email.identity`` + - ``m.login.token`` - ``m.login.dummy`` Password-based @@ -228,6 +229,29 @@ To respond to this type, reply with an auth dict as follows:: "response": "" } +Token-based +~~~~~~~~~~~ +:Type: + ``m.login.token`` +:Description: + The client submits a username and token that was generated by the server. + +To respond to this type, reply with an auth dict as follows:: + + { + "type": "m.login.token", + "user": "", + "token": "", + "nonce": "" + } + +The ``nonce`` should be a random string generated by the client for the +request. The same ``nonce`` should be used if retrying the request. + +The ``token`` may be discovered from e.g. an email or dynamically generated QR +code. + + OAuth2-based ~~~~~~~~~~~~ :Type: From 4c9f524cabbbc74a837743aaa043e977b67ae7d2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 09:32:03 +0100 Subject: [PATCH 150/223] Neaten things up --- specification/1-client_server_api.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 7f9673379..e3817046a 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -234,7 +234,7 @@ Token-based :Type: ``m.login.token`` :Description: - The client submits a username and token that was generated by the server. + The client submits a username and token. To respond to this type, reply with an auth dict as follows:: @@ -248,9 +248,8 @@ To respond to this type, reply with an auth dict as follows:: The ``nonce`` should be a random string generated by the client for the request. The same ``nonce`` should be used if retrying the request. -The ``token`` may be discovered from e.g. an email or dynamically generated QR -code. - +There are many ways a client may receive a ``token``, including via an email or +from an existing logged in device. OAuth2-based ~~~~~~~~~~~~ From 393d2831399e07316686129418d21b671571bfe7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 11:22:31 +0100 Subject: [PATCH 151/223] Add a "limit" to the timeline key in the example filter --- api/client-server/v2_alpha/filter.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 3e564f4b5..9bda2359d 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -51,6 +51,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"] }, "timeline": { + "limit": 10, "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -119,6 +120,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"] }, "timeline": { + "limit": 10, "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From 1aa916d690bac548625f5a460ad916a0c6fdf43d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 12:52:12 +0100 Subject: [PATCH 152/223] s/indicies/indices/ --- api/client-server/v2_alpha/definitions/event_batch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index c7cdbfb30..ed2730c87 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -3,7 +3,7 @@ "properties": { "events": { "type": "array", - "description": "List of indicies into an events array", + "description": "List of indices into an events array", "items": { "type": "integer" } From f50e6d4c0ad8648dc601344b6ac67c468f4e25f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:02:52 +0100 Subject: [PATCH 153/223] Add x-example fields for v2 /sync --- api/client-server/v2_alpha/sync.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 5018080e9..d307a683f 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -33,22 +33,26 @@ paths: type: string description: |- The ID of a filter created using the filter API. + x-example: "66696p746572" - in: query name: since type: string description: |- A point in time to continue a sync from. + x-example: "s72594_4483_1934" - in: query name: set_presence type: string description: |- Set the presence status of this client. + x-example: "online" - in: query name: timeout type: integer description: |- The maximum time to poll in milliseconds before returning this request. + x-example: 30000 responses: 200: description: From 3204c2f2b6cedf748618c594b86c514c2bf0b2b2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:04:37 +0100 Subject: [PATCH 154/223] Fix spelling --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index d307a683f..318a1ab5b 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -108,7 +108,7 @@ paths: private_user_data: description: |- Updates to the data which is private to the user but shared - amoungst their devices. + amongst their devices. allOf: - $ref: "definitions/event_batch.json" examples: From 00fd4aac264619ec6a53afecd0757149ad7c038c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:06:04 +0100 Subject: [PATCH 155/223] s/publically/publicly/ --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 318a1ab5b..e678d2d8e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -102,7 +102,7 @@ paths: - $ref: "definitions/event_batch.json" public_user_data: description: |- - The updates to publically visible user data. + The updates to publicly visible user data. allOf: - $ref: "definitions/event_batch.json" private_user_data: From 97154cc6b2088ea30d80b0c2eff3196a40b9a484 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 13:09:54 +0100 Subject: [PATCH 156/223] s/nonce/txn_id/ --- specification/1-client_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index e3817046a..988f7bc44 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -242,7 +242,7 @@ To respond to this type, reply with an auth dict as follows:: "type": "m.login.token", "user": "", "token": "", - "nonce": "" + "txn_id": "" } The ``nonce`` should be a random string generated by the client for the From db8f3c0d5969a6bc731a1e43e23359ea0dd160fe Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 28 Sep 2015 13:11:34 +0100 Subject: [PATCH 157/223] Desktop clients should have a GUI --- specification/0-feature_profiles.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index ab9771625..86f8f3d8a 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -67,7 +67,7 @@ classified as a mobile client). Desktop (``Desktop``) ~~~~~~~~~~~~~~~~~~~~~ -This is a native application which can run in its own environment outside a +This is a native GUI application which can run in its own environment outside a browser. Command Line Interface (``CLI``) From 41bc09ea22e8b18d0e509e666e30033dd3143e04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:20:01 +0100 Subject: [PATCH 158/223] Split the event_filter into a event_filter and a room_event_filter that extends it. So that we don't include "rooms" and "not_rooms" keys for the public_user_data and private_user_data filters. --- .../v2_alpha/definitions/event_filter.json | 16 -------------- .../definitions/room_event_filter.json | 21 +++++++++++++++++++ .../v2_alpha/definitions/sync_filter.json | 6 +++--- 3 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/room_event_filter.json diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index 76d8be64e..c15b81339 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -37,22 +37,6 @@ "items": { "type": "string" } - }, - "rooms": { - "type": "array", - "description": - "A list of room IDs to include. If this list is absent then all rooms are included.", - "items": { - "type": "string" - } - }, - "not_rooms": { - "type": "array", - "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", - "items": { - "type": "string" - } } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json new file mode 100644 index 000000000..c234448e1 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -0,0 +1,21 @@ +{ + "type": "object", + "properties": { + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index bfc8b51d6..4ad3aa82d 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -7,17 +7,17 @@ "state": { "description": "The state events to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] }, "timeline": { "description": "The message and state update events to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] }, "ephemeral": { "description": "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] } } }, From 940e22940dbde376db926b1b64fcda411cceffdd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:28:58 +0100 Subject: [PATCH 159/223] Document how the "not_foo" keys interact with the "foo" keys --- api/client-server/v2_alpha/definitions/event_filter.json | 4 ++-- api/client-server/v2_alpha/definitions/room_event_filter.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index c15b81339..269bb5f04 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -17,7 +17,7 @@ "not_types": { "type": "array", "description": - "A list of event types to exclude. If this list is absent then no event types are excluded.", + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter", "items": { "type": "string" } @@ -33,7 +33,7 @@ "not_senders": { "type": "array", "description": - "A list of sender IDs to exclude. If this list is absent then no senders are excluded.", + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter", "items": { "type": "string" } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json index c234448e1..5be0fcd28 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -1,5 +1,6 @@ { "type": "object", + "allOf": [{"$ref": "definitions/event_filter.json"}], "properties": { "rooms": { "type": "array", @@ -12,7 +13,7 @@ "not_rooms": { "type": "array", "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter", "items": { "type": "string" } From c2fc1a2fb19c6a46cc2bde736222f855f0f8ba3d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 28 Sep 2015 13:41:31 +0100 Subject: [PATCH 160/223] Add basic module template. This has the core sections for events/server/client/security. --- specification/modules/_template.rst | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 specification/modules/_template.rst diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst new file mode 100644 index 000000000..6ea7aaa35 --- /dev/null +++ b/specification/modules/_template.rst @@ -0,0 +1,47 @@ +Module Heading +============== + +.. _module:short-name: + +A short summary of the module. What features does this module provide? An anchor +should be specified at the top of the module using the format ``module:name``. + +Complicated modules may wish to have architecture diagrams or event flows +(e.g. VoIP call flows) here. Custom subsections can be included but they should +be used *sparingly* to reduce the risk of putting client or server behaviour +information in these custom sections. + +Events +------ +List the new event types introduced by this module, if any. If there are no +new events, this section can be omitted. Event types should be done as +subsections. The section is intended to document the "common shared event +structure" between client and server. Deviations from this shared structure +should be documented in the relevant behaviour section. + +example.event.type +~~~~~~~~~~~~~~~~~~ +There should be JSON Schema docs for this event. You can insert a template like +so: + +{{example_event_type_event}} + +Client behaviour +---------------- +List any new HTTP endpoints. List the steps the client needs to take to +correctly process this module. Listing what data structures the client should be +storing to aid implementation is recommended. + +Server behaviour +---------------- +Does the server need to handle any of the new events in a special way (e.g. +typing timeouts, presence). Advice on how to persist events and/or requests are +recommended to aid implementation. + +Security considerations +----------------------- +This includes privacy leaks: for example leaking presence info. How do +misbehaving clients or servers impact this module? This section should always be +included, if only to say "we've thought about it but there isn't anything to do +here". + From c115b4c2f4cfda43bdb6409edf9f14bfb8004151 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:47:04 +0100 Subject: [PATCH 161/223] Document the valid values for the "set_presence" parameter --- api/client-server/v2_alpha/sync.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index e678d2d8e..0fe9173d9 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -43,9 +43,14 @@ paths: - in: query name: set_presence type: string + enum: ["offline"] description: |- - Set the presence status of this client. - x-example: "online" + Controls whether the client is automatically marked as online by + polling this API. If this parameter is omitted then the client is + automatically marked as online when it uses this API. Otherwise if + the parameter is set to "offline" then the client is not marked as + being online when it uses this API. + x-example: "offline" - in: query name: timeout type: integer From 14b42a41d653f601d7f6a7cc61539a1ccf7f55c7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:58:07 +0100 Subject: [PATCH 162/223] Document wildcard's in filters --- api/client-server/v2_alpha/definitions/event_filter.json | 8 ++++---- .../v2_alpha/definitions/room_event_filter.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index 269bb5f04..1cdcb1f41 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -9,7 +9,7 @@ "types": { "type": "array", "description": - "A list of event types to include. If this list is absent then all event types are included.", + "A list of event types to include. If this list is absent then all event types are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -17,7 +17,7 @@ "not_types": { "type": "array", "description": - "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter", + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -25,7 +25,7 @@ "senders": { "type": "array", "description": - "A list of senders IDs to include. If this list is absent then all senders are included.", + "A list of senders IDs to include. If this list is absent then all senders are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -33,7 +33,7 @@ "not_senders": { "type": "array", "description": - "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter", + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json index 5be0fcd28..86375781c 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -5,7 +5,7 @@ "rooms": { "type": "array", "description": - "A list of room IDs to include. If this list is absent then all rooms are included.", + "A list of room IDs to include. If this list is absent then all rooms are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -13,7 +13,7 @@ "not_rooms": { "type": "array", "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter", + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } From 643468e9141902499550ace65da1bf8eb7e53bc5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 28 Sep 2015 14:01:54 +0100 Subject: [PATCH 163/223] Mention swagger. Clarify how event type template vars are formed. --- specification/modules/_template.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst index 6ea7aaa35..967892a54 100644 --- a/specification/modules/_template.rst +++ b/specification/modules/_template.rst @@ -19,18 +19,24 @@ subsections. The section is intended to document the "common shared event structure" between client and server. Deviations from this shared structure should be documented in the relevant behaviour section. -example.event.type -~~~~~~~~~~~~~~~~~~ -There should be JSON Schema docs for this event. You can insert a template like -so: +m.example.event.type +~~~~~~~~~~~~~~~~~~~~ +There should be JSON Schema docs for this event. Once there is JSON schema, +there will be a template variable with dots in the event type replaced with +underscores. You can insert a template like so: -{{example_event_type_event}} +{{m_example_event_type_event}} Client behaviour ---------------- -List any new HTTP endpoints. List the steps the client needs to take to +List any new HTTP endpoints. These endpoints should be docced using Swagger. You +can insert a template for swagger docs like so: + +{{name-of-yaml-file-without-file-ext_http_api}} + +List the steps the client needs to take to correctly process this module. Listing what data structures the client should be -storing to aid implementation is recommended. +storing to aid implementation is recommended. Server behaviour ---------------- From 70518ae655eb9fae6ac49a3a1ddea700c7defc44 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 28 Sep 2015 14:03:54 +0100 Subject: [PATCH 164/223] Clarify template suffixes --- specification/modules/_template.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst index 967892a54..bb662c051 100644 --- a/specification/modules/_template.rst +++ b/specification/modules/_template.rst @@ -23,14 +23,16 @@ m.example.event.type ~~~~~~~~~~~~~~~~~~~~ There should be JSON Schema docs for this event. Once there is JSON schema, there will be a template variable with dots in the event type replaced with -underscores. You can insert a template like so: +underscores and the suffix ``_event``. You can insert a template like so: {{m_example_event_type_event}} Client behaviour ---------------- -List any new HTTP endpoints. These endpoints should be docced using Swagger. You -can insert a template for swagger docs like so: +List any new HTTP endpoints. These endpoints should be docced using Swagger. +Once there is Swagger, there will be a template variable based on the name of +the YAML file with the suffix ``_http_api``. You can insert a template for +swagger docs like so: {{name-of-yaml-file-without-file-ext_http_api}} From 5abea1f2bc2d36789164a4682cb8a668fb684845 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 28 Sep 2015 14:07:34 +0100 Subject: [PATCH 165/223] Minor tweaks --- specification/modules/_template.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/specification/modules/_template.rst b/specification/modules/_template.rst index bb662c051..9eee98439 100644 --- a/specification/modules/_template.rst +++ b/specification/modules/_template.rst @@ -15,12 +15,12 @@ Events ------ List the new event types introduced by this module, if any. If there are no new events, this section can be omitted. Event types should be done as -subsections. The section is intended to document the "common shared event +subsections. This section is intended to document the "common shared event structure" between client and server. Deviations from this shared structure should be documented in the relevant behaviour section. -m.example.event.type -~~~~~~~~~~~~~~~~~~~~ +``m.example.event.type`` +~~~~~~~~~~~~~~~~~~~~~~~~ There should be JSON Schema docs for this event. Once there is JSON schema, there will be a template variable with dots in the event type replaced with underscores and the suffix ``_event``. You can insert a template like so: @@ -29,7 +29,7 @@ underscores and the suffix ``_event``. You can insert a template like so: Client behaviour ---------------- -List any new HTTP endpoints. These endpoints should be docced using Swagger. +List any new HTTP endpoints. These endpoints should be documented using Swagger. Once there is Swagger, there will be a template variable based on the name of the YAML file with the suffix ``_http_api``. You can insert a template for swagger docs like so: @@ -37,14 +37,15 @@ swagger docs like so: {{name-of-yaml-file-without-file-ext_http_api}} List the steps the client needs to take to -correctly process this module. Listing what data structures the client should be -storing to aid implementation is recommended. +correctly process this module. List what data structures the client should be +storing in order to aid implementation. Server behaviour ---------------- Does the server need to handle any of the new events in a special way (e.g. typing timeouts, presence). Advice on how to persist events and/or requests are -recommended to aid implementation. +recommended to aid implementation. Federation-specific logic should be included +here. Security considerations ----------------------- From 9dd3b073948740b202e7301624eef6beaebd27c0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 14:00:56 +0100 Subject: [PATCH 166/223] Allow '.' characters in event field names to be escaped with '\' so that fields including a '.' can be included in a filter. I considered replacing '.' with '/'. Since '/' was less likely to appear in event field names. However if we used '\' to escape a literal '/' we risk confusing it with the JSON escape '\/'. --- api/client-server/v2_alpha/definitions/sync_filter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index 4ad3aa82d..b4f87e5fd 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -39,7 +39,7 @@ "event_fields": { "type": "array", "description": - "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A server may include more fields than were requested.", + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A literal '.' character in a field name may be escaped using a '\\'. A server may include more fields than were requested.", "items": { "type": "string" } From 6ad6c40147cf5d80801aa076f76dad2bd54d609e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 14:11:45 +0100 Subject: [PATCH 167/223] List the allowed values for the 'event_format' as an enum --- api/client-server/v2_alpha/definitions/sync_filter.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index b4f87e5fd..b909830b5 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -34,7 +34,8 @@ "event_format": { "description": "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", - "type": "string" + "type": "string", + "enum": ["client", "federation"] }, "event_fields": { "type": "array", From 4776e0c04ce70a6a9bc11ec24e3b65c38049573d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 14:49:55 +0100 Subject: [PATCH 168/223] Add creation_content key --- specification/1-client_server_api.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 93e3cb907..260832c9f 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -750,6 +750,16 @@ options which can be set when creating a room: This will tell the server to invite everyone in the list to the newly created room. +``creation_content`` + Type: + Object + Optional: + Yes + Value: + Extra keys to be added to the content of the ``m.room.create`` + Description: + Allows clients to add keys to the content of ``m.room.create``. + Example:: { From ce53a171800d765b9bcc86e019ec11d4583ed4ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 14:51:10 +0100 Subject: [PATCH 169/223] Add txn_id rationale --- specification/1-client_server_api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 988f7bc44..89698c663 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -251,6 +251,13 @@ request. The same ``nonce`` should be used if retrying the request. There are many ways a client may receive a ``token``, including via an email or from an existing logged in device. +The ``txn_id`` may be used by the server to disallow other devices from using +the token, thus providing "single use" tokens while still allowing the device +to retry the request. This would be done by tying the token to the ``txn_id`` +server side, as well as potentially invalidating the token completely once the +device has successfully logged in (e.g. when we receive a request from the +newly provisioned access_token). + OAuth2-based ~~~~~~~~~~~~ :Type: From 6e6bc8a5a9745627eb172dd98c77c140e89f1a95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 14:51:44 +0100 Subject: [PATCH 170/223] Mandate macaroon --- specification/1-client_server_api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 89698c663..893aec734 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -258,6 +258,8 @@ server side, as well as potentially invalidating the token completely once the device has successfully logged in (e.g. when we receive a request from the newly provisioned access_token). +The ``token`` must be a macaroon. + OAuth2-based ~~~~~~~~~~~~ :Type: From affc2cfc923222da6a10bd6c21dd7d24c8031747 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 16:39:54 +0100 Subject: [PATCH 171/223] Add 'm.federate' flag to 'm.room.create; --- event-schemas/schema/v1/m.room.create | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index f2e1ee92a..bcbb10d25 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -12,6 +12,10 @@ "creator": { "type": "string", "description": "The ``user_id`` of the room creator. This is set by the homeserver." + }, + "m.federate": { + "type": "boolean", + "description": "Whether users on other servers can join this room." } }, "required": ["creator"] From 50e1b4c3a7147cb2a03fab56ff0403b998ef559e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Sep 2015 09:17:33 +0100 Subject: [PATCH 172/223] Fix up rst --- api/client-server/v1/sync.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v1/sync.yaml b/api/client-server/v1/sync.yaml index dbfa39b80..8fe30056d 100644 --- a/api/client-server/v1/sync.yaml +++ b/api/client-server/v1/sync.yaml @@ -270,9 +270,9 @@ paths: invite: type: object title: "InviteEvent" - description: "The invite event if `membership` is `invite`" + description: "The invite event if ``membership`` is ``invite``" allOf: - - "$ref": "m.room.member" + - "$ref": "v1-event-schema/m.room.member" messages: type: object title: PaginationChunk From 083a76096f0edf6bf7810bd4f617d532499367ae Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 29 Sep 2015 12:41:07 +0100 Subject: [PATCH 173/223] Support viewing the spec at head --- scripts/speculator/main.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index b4fe2aef0..2609fa7ec 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -136,20 +136,30 @@ func generateAt(repo, sha string) (dst string, err error) { } func serveSpec(w http.ResponseWriter, req *http.Request) { - pr, err := lookupPullRequest(*req.URL, "/spec") - if err != nil { - writeError(w, 400, err) - return - } + var cloneURL string + var sha string + + if strings.ToLower(req.URL.Path) == "/spec/head" { + cloneURL = "https://github.com/matrix-org/matrix-doc.git" + sha = "HEAD" + } else { + pr, err := lookupPullRequest(*req.URL, "/spec") + if err != nil { + writeError(w, 400, err) + return + } - // We're going to run whatever Python is specified in the pull request, which - // may do bad things, so only trust people we trust. - if err := checkAuth(pr); err != nil { - writeError(w, 403, err) - return + // We're going to run whatever Python is specified in the pull request, which + // may do bad things, so only trust people we trust. + if err := checkAuth(pr); err != nil { + writeError(w, 403, err) + return + } + cloneURL = pr.Head.Repo.CloneURL + sha = pr.Head.SHA } - dst, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + dst, err := generateAt(cloneURL, sha) defer os.RemoveAll(dst) if err != nil { writeError(w, 500, err) @@ -287,7 +297,7 @@ func listPulls(w http.ResponseWriter, req *http.Request) { s += fmt.Sprintf(`
  • %d: %s: %s: spec spec diff rst diff
    • `, pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number) } - s += "" + s += `` io.WriteString(w, s) } From 7b4c8a9f68624e106f3b0ee8aca502d98b88d615 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Sep 2015 16:19:27 +0100 Subject: [PATCH 174/223] Indicate default for m.federate key --- event-schemas/schema/v1/m.room.create | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index bcbb10d25..348b6d86e 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -15,7 +15,7 @@ }, "m.federate": { "type": "boolean", - "description": "Whether users on other servers can join this room." + "description": "Whether users on other servers can join this room. Defaults to ``true`` if key does not exist." } }, "required": ["creator"] From 83b9497664f030ac2e1d3bae1a5faddb8425250f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Sep 2015 16:21:10 +0100 Subject: [PATCH 175/223] Add context and example --- specification/1-client_server_api.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 260832c9f..dd58b357a 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,7 +756,8 @@ options which can be set when creating a room: Optional: Yes Value: - Extra keys to be added to the content of the ``m.room.create`` + Extra keys to be added to the content of the ``m.room.create``. The server + will clober certain keys, e.g. ``creator``. Description: Allows clients to add keys to the content of ``m.room.create``. @@ -766,7 +767,10 @@ Example:: "visibility": "public", "room_alias_name": "thepub", "name": "The Grand Duke Pub", - "topic": "All about happy hour" + "topic": "All about happy hour", + "creation_content": { + "m.federate": false + } } The home server will create a ``m.room.create`` event when the room is created, From 82c27884baed11d5dd61bb84786ce05d5c5e47c3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 29 Sep 2015 16:29:16 +0100 Subject: [PATCH 176/223] Make E2E optional --- specification/0-feature_profiles.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 86f8f3d8a..01997dd53 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -17,7 +17,7 @@ Summary ============================ ===== =========== ======== ========= ===== ===== Module / Profile Web Embed-Web Mobile Desktop CLI IoT ============================ ===== =========== ======== ========= ===== ===== - `End-to-End Encryption`_ YES YES YES YES + `End-to-End Encryption`_ `Instant Messaging`_ YES YES YES YES YES YES `Presence`_ YES YES YES YES `Push Notifications`_ YES From cdf9f011e9eb85cde0840548b1b9a1e5b7885b4e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 29 Sep 2015 16:33:34 +0100 Subject: [PATCH 177/223] Add room history visibility as a module. --- specification/0-feature_profiles.rst | 2 ++ specification/modules/history_visibility.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 01997dd53..59e015e9f 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -25,6 +25,7 @@ Summary `Typing Notifications`_ YES YES YES YES `VoIP`_ YES YES YES `Content Repository`_ YES YES YES YES + `History Visibility`_ YES YES YES YES ============================ ===== =========== ======== ========= ===== ===== *Please see each module for more details on what clients need to implement.* @@ -37,6 +38,7 @@ Summary .. _Typing Notifications: `module:typing`_ .. _VoIP: `module:voip`_ .. _Content Repository: `module:content`_ +.. _History Visibility: `module:history-visibility`_ Clients ------- diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 01c2e419a..371282bd8 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -1,6 +1,8 @@ Room History Visibility ----------------------- +.. _module:history-visibility: + Whether a member of a room can see the events that happened in a room from before they joined the room is controlled by the ``history_visibility`` key of the ``m.room.history_visibility`` state event. The valid values for From 0a04672d764e070fc0199c958b8c006498314fbf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 29 Sep 2015 17:57:44 +0100 Subject: [PATCH 178/223] Start converting the presence module. Add Rationale admonition. --- scripts/nature.css | 6 ++ specification/1-client_server_api.rst | 21 +++++ specification/modules/presence.rst | 114 ++++++++++++++++---------- 3 files changed, 97 insertions(+), 44 deletions(-) diff --git a/scripts/nature.css b/scripts/nature.css index 9d67f689b..f1b4edece 100644 --- a/scripts/nature.css +++ b/scripts/nature.css @@ -282,3 +282,9 @@ td[colspan]:not([colspan="1"]) { thead { background: #eeeeee; } + +div.admonition-rationale { + background-color: #efe; + border: 1px solid #ccc; +} + diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 893aec734..4403e0d3e 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1092,6 +1092,27 @@ Profiles {{profile_http_api}} +Events on Change of Profile Information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields cause an automatic +propagation event to occur, informing likely-interested parties of the new +values. This change is conveyed using two separate mechanisms: + + - a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. + - a ``m.presence`` presence status update is sent, again containing the new + values of the ``displayname`` and ``avatar_url`` keys, in addition to the + required ``presence`` key containing the current presence state of the user. + +Both of these should be done automatically by the home server when a user +successfully changes their display name or avatar URL fields. + +Additionally, when home servers emit room membership events for their own +users, they should include the display name and avatar URL fields in these +events so that clients already have these details to hand, and do not have to +perform extra round trips to query it. + Security -------- diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index ddd2adff2..4c84359ad 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,63 +1,89 @@ Presence ======== -Each user has the concept of presence information. This encodes the -"availability" of that user, suitable for display on other user's clients. +Each user has presence information associated with them. This encodes the +"availability" of that user, suitable for display on other clients. This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. The basic piece of presence -information is represented by the ``presence`` key, which is an enum of one -of the following: +which are sent *outside the context of a room*. Their presence state is +represented by the ``presence`` key, which is an enum of one of the following: - ``online`` : The default state when the user is connected to an event stream. - - ``unavailable`` : The user is not reachable at this time. - - ``offline`` : The user is not connected to an event stream. + - ``unavailable`` : The user is not reachable at this time e.g. they are + idle. + - ``offline`` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. - - ``hidden`` : Behaves as offline, but allows the user to see the client - state anyway and generally interact with client features. (Not yet - implemented in synapse). - -In addition, the server maintains a timestamp of the last time it saw a -pro-active event from the user; either sending a message to a room, or -changing presence state from a lower to a higher level of availability -(thus: changing state from ``unavailable`` to ``online`` counts as a -proactive event, whereas in the other direction it will not). This timestamp -is presented via a key called ``last_active_ago``, which gives the relative -number of milliseconds since the message is generated/emitted that the user -was last seen active. Events ------ {{presence_events}} -Presence HTTP API ------------------ -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them +Client behaviour +---------------- + +Clients can manually set/get their presence using the HTTP APIs listed below. {{presence_http_api}} - -Events on Change of Profile Information ---------------------------------------- -Because the profile displayname and avatar information are likely to be used in -many places of a client's display, changes to these fields cause an automatic -propagation event to occur, informing likely-interested parties of the new -values. This change is conveyed using two separate mechanisms: - - - a ``m.room.member`` event is sent to every room the user is a member of, - to update the ``displayname`` and ``avatar_url``. - - a ``m.presence`` presence status update is sent, again containing the new values of the - ``displayname`` and ``avatar_url`` keys, in addition to the required - ``presence`` key containing the current presence state of the user. - -Both of these should be done automatically by the home server when a user -successfully changes their displayname or avatar URL fields. - -Additionally, when home servers emit room membership events for their own -users, they should include the displayname and avatar URL fields in these -events so that clients already have these details to hand, and do not have to -perform extra roundtrips to query it. +Idle timeout +~~~~~~~~~~~~ + +Clients SHOULD implement an "idle timeout". This is a timer which fires after +a period of inactivity on the client. The definition of inactivity varies +depending on the client. For example, web implementations may determine +inactivity to be not moving the mouse for a certain period of time. When this +timer fires it should set the presence state to ``unavailable``. When the user +becomes active again (e.g. by moving the mouse) the client should set the +presence state to ``online``. A timeout value between 1 and 5 minutes is +recommended. + +Server behaviour +---------------- + +Propagating profile information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields SHOULD cause an +automatic propagation event to occur, informing likely-interested parties of the +new values. One of these change mechanisms SHOULD be via ``m.presence`` events. +These events should set ``displayname`` and ``avatar_url`` to the new values +along with the presence-specific keys. This SHOULD be done automatically by the +home server when a user successfully changes their display name or avatar URL. + +.. admonition:: Rationale + + The intention for sending this information in ``m.presence`` is so that any + "user list" can display the *current* name/presence for a user ID outside the + scope of a room (e.g. a user page which has a "start conversation" button). + This is bundled into a single event to avoid "flickering" on this page which + can occur if you received presence first and then display name later (the + user's name would flicker from their user ID to the display name). + + +Last active ago +~~~~~~~~~~~~~~~ +The server maintains a timestamp of the last time it saw a +pro-active event from the user. A pro-active event may be sending a message to a +room or changing presence state to a higher level of availability. Levels of +availability are defined from low to high as follows: + + - ``offline`` + - ``unavailable`` + - ``online`` + - ``free_for_chat`` + +Based on this list, changing state from ``unavailable`` to ``online`` counts as +a pro-active event, whereas ``online`` to ``unavailable`` does not. This +timestamp is presented via a key called ``last_active_ago`` which gives the +relative number of milliseconds since the pro-active event. + +Security considerations +----------------------- + +Presence information is shared with all users who share a room with the target +user. In large public rooms this could be undesirable. From 218cf94ead2dacce3b3f4443250465ef2f23fd24 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 29 Sep 2015 19:10:31 +0100 Subject: [PATCH 179/223] Replace the events array with events_map inside the room objects. Only use indirection for the state and timeline events. Use event_ids to reference the state and timeline events. --- .../v2_alpha/definitions/event_batch.json | 4 +- .../definitions/room_event_batch.json | 12 ++ .../v2_alpha/definitions/timeline_batch.json | 2 +- api/client-server/v2_alpha/sync.yaml | 134 ++++++++++-------- 4 files changed, 90 insertions(+), 62 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/room_event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index ed2730c87..75762d758 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -3,9 +3,9 @@ "properties": { "events": { "type": "array", - "description": "List of indices into an events array", + "description": "List of events", "items": { - "type": "integer" + "type": "object" } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json new file mode 100644 index 000000000..fcf148f36 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of event ids", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index dc62724bb..ddf8d3416 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/event_batch.json"}], + "allOf": [{"$ref":"definitions/room_event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 0fe9173d9..1df7dace1 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -71,13 +71,6 @@ paths: description: |- The batch token to supply in the ``since`` param of the next ``/sync`` request. - events: - type: array - description: |- - A list of event objects that are referred to by index in the - rest of the ``sync`` response. - items: - type: object rooms: type: array description: |- @@ -89,11 +82,20 @@ paths: type: string description: |- The ID of the room. + event_map: + type: object + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object state: description: |- The state updates for the room. allOf: - - $ref: "definitions/event_batch.json" + - $ref: "definitions/room_event_batch.json" timeline: description: |- The timeline of messages and state changes in the room. @@ -120,62 +122,76 @@ paths: application/json: |- { "next_batch": "s72595_4483_1934", - "events": [ - { - "sender": "@bob:example.com", - "type": "com.example.weird.setting", - "content": {"setting1": true, "setting2": false} - }, - { - "sender": "@alice:example.com", - "type": "m.profile.display_name", - "content": {"display_name": "Alice"} - }, - { - "sender": "@alice:example.com", - "type": "m.presence", - "content": {"presence": "online"} - }, - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - }, - { - "sender": "@alice:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - { - "sender": "@bob:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - { - "sender": "@alice:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 - } - ], - "private_user_data": {"events": [0]}, - "public_user_data": {"events": [1, 2]}, + "private_user_data": { + "events": [ + { + "sender": "@bob:example.com", + "type": "com.example.weird.setting", + "content": {"setting1": true, "setting2": false} + } + ] + }, + "public_user_data": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.profile.display_name", + "content": {"display_name": "Alice"} + }, + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + } + ] + }, "rooms": [{ "room_id": "!726s6s6q:example.com", - "ephemeral": {"events": [3]}, - "state": {"events": [4, 5]}, + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, "timeline": { - "events": [5,6], + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], "limited": true, "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] } }] } From 4cb3f78d2bc8205c87e566e16a049732c2f3bad5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 29 Sep 2015 19:48:48 +0100 Subject: [PATCH 180/223] Wrap the "rooms" list inside an object so that we can add keys for pagination later --- api/client-server/v2_alpha/sync.yaml | 165 ++++++++++++++------------- 1 file changed, 85 insertions(+), 80 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 1df7dace1..58a2c4956 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -72,41 +72,44 @@ paths: The batch token to supply in the ``since`` param of the next ``/sync`` request. rooms: - type: array - description: |- - A list of rooms that the client needs to update. - items: - type: object - properties: - room_id: - type: string - description: |- - The ID of the room. - event_map: + type: object + properties: + roomlist: + type: array + description: |- + A list of rooms that the client needs to update. + items: type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" + properties: + room_id: + type: string + description: |- + The ID of the room. + event_map: + type: object + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" public_user_data: description: |- The updates to publicly visible user data. @@ -145,53 +148,55 @@ paths: } ] }, - "rooms": [{ - "room_id": "!726s6s6q:example.com", - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 + "rooms": { + "roomlist": [{ + "room_id": "!726s6s6q:example.com", + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] } - }, - "state": { - "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" - ] - }, - "timeline": { - "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" - ], - "limited": true, - "prev_batch": "t34-23535_0_0" - }, - "ephemeral": { - "events": [ - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - } - ] - } - }] + }] + } } From 73b4090f52f324c598db699899b22eed72503a3d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 10:22:12 +0100 Subject: [PATCH 181/223] Add private_chat_shared_power --- specification/1-client_server_api.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index c5453931a..329a87ddb 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,15 +756,18 @@ options which can be set when creating a room: Optional: Yes Value: - ``private_chat`` or ``public_chat`` + ``private_chat``, ``private_chat_shared_power`` or ``public_chat`` Description: Convenience parameter for setting various default state events based on a preset. - Two presets are defined: + Three presets are defined: - ``private_chat``: Sets the ``join_rules`` to ``invite`` and ``history_visibility`` to ``shared`` + - ``private_chat_shared_power``: Set the ``join_rules`` to + ``invite``, ``history_visibility`` to ``shared`` and gives all invitees + the same power level as the creator. - ``public_chat``: Sets the ``join_rules`` to ``public`` and ``history_visibility`` to ``shared`` From 069e4e39f4d326ef9e4188159767a1194db1ae76 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 10:22:51 +0100 Subject: [PATCH 182/223] Move presence specific sections from intro to presence module --- specification/0-intro.rst | 43 ------------------------------ specification/modules/presence.rst | 35 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 48 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index a5196e990..1b4e8672f 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -338,49 +338,6 @@ Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. -Presence -~~~~~~~~ - -Each user has the concept of presence information. This encodes: - - * Whether the user is currently online - * How recently the user was last active (as seen by the server) - * Whether a given client considers the user to be currently idle - * Arbitrary information about the user's current status (e.g. "in a meeting"). - -This information is collated from both per-device (online; idle; last_active) and -per-user (status) data, aggregated by the user's homeserver and transmitted as -an ``m.presence`` event. This is one of the few events which are sent *outside -the context of a room*. Presence events are sent to all users who subscribe to -this user's presence through a presence list or by sharing membership of a room. - -.. TODO - How do we let users hide their presence information? - -.. TODO - The last_active specifics should be moved to the detailed presence event section - -Last activity is tracked by the server maintaining a timestamp of the last time -it saw a pro-active event from the user. Any event which could be triggered by a -human using the application is considered pro-active (e.g. sending an event to a -room). An example of a non-proactive client activity would be a client setting -'idle' presence status, or polling for events. This timestamp is presented via a -key called ``last_active_ago``, which gives the relative number of milliseconds -since the message is generated/emitted that the user was last seen active. - -N.B. in v1 API, status/online/idle state are muxed into a single 'presence' -field on the ``m.presence`` event. - -Presence Lists -~~~~~~~~~~~~~~ - -Each user's home server stores a "presence list". This stores a list of user IDs -whose presence the user wants to follow. - -To be added to this list, the user being added must be invited by the list owner -and accept the invitation. Once accepted, both user's HSes track the -subscription. - Profiles ~~~~~~~~ diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 4c84359ad..1a359e5a6 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,5 +1,23 @@ Presence ======== + +Each user has the concept of presence information. This encodes: + + * Whether the user is currently online + * How recently the user was last active (as seen by the server) + * Whether a given client considers the user to be currently idle + * Arbitrary information about the user's current status (e.g. "in a meeting"). + +This information is collated from both per-device (``online``, ``idle``, +``last_active``) and per-user (status) data, aggregated by the user's homeserver +and transmitted as an ``m.presence`` event. This is one of the few events which +are sent *outside the context of a room*. Presence events are sent to all users +who subscribe to this user's presence through a presence list or by sharing +membership of a room. + +A presence list is a list of user IDs whose presence the user wants to follow. +To be added to this list, the user being added must be invited by the list owner +who must accept the invitation. Each user has presence information associated with them. This encodes the "availability" of that user, suitable for display on other clients. @@ -15,6 +33,7 @@ represented by the ``presence`` key, which is an enum of one of the following: explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. + Events ------ @@ -24,7 +43,8 @@ Events Client behaviour ---------------- -Clients can manually set/get their presence using the HTTP APIs listed below. +Clients can manually set/get their presence/presence list using the HTTP APIs +listed below. {{presence_http_api}} @@ -43,6 +63,9 @@ recommended. Server behaviour ---------------- +Each user's home server stores a "presence list" per user. Once a user accepts +a presence list, both user's HSes must track the subscription. + Propagating profile information ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -58,10 +81,12 @@ home server when a user successfully changes their display name or avatar URL. The intention for sending this information in ``m.presence`` is so that any "user list" can display the *current* name/presence for a user ID outside the - scope of a room (e.g. a user page which has a "start conversation" button). - This is bundled into a single event to avoid "flickering" on this page which - can occur if you received presence first and then display name later (the - user's name would flicker from their user ID to the display name). + scope of a room e.g. for a user page. This is bundled into a single event for + several reasons. The user's display name can change per room. This + event provides the "canonical" name for the user. In addition, the name is + bundled into a single event for the ease of client implementations. If this + was not done, the client would need to search all rooms for their own + membership event to pull out the display name. Last active ago From 931057accfdd2756686c3c896a22157b3f5faf51 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 30 Sep 2015 10:30:39 +0100 Subject: [PATCH 183/223] Add a top level presence key for the presence events and remove the public_user_data/private_user_data for now --- .../v2_alpha/definitions/sync_filter.json | 9 ++----- api/client-server/v2_alpha/filter.yaml | 22 +++++----------- api/client-server/v2_alpha/sync.yaml | 26 +++---------------- 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index b909830b5..0cd6a7986 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -21,14 +21,9 @@ } } }, - "public_user_data": { + "presence": { "description": - "The public user data, e.g. profile and presence, to include.", - "allOf": [{"$ref": "definitions/event_filter.json"}] - }, - "private_user_data": { - "description": - "Events that are private to a user but shared amoungst their devices, e.g. notification settings, to include.", + "The presence updates to include.", "allOf": [{"$ref": "definitions/event_filter.json"}] }, "event_format": { diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 9bda2359d..30d528447 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -62,14 +62,9 @@ paths: "not_senders": ["@spam:example.com"] } }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] - }, - "server_data": { - "types": [] + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] }, "event_format": "client", "event_fields": ["type", "content", "sender"] @@ -131,14 +126,9 @@ paths: "not_senders": ["@spam:example.com"] } }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] - }, - "server_data": { - "types": [] + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] }, "event_format": "client", "event_fields": ["type", "content", "sender"] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 58a2c4956..238fab630 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -110,37 +110,17 @@ paths: in the timeline or state of the room. E.g. typing. allOf: - $ref: "definitions/event_batch.json" - public_user_data: + presence: description: |- - The updates to publicly visible user data. - allOf: - - $ref: "definitions/event_batch.json" - private_user_data: - description: |- - Updates to the data which is private to the user but shared - amongst their devices. + The updates to the presence status of other users. allOf: - $ref: "definitions/event_batch.json" examples: application/json: |- { "next_batch": "s72595_4483_1934", - "private_user_data": { + "presence": { "events": [ - { - "sender": "@bob:example.com", - "type": "com.example.weird.setting", - "content": {"setting1": true, "setting2": false} - } - ] - }, - "public_user_data": { - "events": [ - { - "sender": "@alice:example.com", - "type": "m.profile.display_name", - "content": {"display_name": "Alice"} - }, { "sender": "@alice:example.com", "type": "m.presence", From 4e1e82f995f899d397ac5a21194a48973e262948 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 10:43:02 +0100 Subject: [PATCH 184/223] Fix paragraph CSS and adjust where paragraphs are in the RST The CSS for `nature.css` was such that it was preventing `p` tags from having sufficient vertical whitespace. This meant that you couldn't insert any kind of spacing between lengthy sections (they just appeared as new lines). This PR fixes this so you can actually have some whitespace between paragraphs. As a result of this change, some parts of the spec appeared to have too much whitespace. These were often sections which shouldn't have begun a new paragraph anyway (e.g. a single sentence being an entire paragraph, `TODO` blocks resulting in new paragraphs). This PR fixes the most offending areas where we shouldn't have been inserting new paragraphs. --- scripts/nature.css | 2 +- specification/0-intro.rst | 69 +++++++++------------ specification/1-client_server_api.rst | 34 ++++------ specification/3-application_service_api.rst | 18 +++--- specification/4-server_server_api.rst | 12 ++-- specification/modules/content_repo.rst | 24 +++---- specification/modules/push_cs_api.rst | 17 +++-- specification/modules/voip_events.rst | 9 ++- 8 files changed, 77 insertions(+), 108 deletions(-) diff --git a/scripts/nature.css b/scripts/nature.css index 9d67f689b..ac171923b 100644 --- a/scripts/nature.css +++ b/scripts/nature.css @@ -245,7 +245,7 @@ div.viewcode-block:target { } p { - margin: 0; + /* margin: 0; - setting this leads to no spacing between paragraphs which looks ugly */ } ul li dd { diff --git a/specification/0-intro.rst b/specification/0-intro.rst index a5196e990..93458d97d 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -24,7 +24,6 @@ Introduction The Matrix specification is still evolving: the APIs are not yet frozen and this document is in places a work in progress or stale. We have made every effort to clearly flag areas which are still being finalised. - We're publishing it at this point because it's complete enough to be more than useful and provide a canonical reference to how Matrix is evolving. Our end goal is to mirror WHATWG's `Living Standard @@ -34,10 +33,9 @@ Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice over IP (VoIP) and Internet of Things (IoT) communication, designed to create and support a new global real-time communication ecosystem. The intention is to provide an open decentralised pubsub layer for the internet for securely -persisting and publishing/subscribing JSON objects. - -This specification is the ongoing result of standardising the APIs used by the -various components of the Matrix ecosystem to communicate with one another. +persisting and publishing/subscribing JSON objects. This specification is the +ongoing result of standardising the APIs used by the various components of the +Matrix ecosystem to communicate with one another. The principles that Matrix attempts to follow are: @@ -214,10 +212,8 @@ which have the form:: There is exactly one room ID for each room. Whilst the room ID does contain a domain, it is simply for globally namespacing room IDs. The room does NOT reside on the domain specified. Room IDs are not meant to be human readable. -They are case-sensitive. - -The following conceptual diagram shows an ``m.room.message`` event being sent to -the room ``!qporfwt:matrix.org``:: +They are case-sensitive. The following conceptual diagram shows an +``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: { @alice:matrix.org } { @bob:domain.com } | ^ @@ -258,28 +254,28 @@ the room ``!qporfwt:matrix.org``:: Federation maintains *shared data structures* per-room between multiple home servers. The data is split into ``message events`` and ``state events``. -``Message events`` describe transient 'once-off' activity in a room such as an -instant messages, VoIP call setups, file transfers, etc. They generally describe -communication activity. +Message events: + These describe transient 'once-off' activity in a room such as an + instant messages, VoIP call setups, file transfers, etc. They generally + describe communication activity. -``State events`` describe updates to a given piece of persistent information -('state') related to a room, such as the room's name, topic, membership, -participating servers, etc. State is modelled as a lookup table of key/value -pairs per room, with each key being a tuple of ``state_key`` and ``event type``. -Each state event updates the value of a given key. +State events: + These describe updates to a given piece of persistent information + ('state') related to a room, such as the room's name, topic, membership, + participating servers, etc. State is modelled as a lookup table of key/value + pairs per room, with each key being a tuple of ``state_key`` and ``event type``. + Each state event updates the value of a given key. The state of the room at a given point is calculated by considering all events preceding and including a given event in the graph. Where events describe the same state, a merge conflict algorithm is applied. The state resolution algorithm is transitive and does not depend on server state, as it must consistently select the same event irrespective of the server or the order the -events were received in. - -Events are signed by the originating server (the signature includes the parent -relations, type, depth and payload hash) and are pushed over federation to the -participating servers in a room, currently using full mesh topology. Servers may -also request backfill of events over federation from the other servers -participating in a room. +events were received in. Events are signed by the originating server (the +signature includes the parent relations, type, depth and payload hash) and are +pushed over federation to the participating servers in a room, currently using +full mesh topology. Servers may also request backfill of events over federation +from the other servers participating in a room. Room Aliases @@ -324,12 +320,10 @@ Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix users. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces *linked* to their account. - Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social network accounts and phone numbers to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts. - In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. @@ -410,6 +404,10 @@ dedicated API. The API is symmetrical to managing Profile data. API Standards ------------- +.. TODO + Need to specify any HMAC or access_token lifetime/ratcheting tricks + We need to specify capability negotiation for extensible transports + The mandatory baseline for communication in Matrix is exchanging JSON objects over HTTP APIs. HTTPS is mandated as the baseline for server-server (federation) communication. HTTPS is recommended for client-server @@ -417,20 +415,11 @@ communication, although HTTP may be supported as a fallback to support basic HTTP clients. More efficient optional transports for client-server communication will in future be supported as optional extensions - e.g. a packed binary encoding over stream-cipher encrypted TCP socket for -low-bandwidth/low-roundtrip mobile usage. - -.. TODO - We need to specify capability negotiation for extensible transports - -For the default HTTP transport, all API calls use a Content-Type of -``application/json``. In addition, all strings MUST be encoded as UTF-8. - -Clients are authenticated using opaque ``access_token`` strings (see -`Client Authentication`_ for details), passed as a query string parameter on -all requests. - -.. TODO - Need to specify any HMAC or access_token lifetime/ratcheting tricks +low-bandwidth/low-roundtrip mobile usage. For the default HTTP transport, all +API calls use a Content-Type of ``application/json``. In addition, all strings +MUST be encoded as UTF-8. Clients are authenticated using opaque +``access_token`` strings (see `Client Authentication`_ for details), passed as a +query string parameter on all requests. Any errors which occur at the Matrix API level MUST return a "standard error response". This is a JSON object which looks like:: diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 893aec734..7d8d57e40 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -672,13 +672,10 @@ to add keys that are, for example offensive or illegal. Since some events cannot be simply deleted, e.g. membership events, we instead 'redact' events. This involves removing all keys from an event that are not required by the protocol. This stripped down event is thereafter returned anytime a client or -remote server requests it. - -Events that have been redacted include a ``redacted_because`` key whose value -is the event that caused it to be redacted, which may include a reason. - -Redacting an event cannot be undone, allowing server owners to delete the -offending content from the databases. +remote server requests it. Redacting an event cannot be undone, allowing server +owners to delete the offending content from the databases. Events that have been +redacted include a ``redacted_because`` key whose value is the event that caused +it to be redacted, which may include a reason. .. TODO Currently, only room admins can redact events by sending a ``m.room.redaction`` @@ -709,12 +706,10 @@ one of the following event types: .. TODO Need to update m.room.power_levels to reflect new power levels formatting -The redaction event should be added under the key ``redacted_because``. - -When a client receives a redaction event it should change the redacted event +The redaction event should be added under the key ``redacted_because``. When a +client receives a redaction event it should change the redacted event in the same way a server does. - Rooms ----- @@ -853,18 +848,14 @@ Permissions Permissions for rooms are done via the concept of power levels - to do any action in a room a user must have a suitable power level. Power levels are -stored as state events in a given room. - -The power levels required for operations and the power levels for users are -defined in ``m.room.power_levels``, where both a default and specific users' -power levels can be set. - +stored as state events in a given room. The power levels required for operations +and the power levels for users are defined in ``m.room.power_levels``, where +both a default and specific users' power levels can be set. By default all users have a power level of 0, other than the room creator whose power level defaults to 100. Users can grant other users increased power levels up to their own power level. For example, user A with a power level of 50 could increase the power level of user B to a maximum of level 50. Power levels for users are tracked per-room even if the user is not present in the room. - The keys contained in ``m.room.power_levels`` determine the levels required for certain operations such as kicking, banning and sending state events. See `m.room.power_levels`_ for more information. @@ -881,10 +872,9 @@ room. There are several states in which a user may be, in relation to a room: - Banned (the user is not allowed to join the room) Some rooms require that users be invited to it before they can join; others -allow anyone to join. - -Whether a given room is an "invite-only" room is determined by the room config -key ``m.room.join_rules``. It can have one of the following values: +allow anyone to join. Whether a given room is an "invite-only" room is +determined by the room config key ``m.room.join_rules``. It can have one of the +following values: ``public`` This room is free for anyone to join without an invite. diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index e982390b4..bdde77894 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -4,11 +4,9 @@ Application Service API The Matrix client-server API and server-server APIs provide the means to implement a consistent self-contained federated messaging fabric. However, they provide limited means of implementing custom server-side behaviour in Matrix -(e.g. gateways, filters, extensible hooks etc). - -The Application Service API defines a standard API to allow such extensible -functionality to be implemented irrespective of the underlying homeserver -implementation. +(e.g. gateways, filters, extensible hooks etc). The Application Service API +defines a standard API to allow such extensible functionality to be implemented +irrespective of the underlying homeserver implementation. .. TODO-spec Add in Client-Server services? Overview of bots? Seems weird to be in the spec @@ -18,12 +16,10 @@ Passive Application Services ---------------------------- "Passive" application services can only observe events from a given home server. They cannot prevent events from being sent, nor can they modify the content of -the event being sent. - -In order to observe events from a homeserver, the homeserver needs to be -configured to pass certain types of traffic to the application service. This -is achieved by manually configuring the homeserver with information about the -AS.. +the event being sent. In order to observe events from a homeserver, the +homeserver needs to be configured to pass certain types of traffic to the +application service. This is achieved by manually configuring the homeserver +with information about the AS. .. NOTE:: Previously, application services could register with a homeserver via HTTP diff --git a/specification/4-server_server_api.rst b/specification/4-server_server_api.rst index 8d1f8898f..c5ff2b877 100644 --- a/specification/4-server_server_api.rst +++ b/specification/4-server_server_api.rst @@ -59,13 +59,11 @@ and an optional TLS port. .. ** If the port is present then the server is discovered by looking up an AAAA or -A record for the DNS name and connecting to the specified TLS port. - -If the port is absent then the server is discovered by looking up a -``_matrix._tcp`` SRV record for the DNS name. If this record does not exist -then the server is discovered by looking up an AAAA or A record on the DNS -name and taking the default fallback port number of 8448. - +A record for the DNS name and connecting to the specified TLS port. If the port +is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV +record for the DNS name. If this record does not exist then the server is +discovered by looking up an AAAA or A record on the DNS name and taking the +default fallback port number of 8448. Home servers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 2c45ced74..5a3cf9b93 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -56,21 +56,21 @@ Homeservers must never upscale images. Security considerations ----------------------- -Clients may try to upload very large files. Homeservers should not store files -that are too large and should not serve them to clients. + - Clients may try to upload very large files. Homeservers should not store files + that are too large and should not serve them to clients. -Clients may try to upload very large images. Homeservers should not attempt to -generate thumbnails for images that are too large. + - Clients may try to upload very large images. Homeservers should not attempt to + generate thumbnails for images that are too large. -Remote homeservers may host very large files or images. Homeserver should not -proxy or thumbnail large files or images from remote homeservers. + - Remote homeservers may host very large files or images. Homeserver should not + proxy or thumbnail large files or images from remote homeservers. -Clients may try to upload a large number of files. Homeservers should limit the -number and total size of media that can be uploaded by clients. + - Clients may try to upload a large number of files. Homeservers should limit the + number and total size of media that can be uploaded by clients. -Clients may try to access a large number of remote files through a homeserver. -Homeservers should restrict the number and size of remote files that it caches. + - Clients may try to access a large number of remote files through a homeserver. + Homeservers should restrict the number and size of remote files that it caches. -Clients or remote homeservers may try to upload malicious files targeting -vulnerabilities in either the homeserver thumbnailing or the client decoders. + - Clients or remote homeservers may try to upload malicious files targeting + vulnerabilities in either the homeserver thumbnailing or the client decoders. diff --git a/specification/modules/push_cs_api.rst b/specification/modules/push_cs_api.rst index b78b44fde..c63019262 100644 --- a/specification/modules/push_cs_api.rst +++ b/specification/modules/push_cs_api.rst @@ -99,16 +99,13 @@ be redundant. Actions for the highest priority rule and only that rule apply (for example, a set_tweak action in a lower priority rule will not apply if a higher priority rule matches, even if that rule does not specify any tweaks). -Rules also have an identifier, rule_id, which is a string. The rule_id is -unique within the kind of rule and scope: rule_ids need not be unique between -rules of the same kind on different devices. - -A home server may also have server default rules of each kind and in each scope. -Server default rules are lower priority than user-defined rules in each scope. -Server default rules (and only server default rules) begin with a dot ('.') -character. - -In addition, all rules may be enabled or disabled. Disabled rules never match. +Rules also have an identifier, ``rule_id``, which is a string. The ``rule_id`` +is unique within the kind of rule and scope: ``rule_ids`` need not be unique +between rules of the same kind on different devices. A home server may also have +server default rules of each kind and in each scope. Server default rules are +lower priority than user-defined rules in each scope. Server default rules (and +only server default rules) begin with a dot ('.') character. In addition, all +rules may be enabled or disabled. Disabled rules never match. If no rules match an event, the Home Server should not notify for the message (that is to say, the default action is "dont-notify"). Events that the user sent diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index a54682379..33998cd95 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -44,11 +44,10 @@ Glare This specification aims to address the problem of two users calling each other at roughly the same time and their invites crossing on the wire. It is a far better experience for the users if their calls are connected if it is clear -that their intention is to set up a call with one another. - -In Matrix, calls are to rooms rather than users (even if those rooms may only -contain one other user) so we consider calls which are to the same room. The -rules for dealing with such a situation are as follows: +that their intention is to set up a call with one another. In Matrix, calls are +to rooms rather than users (even if those rooms may only contain one other user) +so we consider calls which are to the same room. The rules for dealing with such +a situation are as follows: - If an invite to a room is received whilst the client is preparing to send an invite to the same room, the client should cancel its outgoing call and From a49d5f67f1ee8e06c2c7a3e0baaba5f120f7a4e3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 10:54:06 +0100 Subject: [PATCH 185/223] Delete rather than comment out the offending CSS --- scripts/nature.css | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/nature.css b/scripts/nature.css index ac171923b..7c4af305f 100644 --- a/scripts/nature.css +++ b/scripts/nature.css @@ -244,10 +244,6 @@ div.viewcode-block:target { border-bottom: 1px solid #ac9; } -p { - /* margin: 0; - setting this leads to no spacing between paragraphs which looks ugly */ -} - ul li dd { margin-top: 0; } From ede43fbe9fc59b0f4eff83d7e1e8cc0d69c180a6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 14:31:26 +0100 Subject: [PATCH 186/223] Modify how speculator uses git repos Modified so it has a "master" repo and local A/B repos which pull from the "master". This saves an extra git clone operation per HTTP request. --- scripts/speculator/main.go | 72 +++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 2609fa7ec..6ee80150d 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -24,6 +24,7 @@ import ( "strconv" "strings" "syscall" + "time" ) type PullRequest struct { @@ -58,16 +59,24 @@ func (u *User) IsTrusted() bool { return allowedMembers[u.Login] } -const pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" +const ( + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" +) + + +func gitClone(url string, shared bool) (string, error) { + directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) + cmd := exec.Command("git", "clone", url, directory) + if shared { + cmd.Args = append(cmd.Args, "--shared") + } -func gitClone(url string) (string, error) { - dst := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) - cmd := exec.Command("git", "clone", url, dst) err := cmd.Run() if err != nil { return "", fmt.Errorf("error cloning repo: %v", err) } - return dst, nil + return directory, nil } func gitCheckout(path, sha string) error { @@ -80,6 +89,16 @@ func gitCheckout(path, sha string) error { return nil } +func gitFetch(path string) error { + cmd := exec.Command("git", "fetch") + cmd.Dir = path + err := cmd.Run() + if err != nil { + return fmt.Errorf("error fetching repo: %v", err) + } + return nil +} + func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) { if !strings.HasPrefix(url.Path, pathPrefix+"/") { return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) @@ -119,10 +138,18 @@ func writeError(w http.ResponseWriter, code int, err error) { io.WriteString(w, fmt.Sprintf("%v\n", err)) } +type server struct { + matrixDocCloneURL string +} + // generateAt generates spec from repo at sha. // Returns the path where the generation was done. -func generateAt(repo, sha string) (dst string, err error) { - dst, err = gitClone(repo) +func (s *server) generateAt(sha string) (dst string, err error) { + err = gitFetch(s.matrixDocCloneURL) + if err != nil { + return + } + dst, err = gitClone(s.matrixDocCloneURL, true) if err != nil { return } @@ -135,12 +162,10 @@ func generateAt(repo, sha string) (dst string, err error) { return } -func serveSpec(w http.ResponseWriter, req *http.Request) { - var cloneURL string +func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string if strings.ToLower(req.URL.Path) == "/spec/head" { - cloneURL = "https://github.com/matrix-org/matrix-doc.git" sha = "HEAD" } else { pr, err := lookupPullRequest(*req.URL, "/spec") @@ -155,11 +180,10 @@ func serveSpec(w http.ResponseWriter, req *http.Request) { writeError(w, 403, err) return } - cloneURL = pr.Head.Repo.CloneURL sha = pr.Head.SHA } - dst, err := generateAt(cloneURL, sha) + dst, err := s.generateAt(sha) defer os.RemoveAll(dst) if err != nil { writeError(w, 500, err) @@ -181,7 +205,7 @@ func checkAuth(pr *PullRequest) error { return nil } -func serveRSTDiff(w http.ResponseWriter, req *http.Request) { +func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) { pr, err := lookupPullRequest(*req.URL, "/diff/rst") if err != nil { writeError(w, 400, err) @@ -195,14 +219,14 @@ func serveRSTDiff(w http.ResponseWriter, req *http.Request) { return } - base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA) + base, err := s.generateAt(pr.Base.SHA) defer os.RemoveAll(base) if err != nil { writeError(w, 500, err) return } - head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + head, err := s.generateAt(pr.Head.SHA) defer os.RemoveAll(head) if err != nil { writeError(w, 500, err) @@ -219,7 +243,7 @@ func serveRSTDiff(w http.ResponseWriter, req *http.Request) { w.Write(diff.Bytes()) } -func serveHTMLDiff(w http.ResponseWriter, req *http.Request) { +func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { pr, err := lookupPullRequest(*req.URL, "/diff/html") if err != nil { writeError(w, 400, err) @@ -233,14 +257,14 @@ func serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA) + base, err := s.generateAt(pr.Base.SHA) defer os.RemoveAll(base) if err != nil { writeError(w, 500, err) return } - head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + head, err := s.generateAt(pr.Head.SHA) defer os.RemoveAll(head) if err != nil { writeError(w, 500, err) @@ -327,9 +351,15 @@ func main() { "Kegsay": true, "NegativeMjark": true, } - http.HandleFunc("/spec/", serveSpec) - http.HandleFunc("/diff/rst/", serveRSTDiff) - http.HandleFunc("/diff/html/", serveHTMLDiff) + rand.Seed(time.Now().Unix()) + masterCloneDir, err := gitClone(matrixDocCloneURL, false) + if err != nil { + log.Fatal(err) + } + s := server{masterCloneDir} + http.HandleFunc("/spec/", s.serveSpec) + http.HandleFunc("/diff/rst/", s.serveRSTDiff) + http.HandleFunc("/diff/html/", s.serveHTMLDiff) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) From b6f0b67da66d4d9d286ad6efb641f79583456cbc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 15:21:32 +0100 Subject: [PATCH 187/223] Speed up continuserv Ignore .git directory as that shouldn't affect spec generation. Also, when we receive writes from the OS, wait a bit before re-generating the spec to clump together multiple writes rather than re-generating one after another and waiting for no more writes before serving the request. --- scripts/continuserv/main.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 658ae0fb6..573c2c955 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "sync/atomic" + "time" fsnotify "gopkg.in/fsnotify.v1" ) @@ -67,7 +68,6 @@ func watchFS(ch chan struct{}, w *fsnotify.Watcher) { select { case e := <-w.Events: if filter(e) { - wg.Add(1) fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) ch <- struct{}{} } @@ -98,6 +98,11 @@ func filter(e fsnotify.Event) bool { return false } + // Ignore the .git directory - It's very noisy + if strings.Contains(e.Name, "/.git/") { + return false + } + // Avoid infinite cycles being caused by writing actual output if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") { return false @@ -133,8 +138,20 @@ func populateOnce(dir string) { } func doPopulate(ch chan struct{}, dir string) { - for _ = range ch { - populateOnce(dir) + var pending int + for { + select { + case <-ch: + if pending == 0 { + wg.Add(1) + } + pending++ + case <-time.After(10 * time.Millisecond): + if pending > 0 { + pending = 0 + populateOnce(dir) + } + } } } From 2b7e02c0805b81d1c2ad72245f4dc13bdb56a445 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 11:26:02 +0100 Subject: [PATCH 188/223] Add sections for typing. Add swagger, JSON schema and example m.typing event --- api/client-server/v1/typing.yaml | 77 +++++++++++++++++++ event-schemas/examples/v1/m.typing | 7 ++ event-schemas/schema/v1/m.typing | 28 +++++++ .../modules/typing_notifications.rst | 47 +++++------ 4 files changed, 131 insertions(+), 28 deletions(-) create mode 100644 api/client-server/v1/typing.yaml create mode 100644 event-schemas/examples/v1/m.typing create mode 100644 event-schemas/schema/v1/m.typing diff --git a/api/client-server/v1/typing.yaml b/api/client-server/v1/typing.yaml new file mode 100644 index 000000000..737c69284 --- /dev/null +++ b/api/client-server/v1/typing.yaml @@ -0,0 +1,77 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Typing API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/typing/{userId}": + put: + summary: Informs the server that the user has started or stopped typing. + description: |- + This tells the server that the user is typing for the next N + milliseconds where N is the value specified in the ``timeout`` key. + Alternatively, if ``typing`` is ``false``, it tells the server that the + user has stopped typing. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + description: The user who has started to type. + required: true + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + description: The room in which the user is typing. + required: true + x-example: "!wefh3sfukhs:example.com" + - in: body + name: typingState + description: The current typing state. + required: true + schema: + type: object + example: |- + { + "typing": true, + "timeout": 30000 + } + properties: + typing: + type: boolean + description: |- + Whether the user is typing or not. If ``false``, the ``timeout`` + key can be omitted. + timeout: + type: integer + description: The length of time in milliseconds to mark this user as typing. + required: ["typing"] + responses: + 200: + description: The new typing state was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/v1/m.typing new file mode 100644 index 000000000..bd53f6fb6 --- /dev/null +++ b/event-schemas/examples/v1/m.typing @@ -0,0 +1,7 @@ +{ + "type": "m.typing", + "room_id": "!z0mnsuiwhifuhwwfw:matrix.org", + "content": { + "user_ids": ["@alice:matrix.org", "@bob:example.com"] + } +} \ No newline at end of file diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/v1/m.typing new file mode 100644 index 000000000..b712f6ecb --- /dev/null +++ b/event-schemas/schema/v1/m.typing @@ -0,0 +1,28 @@ +{ + "type": "object", + "title": "Typing Event", + "description": "Informs the client of the list of users currently typing.", + "properties": { + "content": { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The list of user IDs typing in this room, if any." + } + }, + "required": ["user_ids"] + }, + "type": { + "type": "string", + "enum": ["m.typing"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["type", "room_id", "content"] +} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index 25b714ab4..b32e3411e 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,45 +1,30 @@ Typing Notifications --------------------- +==================== -Client APIs -~~~~~~~~~~~ +Events +------ -To set "I am typing for the next N msec":: +{{m_typing_event}} - PUT .../rooms//typing/ - Content: { "typing": true, "timeout": N } - # timeout is in milliseconds; suggested no more than 20 or 30 seconds +Client behaviour +---------------- + + - suggested no more than 20-30 seconds This should be re-sent by the client to continue informing the server the user is still typing; a safety margin of 5 seconds before the expected timeout runs out is recommended. Just keep declaring a new timeout, it will replace the old one. -To set "I am no longer typing":: - - PUT ../rooms//typing/ - Content: { "typing": false } - -Client Events -~~~~~~~~~~~~~ - -All room members will receive an event on the event stream:: - - { - "type": "m.typing", - "room_id": "!room-id-here:matrix.org", - "content": { - "user_ids": ["list of", "every user", "who is", "currently typing"] - } - } - -The client must use this list to *REPLACE* its knowledge of every user who is +Event: The client must use this list to *REPLACE* its knowledge of every user who is currently typing. The reason for this is that the server DOES NOT remember users who are not currently typing, as that list gets big quickly. The client should mark as not typing, any user ID who is not in that list. -Server APIs -~~~~~~~~~~~ +{{typing_http_api}} + +Server behaviour +---------------- Servers will emit EDUs in the following form:: @@ -59,3 +44,9 @@ originating HSes to ensure they eventually send "stop" notifications. ((This will eventually need addressing, as part of the wider typing/presence timer addition work)) +Security considerations +----------------------- + +Clients may not wish to inform everyone in a room that they are typing and +instead only specific users in the room. + From a82f2ad4ac0b622899fcc039b05d2cd14c26e37c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 11:55:34 +0100 Subject: [PATCH 189/223] Flesh out typing module --- specification/0-intro.rst | 2 + .../modules/typing_notifications.rst | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index 93458d97d..5fec59c5c 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -180,6 +180,8 @@ of a "Room". Event Graphs ~~~~~~~~~~~~ +.. _sect:event-graph: + Events exchanged in the context of a room are stored in a directed acyclic graph (DAG) called an ``event graph``. The partial ordering of this graph gives the chronological ordering of events within the room. Each event in the graph has a diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index b32e3411e..110cd1348 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,6 +1,13 @@ Typing Notifications ==================== +Users often desire to see when another user is typing. This can be achieved +using typing notifications. These are ephemeral events scoped to a ``room_id``. +This means they do not form part of the `Event Graph`_ but still have a +``room_id`` key. + +.. _Event Graph: `sect:event-graph`_ + Events ------ @@ -9,24 +16,28 @@ Events Client behaviour ---------------- - - suggested no more than 20-30 seconds - -This should be re-sent by the client to continue informing the server the user -is still typing; a safety margin of 5 seconds before the expected -timeout runs out is recommended. Just keep declaring a new timeout, it will -replace the old one. - -Event: The client must use this list to *REPLACE* its knowledge of every user who is -currently typing. The reason for this is that the server DOES NOT remember -users who are not currently typing, as that list gets big quickly. The client -should mark as not typing, any user ID who is not in that list. +When a client receives an ``m.typing`` event, it MUST use the user ID list to +**REPLACE** its knowledge of every user who is currently typing. The reason for +this is that the server *does not remember* users who are not currently typing +as that list gets big quickly. The client should mark as not typing any user ID +who is not in that list. + +It is recommended that clients store a ``boolean`` indicating whether the user +is typing or not. Whilst this value is ``true`` a timer should fire periodically +every N seconds to send a typing HTTP request. The value of N is recommended to +be no more than 20-30 seconds. This request should be re-sent by the client to +continue informing the server the user is still typing. As subsequent +requests will replace older requests, a safety margin of 5 seconds before the +expected timeout runs out is recommended. When the user stops typing, the +state change of the ``boolean`` to ``false`` should trigger another HTTP request +to inform the server that the user has stopped typing. {{typing_http_api}} Server behaviour ---------------- -Servers will emit EDUs in the following form:: +Servers MUST emit typing EDUs in the following form:: { "type": "m.typing", @@ -37,8 +48,8 @@ Servers will emit EDUs in the following form:: } } -Server EDUs don't (currently) contain timing information; it is up to -originating HSes to ensure they eventually send "stop" notifications. +This does not contain timing information so it is up to originating homeservers +to ensure they eventually send "stop" notifications. .. TODO ((This will eventually need addressing, as part of the wider typing/presence From 097dc501805891c594dc218d6751fb529d2dd6a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 15:45:29 +0100 Subject: [PATCH 190/223] Minor tweaks --- specification/modules/typing_notifications.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index 110cd1348..74bd76bcd 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,10 +1,10 @@ Typing Notifications ==================== -Users often desire to see when another user is typing. This can be achieved -using typing notifications. These are ephemeral events scoped to a ``room_id``. -This means they do not form part of the `Event Graph`_ but still have a -``room_id`` key. +Users may wish to be informed when another user is typing in a room. This can be +achieved using typing notifications. These are ephemeral events scoped to a +``room_id``. This means they do not form part of the `Event Graph`_ but still +have a ``room_id`` key. .. _Event Graph: `sect:event-graph`_ @@ -37,7 +37,8 @@ to inform the server that the user has stopped typing. Server behaviour ---------------- -Servers MUST emit typing EDUs in the following form:: +Servers MUST emit typing EDUs in a different form to ``m.typing`` events which +are shown to clients. This form looks like:: { "type": "m.typing", From 9964dd1401ac0127a8476b14106274c9d84d0fbd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:11:31 +0100 Subject: [PATCH 191/223] Make explicit the state event only includes some keys --- event-schemas/schema/v1/m.room.member | 1 + 1 file changed, 1 insertion(+) diff --git a/event-schemas/schema/v1/m.room.member b/event-schemas/schema/v1/m.room.member index 32302b85f..c0fb103c4 100644 --- a/event-schemas/schema/v1/m.room.member +++ b/event-schemas/schema/v1/m.room.member @@ -39,6 +39,7 @@ "items": { "type": "object", "title": "StateEvent", + "description": "A stripped down state event, with only the ``type``, ``state_key`` and ``content`` keys.", "properties": { "type": { "type": "string" From 0320e8cef32d59ac84131f445937ddc2c05523a3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 16:41:47 +0100 Subject: [PATCH 192/223] Table tweaks from PR --- specification/0-feature_profiles.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 59e015e9f..1d88311b1 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -14,19 +14,19 @@ in order for it to be classified as that profile. Summary ------- -============================ ===== =========== ======== ========= ===== ===== - Module / Profile Web Embed-Web Mobile Desktop CLI IoT -============================ ===== =========== ======== ========= ===== ===== - `End-to-End Encryption`_ - `Instant Messaging`_ YES YES YES YES YES YES - `Presence`_ YES YES YES YES - `Push Notifications`_ YES - `Receipts`_ YES YES YES YES - `Typing Notifications`_ YES YES YES YES - `VoIP`_ YES YES YES - `Content Repository`_ YES YES YES YES - `History Visibility`_ YES YES YES YES -============================ ===== =========== ======== ========= ===== ===== +============================ ========== ========== ========== ========== ========== + Module / Profile Web Mobile Desktop CLI Embedded +============================ ========== ========== ========== ========== ========== + `Instant Messaging`_ Required Required Required Required Optional + `Presence`_ Required Required Required Required Optional + `Push Notifications`_ Optional Required Optional Optional Optional + `Receipts`_ Required Required Required Required Optional + `Typing Notifications`_ Required Required Required Required Optional + `VoIP`_ Required Required Required Optional Optional + `Content Repository`_ Required Required Required Optional Optional + `History Visibility`_ Required Required Required Required Optional + `End-to-End Encryption`_ Optional Optional Optional Optional Optional +============================ ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* From be9402b66fbb49c4c23c57e7aec93d1f1efc03f3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 16:43:09 +0100 Subject: [PATCH 193/223] Move feature profiles section to modules as a sub-section --- specification/0-feature_profiles.rst | 2 +- specification/targets.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 1d88311b1..0580c18a6 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -25,7 +25,7 @@ Summary `VoIP`_ Required Required Required Optional Optional `Content Repository`_ Required Required Required Optional Optional `History Visibility`_ Required Required Required Required Optional - `End-to-End Encryption`_ Optional Optional Optional Optional Optional + `End-to-End Encryption`_ Optional Optional Optional Optional Optional ============================ ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* diff --git a/specification/targets.yaml b/specification/targets.yaml index 62585c698..d77bf8b59 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,11 +2,11 @@ targets: main: # arbitrary name to identify this build target files: # the sort order of files to cat - 0-intro.rst - - { 1: 0-feature_profiles.rst } - 1-client_server_api.rst - { 1: 0-events.rst } - { 1: 0-event_signing.rst } - 2-modules.rst + - { 1: 0-feature_profiles.rst } - { 1: "group:modules" } # reference a group of files - 3-application_service_api.rst - 4-server_server_api.rst From 91b6347f74c4e01ab62b56ee784b735c97d60df0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 16:48:47 +0100 Subject: [PATCH 194/223] Explain what 'embedded' clients are --- specification/0-feature_profiles.rst | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 0580c18a6..5a7e5513c 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -50,14 +50,6 @@ This is a web page which heavily uses Matrix for communication. Single-page web apps would be classified as a stand-alone web client, as would multi-page web apps which use Matrix on nearly every page. -Embedded web (``EmbedWeb``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is a Matrix client which is embedded in another website, e.g. using -iframes. These embedded clients are typically for a single purpose -related to the website in question, and are not intended to be fully-fledged -communication apps. - Mobile (``Mobile``) ~~~~~~~~~~~~~~~~~~~ @@ -77,9 +69,25 @@ Command Line Interface (``CLI``) This is a client which is used via a text-based terminal. -Internet of Things (``IoT``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Embedded (``Embedded``) +~~~~~~~~~~~~~~~~~~~~~~~ + +This is a client which is embedded into another application or an embedded +device. + +Application ++++++++++++ + +This is a Matrix client which is embedded in another website, e.g. using +iframes. These embedded clients are typically for a single purpose +related to the website in question, and are not intended to be fully-fledged +communication apps. + +Device +++++++ This is a client which is typically running on an embedded device such as a -kettle, fridge or car. +kettle, fridge or car. These clients tend to perform a few operations and run +in a resource constrained environment. Like embedded applications, they are +not intended to be fully-fledged communication systems. From 6c3e70d2721554827d14d27ff07e892976ffcd24 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 17:32:44 +0100 Subject: [PATCH 195/223] Start fleshing out voip module --- specification/1-client_server_api.rst | 2 + specification/modules/voip_events.rst | 84 ++++++++++++++++++--------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 7d8d57e40..db88bbd6b 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -427,6 +427,8 @@ the complete dataset is provided in "chunk". Events ------ +.. _sect:events: + Overview ~~~~~~~~ diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index 33998cd95..f1b8ae058 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -1,17 +1,18 @@ Voice over IP -------------- -Matrix can also be used to set up VoIP calls. This is part of the core -specification, although is at a relatively early stage. Voice (and video) over -Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like -any other event. This means that clients must only send call events to rooms -with exactly two participants as currently the WebRTC standard is based around -two-party communication. +============= -{{voip_events}} +This module outlines how two users in a room can set up a Voice over IP (VoIP) +call to each other. Voice and video calls are built upon the WebRTC 1.0 standard. +Call signalling is achieved by sending `message events`_ to the room. As a result, +this means that clients MUST only send call events to rooms with exactly two +participants as currently the WebRTC standard is based around two-party +communication. + +.. _message events: `sect:events`_ Message Exchange -~~~~~~~~~~~~~~~~ -A call is set up with messages exchanged as follows: +---------------- +A call is set up with message events exchanged as follows: :: @@ -38,28 +39,57 @@ Or a rejected call: Calls are negotiated according to the WebRTC specification. +Events +------ + +{{voip_events}} + +Client behaviour +---------------- Glare ~~~~~ -This specification aims to address the problem of two users calling each other -at roughly the same time and their invites crossing on the wire. It is a far -better experience for the users if their calls are connected if it is clear -that their intention is to set up a call with one another. In Matrix, calls are -to rooms rather than users (even if those rooms may only contain one other user) -so we consider calls which are to the same room. The rules for dealing with such -a situation are as follows: - - - If an invite to a room is received whilst the client is preparing to send an - invite to the same room, the client should cancel its outgoing call and - instead automatically accept the incoming call on behalf of the user. - - If an invite to a room is received after the client has sent an invite to - the same room and is waiting for a response, the client should perform a - lexicographical comparison of the call IDs of the two calls and use the - lesser of the two calls, aborting the greater. If the incoming call is the - lesser, the client should accept this call on behalf of the user. + +"Glare" is a problem which occurs when two users call each other at roughly the +same time. This results in the call failing to set up as there already is an +incoming/outgoing call. A glare resolution algorithm can be used to determine +which call to hangup and which call to answer. If both clients implement the +same algorithm then they will both select the same call and the call will be +successfully connected. + + +As calls are "placed" to rooms rather than users, the glare resolution algorithm +outlined below is only considered for calls which are to the same room. The +algorithm is as follows: + + - If an invite to a room is received whilst the client is **preparing to send** + an invite to the same room: + + * the client should cancel its outgoing call and instead + automatically accept the incoming call on behalf of the user. + + - If an invite to a room is received **after the client has sent** an invite to + the same room and is waiting for a response: + + * the client should perform a lexicographical comparison of the call IDs of + the two calls and use the **lesser** of the two calls, aborting the + greater. If the incoming call is the lesser, the client should accept + this call on behalf of the user. + The call setup should appear seamless to the user as if they had simply placed -a call and the other party had accepted. Thusly, any media stream that had been +a call and the other party had accepted. This means any media stream that had been setup for use on a call should be transferred and used for the call that replaces it. +Server behaviour +---------------- + +TURN Servers +~~~~~~~~~~~~ + +Security considerations +----------------------- + + + From d092b22848e4ec63aee7c157e2bc2203d6d48d87 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 09:23:08 +0100 Subject: [PATCH 196/223] Rename to 'Managing history visibility' --- specification/0-feature_profiles.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/specification/0-feature_profiles.rst b/specification/0-feature_profiles.rst index 5a7e5513c..b9f12b74b 100644 --- a/specification/0-feature_profiles.rst +++ b/specification/0-feature_profiles.rst @@ -14,19 +14,19 @@ in order for it to be classified as that profile. Summary ------- -============================ ========== ========== ========== ========== ========== - Module / Profile Web Mobile Desktop CLI Embedded -============================ ========== ========== ========== ========== ========== - `Instant Messaging`_ Required Required Required Required Optional - `Presence`_ Required Required Required Required Optional - `Push Notifications`_ Optional Required Optional Optional Optional - `Receipts`_ Required Required Required Required Optional - `Typing Notifications`_ Required Required Required Required Optional - `VoIP`_ Required Required Required Optional Optional - `Content Repository`_ Required Required Required Optional Optional - `History Visibility`_ Required Required Required Required Optional - `End-to-End Encryption`_ Optional Optional Optional Optional Optional -============================ ========== ========== ========== ========== ========== +===================================== ========== ========== ========== ========== ========== + Module / Profile Web Mobile Desktop CLI Embedded +===================================== ========== ========== ========== ========== ========== + `Instant Messaging`_ Required Required Required Required Optional + `Presence`_ Required Required Required Required Optional + `Push Notifications`_ Optional Required Optional Optional Optional + `Receipts`_ Required Required Required Required Optional + `Typing Notifications`_ Required Required Required Required Optional + `VoIP`_ Required Required Required Optional Optional + `Content Repository`_ Required Required Required Optional Optional + `Managing History Visibility`_ Required Required Required Required Optional + `End-to-End Encryption`_ Optional Optional Optional Optional Optional +===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -38,7 +38,7 @@ Summary .. _Typing Notifications: `module:typing`_ .. _VoIP: `module:voip`_ .. _Content Repository: `module:content`_ -.. _History Visibility: `module:history-visibility`_ +.. _Managing History Visibility: `module:history-visibility`_ Clients ------- From e82661413e3bf51dd248778b0f3dd9a83b29efb9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 11:04:42 +0100 Subject: [PATCH 197/223] Add /turnServer endpoint --- api/client-server/v1/voip.yaml | 68 +++++++++++++++++++++++++++ specification/modules/voip_events.rst | 30 +++++++----- 2 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 api/client-server/v1/voip.yaml diff --git a/api/client-server/v1/voip.yaml b/api/client-server/v1/voip.yaml new file mode 100644 index 000000000..5fdf1ca7a --- /dev/null +++ b/api/client-server/v1/voip.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Voice over IP API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/turnServer": + get: + summary: Obtain TURN server credentials. + description: |- + This API provides credentials for the client to use when initiating + calls. + security: + - accessToken: [] + responses: + 200: + description: The TURN server credentials. + examples: + application/json: |- + { + "username":"1443779631:@user:example.com", + "password":"JlKfBy1QwLrO20385QyAtEyIv0=", + "uris":[ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp" + ], + "ttl":86400 + } + schema: + type: object + properties: + username: + type: string + description: |- + The username to use. + password: + type: string + description: |- + The password to use. + uris: + type: array + items: + type: string + description: A list of TURN URIs + ttl: + type: integer + description: The time-to-live in seconds + required: ["username", "password", "uris", "ttl"] + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index f1b8ae058..1bf1ea1d7 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -10,8 +10,14 @@ communication. .. _message events: `sect:events`_ -Message Exchange +Events +------ + +{{voip_events}} + +Client behaviour ---------------- + A call is set up with message events exchanged as follows: :: @@ -39,14 +45,6 @@ Or a rejected call: Calls are negotiated according to the WebRTC specification. -Events ------- - -{{voip_events}} - -Client behaviour ----------------- - Glare ~~~~~ @@ -72,7 +70,7 @@ algorithm is as follows: the same room and is waiting for a response: * the client should perform a lexicographical comparison of the call IDs of - the two calls and use the **lesser** of the two calls, aborting the + the two calls and use the *lesser* of the two calls, aborting the greater. If the incoming call is the lesser, the client should accept this call on behalf of the user. @@ -85,11 +83,17 @@ replaces it. Server behaviour ---------------- -TURN Servers -~~~~~~~~~~~~ +The server MAY provide a TURN server which clients can use to contact the +remote party. This server should be accessible via the HTTP endpoint listed +below. + +{{voip_http_api}} + Security considerations ----------------------- - +Calls should only be placed to rooms with one other user in them. If they are +placed to group chat rooms it is possible that another user will intercept and +answer the call. From 3b73b07babbcd86dbb8318f4813b27322a689848 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 11:11:08 +0100 Subject: [PATCH 198/223] Clarifications that room invites are m.call.invites not actual invites --- specification/modules/voip_events.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specification/modules/voip_events.rst b/specification/modules/voip_events.rst index 1bf1ea1d7..9d27c23b5 100644 --- a/specification/modules/voip_events.rst +++ b/specification/modules/voip_events.rst @@ -60,14 +60,14 @@ As calls are "placed" to rooms rather than users, the glare resolution algorithm outlined below is only considered for calls which are to the same room. The algorithm is as follows: - - If an invite to a room is received whilst the client is **preparing to send** - an invite to the same room: + - If an ``m.call.invite`` to a room is received whilst the client is + **preparing to send** an ``m.call.invite`` to the same room: * the client should cancel its outgoing call and instead automatically accept the incoming call on behalf of the user. - - If an invite to a room is received **after the client has sent** an invite to - the same room and is waiting for a response: + - If an ``m.call.invite`` to a room is received **after the client has sent** + an ``m.call.invite`` to the same room and is waiting for a response: * the client should perform a lexicographical comparison of the call IDs of the two calls and use the *lesser* of the two calls, aborting the @@ -83,9 +83,9 @@ replaces it. Server behaviour ---------------- -The server MAY provide a TURN server which clients can use to contact the -remote party. This server should be accessible via the HTTP endpoint listed -below. +The homeserver MAY provide a TURN server which clients can use to contact the +remote party. The following HTTP API endpoints will be used by clients in order +to get information about the TURN server. {{voip_http_api}} From 365a9076b93500cd87e85e68129e6ba7729f7625 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 12:11:26 +0100 Subject: [PATCH 199/223] Add nested dict template support; Add x-pattern For cases where event schema specify `patternProperties` it would be nice to give that pattern a "human-readable" form rather than a raw regex. This is now supported by specifying `x-pattern` in the value part of the specified pattern e.g. `patternProperties:{ "^.*":{ x-pattern: "$THING", ... } }` Templating had limited record type descriptions limited to value primitives e.g. `{string: integer}`. It now supports inspecting the values recursively if the value is `object`. Updated `m.receipt` to take both these points into account to make it read better. Tweak receipt module text. --- event-schemas/schema/v1/m.receipt | 9 ++++--- specification/modules/receipts.rst | 37 ++++++++++++---------------- templating/matrix_templates/units.py | 26 +++++++++++++++---- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 0f365eed5..5be232ad4 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -5,17 +5,20 @@ "properties": { "content": { "type": "object", - "description": "The event ids which the receipts relate to.", "patternProperties": { "^\\$": { "type": "object", - "description": "The types of the receipts.", + "x-pattern": "$EVENT_ID", + "description": "The mapping of event ID to receipt type. The event ID is the ID which the receipts relate to and *not* an ID for the receipt itself. The key in the object is an enum which must be ``read``.", "additionalProperties": { "type": "object", - "description": "User ids of the receipts", + "title": "Users", "patternProperties": { "^@": { "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", "properties": { "ts": { "type": "number", diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index e2f83eea8..f32c12dbf 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -6,8 +6,16 @@ have interacted with. For example, which events the user has read. For efficiency this is done as "up to" markers, i.e. marking a particular event as, say, ``read`` indicates the user has read all events *up to* that event. -Client-Server API -~~~~~~~~~~~~~~~~~ +Events +------ + +{{m_receipt_event}} + +Client behaviour +---------------- + + - When clients should send receipts + - What clients should do when they receive these receipts Clients will receive receipts in the following format:: @@ -25,22 +33,6 @@ Clients will receive receipts in the following format:: } } -For example:: - - { - "type": "m.receipt", - "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", - "content": { - "$1435641916114394fHBLK:matrix.org": { - "read": { - "@erikj:jki.re": { "ts": 1436451550453 }, - ... - } - }, - ... - } - } - For efficiency, receipts are batched into one event per room. In the initialSync and v2 sync APIs the receipts are listed in a separate top level ``receipts`` key. Each ``user_id``, ``receipt_type`` pair must be associated with only a @@ -56,9 +48,8 @@ A client can update the markers for its user by issuing a request:: Where the contents of the ``POST`` will be included in the content sent to other users. The server will automatically set the ``ts`` field. - -Server-Server API -~~~~~~~~~~~~~~~~~ +Server behaviour +---------------- Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -75,3 +66,7 @@ format of the EDUs are:: These are always sent as deltas to previously sent receipts. +Security considerations +----------------------- + + diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 50fa784e6..473fdd820 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -49,8 +49,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): } tables = [fields] - props = obj.get("properties", obj.get("patternProperties")) parents = obj.get("allOf") + props = obj.get("properties") + if not props: + props = obj.get("patternProperties") + if props: + # try to replace horrible regex key names with pretty x-pattern ones + for key_name in props.keys(): + pretty_key = props[key_name].get("x-pattern") + if pretty_key: + props[pretty_key] = props[key_name] + del props[key_name] if not props and not parents: raise Exception( "Object %s has no properties or parents." % obj @@ -70,10 +79,17 @@ def get_json_schema_object_fields(obj, enforce_title=False): if props[key_name]["type"] == "object": if props[key_name].get("additionalProperties"): # not "really" an object, just a KV store - value_type = ( - "{string: %s}" % - props[key_name]["additionalProperties"]["type"] - ) + prop_val = props[key_name]["additionalProperties"]["type"] + if prop_val == "object": + nested_object = get_json_schema_object_fields( + props[key_name]["additionalProperties"], + enforce_title=True + ) + value_type = "{string: %s}" % nested_object[0]["title"] + if not nested_object[0].get("no-table"): + tables += nested_object + else: + value_type = "{string: %s}" % prop_val else: nested_object = get_json_schema_object_fields( props[key_name], From c972dad8b383a8345864d32ba6997b5da1fd57fa Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 15:41:54 +0100 Subject: [PATCH 200/223] Flesh out receipts module. Add receipts swagger Add templating support for v2 apis. --- api/client-server/v2_alpha/receipts.yaml | 68 +++++++++++++++++++++ event-schemas/examples/v1/m.receipt | 2 +- event-schemas/schema/v1/m.receipt | 35 ++++++----- specification/modules/receipts.rst | 76 ++++++++++++++---------- templating/matrix_templates/units.py | 30 ++++++---- 5 files changed, 151 insertions(+), 60 deletions(-) create mode 100644 api/client-server/v2_alpha/receipts.yaml diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/v2_alpha/receipts.yaml new file mode 100644 index 000000000..4ef435b02 --- /dev/null +++ b/api/client-server/v2_alpha/receipts.yaml @@ -0,0 +1,68 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 Receipts API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/receipt/{receiptType}/{eventId}": + post: + summary: Send a receipt for the given event ID. + description: |- + This API updates the marker for the given receipt type to the event ID + specified. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: roomId + description: The room in which to send the event. + required: true + x-example: "!wefuh21ffskfuh345:example.com" + - in: path + type: string + name: receiptType + description: The type of receipt to send. + required: true + x-example: "m.read" + enum: ["m.read"] + - in: path + type: string + name: eventId + description: The event ID to acknowledge up to. + required: true + x-example: "$1924376522eioj:example.com" + - in: body + description: |- + Extra receipt information to attach to ``content`` if any. The + server will automatically set the ``ts`` field. + schema: + type: object + example: |- + {} + responses: + 200: + description: The receipt was sent. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" \ No newline at end of file diff --git a/event-schemas/examples/v1/m.receipt b/event-schemas/examples/v1/m.receipt index 83515317a..bd0b726c3 100644 --- a/event-schemas/examples/v1/m.receipt +++ b/event-schemas/examples/v1/m.receipt @@ -3,7 +3,7 @@ "room_id": "!KpjVgQyZpzBwvMBsnT:matrix.org", "content": { "$1435641916114394fHBLK:matrix.org": { - "read": { + "m.read": { "@rikj:jki.re": { "ts": 1436451550453 } diff --git a/event-schemas/schema/v1/m.receipt b/event-schemas/schema/v1/m.receipt index 5be232ad4..d0f79ac4a 100644 --- a/event-schemas/schema/v1/m.receipt +++ b/event-schemas/schema/v1/m.receipt @@ -9,25 +9,28 @@ "^\\$": { "type": "object", "x-pattern": "$EVENT_ID", - "description": "The mapping of event ID to receipt type. The event ID is the ID which the receipts relate to and *not* an ID for the receipt itself. The key in the object is an enum which must be ``read``.", - "additionalProperties": { - "type": "object", - "title": "Users", - "patternProperties": { - "^@": { - "type": "object", - "title": "Receipt", - "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", - "x-pattern": "$USER_ID", - "properties": { - "ts": { - "type": "number", - "description": "The timestamp the receipt was sent at" + "title": "Receipts", + "description": "The mapping of event ID to a collection of receipts for this event ID. The event ID is the ID of the event being acknowledged and *not* an ID for the receipt itself.", + "properties": { + "m.read": { + "type": "object", + "title": "Users", + "description": "A collection of users who have sent ``m.read`` receipts for this event.", + "patternProperties": { + "^@": { + "type": "object", + "title": "Receipt", + "description": "The mapping of user ID to receipt. The user ID is the entity who sent this receipt.", + "x-pattern": "$USER_ID", + "properties": { + "ts": { + "type": "number", + "description": "The timestamp the receipt was sent at." + } } } } - }, - "additionalProperties": false + } } } }, diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index f32c12dbf..9dabba30e 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -1,56 +1,63 @@ Receipts --------- +======== -Receipts are used to publish which events in a room the user or their devices -have interacted with. For example, which events the user has read. For -efficiency this is done as "up to" markers, i.e. marking a particular event -as, say, ``read`` indicates the user has read all events *up to* that event. +This module adds in support for receipts. These receipts are a form of +acknowledgement of an event. This module defines a single acknowledgement: +``m.read`` which indicates that the user has read up to a given event. + +Sending a receipt for each event can result in sending large amounts of traffic +to a homeserver. To prevent this from becoming a problem, receipts are implemented +using "up to" markers. This marker indicates that the acknowledgement applies +to all events "up to and including" the event specified. For example, marking +an event as "read" would indicate that the user had read all events *up to* the +referenced event. Events ------ +Each ``user_id``, ``receipt_type`` pair must be associated with only a +single ``event_id``. {{m_receipt_event}} Client behaviour ---------------- - - When clients should send receipts - - What clients should do when they receive these receipts - -Clients will receive receipts in the following format:: +In v1 ``/initialSync``, receipts are listed in a separate top level ``receipts`` +key. In v2 ``/sync``, receipts are contained in the ``ephemeral`` block for a +room. New receipts that come down the event streams are deltas which update +existing mappings. Clients should replace older receipt acknowledgements based +on ``user_id`` and ``receipt_type`` pairs. For example:: - { - "type": "m.receipt", - "room_id": , - "content": { - : { - : { - : { "ts": , ... }, - ... - } - }, - ... - } - } + Client receives m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $aaa:example.com -For efficiency, receipts are batched into one event per room. In the initialSync -and v2 sync APIs the receipts are listed in a separate top level ``receipts`` -key. Each ``user_id``, ``receipt_type`` pair must be associated with only a -single ``event_id``. New receipts that come down the event streams are deltas. -Deltas update existing mappings, clobbering based on ``user_id``, -``receipt_type`` pairs. + Client receives another m.receipt: + user = @alice:example.com + receipt_type = m.read + event_id = $bbb:example.com + The client should replace the older acknowledgement for $aaa:example.com with + this one for $bbb:example.com -A client can update the markers for its user by issuing a request:: +Clients should send read receipts when there is some certainty that the event in +question has been **displayed** to the user. Simply receiving an event does not +provide enough certainty that the user has seen the event. The user SHOULD need +to *take some action* such as viewing the room that the event was sent to or +dismissing a notification in order for the event to count as "read". - POST /_matrix/client/v2_alpha/rooms//receipt/read/ +A client can update the markers for its user by interacting with the following +HTTP APIs. -Where the contents of the ``POST`` will be included in the content sent to -other users. The server will automatically set the ``ts`` field. +{{v2_receipts_http_api}} Server behaviour ---------------- +For efficiency, receipts SHOULD be batched into one event per room before +delivering them to clients. + Receipts are sent across federation as EDUs with type ``m.receipt``. The format of the EDUs are:: @@ -64,9 +71,12 @@ format of the EDUs are:: ... } -These are always sent as deltas to previously sent receipts. +These are always sent as deltas to previously sent receipts. Currently only a +single ```` should be used: ``m.read``. Security considerations ----------------------- +As receipts are sent outside the context of the event graph, there are no +integrity checks performed on the contents of ``m.receipt`` events. diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 473fdd820..eca52acb1 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -19,6 +19,7 @@ import yaml V1_CLIENT_API = "../api/client-server/v1" V1_EVENT_EXAMPLES = "../event-schemas/examples/v1" V1_EVENT_SCHEMA = "../event-schemas/schema/v1" +V2_CLIENT_API = "../api/client-server/v2_alpha" CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema" CHANGELOG = "../CHANGELOG.rst" TARGETS = "../specification/targets.yaml" @@ -336,18 +337,27 @@ class MatrixUnits(Units): } def load_swagger_apis(self): - path = V1_CLIENT_API + paths = [ + V1_CLIENT_API, V2_CLIENT_API + ] apis = {} - for filename in os.listdir(path): - if not filename.endswith(".yaml"): + for path in paths: + is_v2 = (path == V2_CLIENT_API) + if not os.path.exists(V2_CLIENT_API): + self.log("Skipping v2 apis: %s does not exist.", V2_CLIENT_API) continue - self.log("Reading swagger API: %s" % filename) - with open(os.path.join(path, filename), "r") as f: - # strip .yaml - group_name = filename[:-5] - api = yaml.load(f.read()) - api["__meta"] = self._load_swagger_meta(api, group_name) - apis[group_name] = api + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + self.log("Reading swagger API: %s" % filename) + with open(os.path.join(path, filename), "r") as f: + # strip .yaml + group_name = filename[:-5] + if is_v2: + group_name = "v2_" + group_name + api = yaml.load(f.read()) + api["__meta"] = self._load_swagger_meta(api, group_name) + apis[group_name] = api return apis def load_common_event_fields(self): From 560cd7a58fe6ecdb713dd3cd30b58b3e98a04d3d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 15:54:45 +0100 Subject: [PATCH 201/223] This isn't javascript. s/,/%/ --- templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index eca52acb1..71b6acc6d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -344,7 +344,7 @@ class MatrixUnits(Units): for path in paths: is_v2 = (path == V2_CLIENT_API) if not os.path.exists(V2_CLIENT_API): - self.log("Skipping v2 apis: %s does not exist.", V2_CLIENT_API) + self.log("Skipping v2 apis: %s does not exist." % V2_CLIENT_API) continue for filename in os.listdir(path): if not filename.endswith(".yaml"): From 87b6dd845e0e6c685800bbc13f9459a7d21ad59e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 17:55:16 +0100 Subject: [PATCH 202/223] Flesh out content repo; modify templating to support headers Edit content-repo.yaml to include examples and headers. Restructure content module to conform to the module template. Adjust the HTTP API template to give 1 more char to the response param to fit "Content-Disposition" correctly. Edit the templating system to support displaying enums for swagger APIs (before it was just JSON schema). Also add support for introspecting headers from swagger. Finally, replace - with _ when forming the {{ template_var }} else things whine. --- api/client-server/v1/content-repo.yaml | 49 +++++++++++++++++-- specification/modules/content_repo.rst | 30 ++++++++++-- .../matrix_templates/templates/http-api.tmpl | 12 ++--- templating/matrix_templates/units.py | 22 +++++++-- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index fe3d1dc37..aa47dc184 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -15,16 +15,22 @@ paths: summary: Upload some content to the content repository. produces: ["application/json"] parameters: + - in: header + name: Content-Type + type: string + description: The content type of the file being uploaded + x-example: "Content-Type: audio/mpeg" - in: body - name: content + name: "" description: The content to be uploaded. required: true schema: type: string + example: "" format: byte responses: 200: - description: Information about the uploaded content. + description: The MXC URI for the uploaded content. schema: type: object required: ["content_uri"] @@ -32,6 +38,11 @@ paths: content_uri: type: string description: "The MXC URI to the uploaded content." + examples: + "application/json": |- + { + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + } "/download/{serverName}/{mediaId}": get: summary: "Download content from the content repository." @@ -40,20 +51,32 @@ paths: - in: path type: string name: serverName + x-example: matrix.org required: true description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) responses: 200: - description: "The content downloaded." + description: "The content that was previously uploaded." + headers: + Content-Type: + description: "The content type of the file that was previously uploaded." + x-example: "audio/mpeg" + type: "string" + Content-Disposition: + description: "The name of the file that was previously uploaded, if set." + x-example: "attachment;filename=03-cool.mp3" + type: "string" schema: type: file + name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -63,31 +86,47 @@ paths: type: string name: serverName required: true + x-example: matrix.org description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) - in: query type: integer + x-example: 64 name: width - description: The desired width of the thumbnail. + description: |- + The *desired* width of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: integer + x-example: 64 name: height - description: The desired height of the thumbnail. + description: |- + The *desired* height of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: string enum: ["crop", "scale"] name: method + x-example: "scale" description: The desired resizing method. responses: 200: description: "A thumbnail of the requested content." + headers: + Content-Type: + description: "The content type of the thumbnail." + x-example: "image/jpeg" + type: "string" + enum: ["image/jpeg", "image/png"] schema: type: file + name: "" diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 83333f379..de464e203 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -3,8 +3,23 @@ Content repository .. _module:content: -HTTP API --------- +This module allows users to upload content to their homeserver which is +retrievable from other homeservers. Its' purpose is to allow users to share +attachments in a room. Content locations are represented as Matrix Content (MXC) +URIs. They look like:: + + mxc:/// + + : The name of the homeserver where this content can be found, e.g. matrix.org + : An opaque ID which identifies the content. + +Client behaviour +---------------- + +Clients can upload and download content using the following HTTP APIs. + +{{content_repo_http_api}} + Uploads are POSTed to a resource which returns a token which is used to GET the download. Uploads are POSTed to the sender's local homeserver, but are @@ -49,6 +64,9 @@ width and height are close to the requested size and the aspect matches the requested size. The client should scale the image if it needs to fit within a given rectangle. +Server behaviour +---------------- + Homeservers may generate thumbnails for content uploaded to remote homeservers themselves or may rely on the remote homeserver to thumbnail the content. Homeservers may return thumbnails of a different size to that @@ -58,13 +76,19 @@ Homeservers must never upscale images. Security considerations ----------------------- +The HTTP GET endpoint does not require any authentication. Knowing the URL of +the content is sufficient to retrieve the content, even if the entity isn't in +the room. + +Homeservers have additional concerns: + - Clients may try to upload very large files. Homeservers should not store files that are too large and should not serve them to clients. - Clients may try to upload very large images. Homeservers should not attempt to generate thumbnails for images that are too large. - - Remote homeservers may host very large files or images. Homeserver should not + - Remote homeservers may host very large files or images. Homeservers should not proxy or thumbnail large files or images from remote homeservers. - Clients may try to upload a large number of files. Homeservers should limit the diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index eb3f3e64e..472c9d7ae 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -31,18 +31,18 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== +=================== ================= ========================================== Param Type Description -================== ================= =========================================== +=================== ================= ========================================== {% for row in table.rows -%} {# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} +{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} +{# It also needs to then wrap inside the desc col (42 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(20-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(18 - (row.type|length))) |indent_block(38)}} {% endfor -%} -================== ================= =========================================== +=================== ================= ========================================== {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 50fa784e6..04f3bae10 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -151,6 +151,13 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + + if param.get("enum"): + val_type = "enum" + desc += ( + " One of: %s" % json.dumps(param.get("enum")) + ) + refType = Units.prop(param, "schema/$ref/") # Error,Event schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads if not val_type and refType: @@ -255,7 +262,7 @@ class MatrixUnits(Units): res_type = Units.prop(good_response, "schema/type") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that - endpoint["res_tables"].append({ + good_table = { "title": None, "rows": [{ "key": good_response["schema"].get("name", ""), @@ -263,7 +270,16 @@ class MatrixUnits(Units): "desc": res.get("description", ""), "req_str": "" }] - }) + } + if good_response.get("headers"): + for (header_name, header) in good_response.get("headers").iteritems(): + good_table["rows"].append({ + "key": header_name, + "type": "Header<" + header["type"] + ">", + "desc": header["description"], + "req_str": "" + }) + endpoint["res_tables"].append(good_table) elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] @@ -328,7 +344,7 @@ class MatrixUnits(Units): self.log("Reading swagger API: %s" % filename) with open(os.path.join(path, filename), "r") as f: # strip .yaml - group_name = filename[:-5] + group_name = filename[:-5].replace("-", "_") api = yaml.load(f.read()) api["__meta"] = self._load_swagger_meta(api, group_name) apis[group_name] = api From 8c4d7f50510edb2a8e0932248efed1949bf1715a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 18:03:34 +0100 Subject: [PATCH 203/223] Do not try to parse non-json request examples as json --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index a0cd0658b..f08b2dc1a 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -34,7 +34,7 @@ def check_parameter(filepath, request, parameter): example = None try: example_json = schema.get('example') - if example_json: + if example_json and not schema.get("format") == "byte": example = json.loads(example_json) except Exception as e: raise ValueError("Error parsing JSON example request for %r" % ( From 30232f20aa571cbd402ff56eaa970bbc0fc8d34c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Oct 2015 19:13:09 -0500 Subject: [PATCH 204/223] speculator: Merge after fetching, so that /spec/head works --- scripts/speculator/main.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 6ee80150d..442400dfe 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -60,11 +60,10 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" ) - func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) cmd := exec.Command("git", "clone", url, directory) @@ -80,21 +79,22 @@ func gitClone(url string, shared bool) (string, error) { } func gitCheckout(path, sha string) error { - cmd := exec.Command("git", "checkout", sha) - cmd.Dir = path - err := cmd.Run() - if err != nil { - return fmt.Errorf("error checking out repo: %v", err) + return runGitCommand(path, []string{"checkout", sha}) +} + +func gitFetchAndMerge(path string) error { + if err := runGitCommand(path, []string{"fetch"}); err != nil { + return err } - return nil + return runGitCommand(path, []string{"merge"}) } -func gitFetch(path string) error { - cmd := exec.Command("git", "fetch") +func runGitCommand(path string, args []string) error { + cmd := exec.Command("git", args...) cmd.Dir = path err := cmd.Run() if err != nil { - return fmt.Errorf("error fetching repo: %v", err) + return fmt.Errorf("error running %s: %v", strings.Join(cmd.Args, " "), err) } return nil } @@ -145,7 +145,7 @@ type server struct { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = gitFetchAndMerge(s.matrixDocCloneURL) if err != nil { return } From a69e03f57778b9ef743cb03430db472e1b0a28a9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Oct 2015 19:15:30 -0500 Subject: [PATCH 205/223] speculator: Report listening port --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 6ee80150d..ece653726 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -362,6 +362,8 @@ func main() { http.HandleFunc("/diff/html/", s.serveHTMLDiff) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) + + fmt.Printf("Listening on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } From 3d9dbe42e691f446830e348eb680297728dadede Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:21:48 +0100 Subject: [PATCH 206/223] Bump to swagger-parser 3.2.1 - remove x- keys on headers Removed x- keys due to https://github.com/BigstickCarpet/swagger-parser/issues/23 --- api/client-server/v1/content-repo.yaml | 5 ----- api/package.json | 2 +- api/validator.js | 10 +++++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index aa47dc184..8e6e8d1a4 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -68,15 +68,12 @@ paths: headers: Content-Type: description: "The content type of the file that was previously uploaded." - x-example: "audio/mpeg" type: "string" Content-Disposition: description: "The name of the file that was previously uploaded, if set." - x-example: "attachment;filename=03-cool.mp3" type: "string" schema: type: file - name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -122,11 +119,9 @@ paths: headers: Content-Type: description: "The content type of the thumbnail." - x-example: "image/jpeg" type: "string" enum: ["image/jpeg", "image/png"] schema: type: file - name: "" diff --git a/api/package.json b/api/package.json index 151934935..84b9dd7b5 100644 --- a/api/package.json +++ b/api/package.json @@ -10,6 +10,6 @@ "license": "ISC", "dependencies": { "nopt": "^3.0.2", - "swagger-parser": "^2.4.1" + "swagger-parser": "^3.2.1" } } diff --git a/api/validator.js b/api/validator.js index 3b89a5a36..0d76c09d3 100644 --- a/api/validator.js +++ b/api/validator.js @@ -26,11 +26,10 @@ if (!opts.schema) { } -var errFn = function(err, api, metadata) { +var errFn = function(err, api) { if (!err) { return; } - console.log(metadata); console.error(err); process.exit(1); }; @@ -46,11 +45,12 @@ if (isDir) { files.forEach(function(f) { var suffix = ".yaml"; if (f.indexOf(suffix, f.length - suffix.length) > 0) { - parser.parse(path.join(opts.schema, f), function(err, api, metadata) { + parser.validate(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); } else { + console.error("%s is not valid.", f); errFn(err, api, metadata); } }); @@ -59,12 +59,12 @@ if (isDir) { }); } else{ - parser.parse(opts.schema, function(err, api, metadata) { + parser.validate(opts.schema, function(err, api) { if (!err) { console.log("%s is valid", opts.schema); } else { - errFn(err, api, metadata); + errFn(err, api); } }); }; From dbc72c43aca270d86c128caba1cde434642bcb8a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 10:28:29 +0100 Subject: [PATCH 207/223] s/private_chat_shared_power/trusted_private_chat/ --- specification/1-client_server_api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 329a87ddb..1c64f0218 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,7 +756,7 @@ options which can be set when creating a room: Optional: Yes Value: - ``private_chat``, ``private_chat_shared_power`` or ``public_chat`` + ``private_chat``, ``trusted_private_chat`` or ``public_chat`` Description: Convenience parameter for setting various default state events based on a preset. @@ -765,9 +765,9 @@ options which can be set when creating a room: - ``private_chat``: Sets the ``join_rules`` to ``invite`` and ``history_visibility`` to ``shared`` - - ``private_chat_shared_power``: Set the ``join_rules`` to - ``invite``, ``history_visibility`` to ``shared`` and gives all invitees - the same power level as the creator. + - ``trusted_private_chat``: Set the ``join_rules`` to ``invite``, + ``history_visibility`` to ``shared`` and gives all invitees the same + power level as the creator. - ``public_chat``: Sets the ``join_rules`` to ``public`` and ``history_visibility`` to ``shared`` From 912a8ca7605278322cc8f8c921002022ea4e34c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 10:30:45 +0100 Subject: [PATCH 208/223] Be more explicit about keys that are clobbered --- specification/1-client_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index dd58b357a..40b46be08 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -757,7 +757,8 @@ options which can be set when creating a room: Yes Value: Extra keys to be added to the content of the ``m.room.create``. The server - will clober certain keys, e.g. ``creator``. + will clober the following keys: ``creator``. Future versions of this + spec may allow the server to clobber other keys if required. Description: Allows clients to add keys to the content of ``m.room.create``. From 4dabcd112ef787c2f09ddbd61415e145e1b146e3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:44:50 +0100 Subject: [PATCH 209/223] Remove redundant info now we have the http api template. Minor tweaks to display of schema with no names but a type --- specification/modules/content_repo.rst | 46 +++++--------------------- templating/matrix_templates/units.py | 3 +- 2 files changed, 11 insertions(+), 38 deletions(-) diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index de464e203..9ac5e1991 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -10,9 +10,15 @@ URIs. They look like:: mxc:/// - : The name of the homeserver where this content can be found, e.g. matrix.org + : The name of the homeserver where this content originated, e.g. matrix.org : An opaque ID which identifies the content. +Uploads are POSTed to a resource on the user's local homeserver which returns a +token which is used to GET the download. Content is downloaded from the +recipient's local homeserver, which must first transfer the content from the +origin homeserver using the same API (unless the origin and destination +homeservers are the same). + Client behaviour ---------------- @@ -20,42 +26,8 @@ Clients can upload and download content using the following HTTP APIs. {{content_repo_http_api}} - -Uploads are POSTed to a resource which returns a token which is used to GET -the download. Uploads are POSTed to the sender's local homeserver, but are -downloaded from the recipient's local homeserver, which must thus first transfer -the content from the origin homeserver using the same API (unless the origin -and destination homeservers are the same). The upload/download API is:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content-uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - -Clients can get thumbnails by supplying a desired width and height and -thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - +Thumbnails +~~~~~~~~~~ The thumbnail methods are "crop" and "scale". "scale" tries to return an image where either the width or the height is smaller than the requested size. The client should then scale and letterbox the image if it needs to diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 04f3bae10..1e449b5d6 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -260,12 +260,13 @@ class MatrixUnits(Units): if good_response: self.log("Found a 200 response for this API") res_type = Units.prop(good_response, "schema/type") + res_name = Units.prop(good_response, "schema/name") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that good_table = { "title": None, "rows": [{ - "key": good_response["schema"].get("name", ""), + "key": "<" + res_type + ">" if not res_name else res_name, "type": res_type, "desc": res.get("description", ""), "req_str": "" From a73cc50aa963f05d86783272a1ab105a70a84f34 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 13:03:39 +0100 Subject: [PATCH 210/223] s|client/api/v2_alpha|client/v2_alpha| --- api/client-server/v2_alpha/filter.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 30d528447..37a0a3aaa 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 238fab630..7daa33f52 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: From 0e8f1b5475f70d8436ec3a1d19886c2fe71ca677 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 2 Oct 2015 07:33:26 -0500 Subject: [PATCH 211/223] Quote args --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 442400dfe..07ffa44ef 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -94,7 +94,7 @@ func runGitCommand(path string, args []string) error { cmd.Dir = path err := cmd.Run() if err != nil { - return fmt.Errorf("error running %s: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) } return nil } From c8ddf1af09581ca5d54a1e21561d9adce2b3d99d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:00:15 +0100 Subject: [PATCH 212/223] Add changelog --- CHANGELOG.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78aebddc9..66f3d0a80 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,29 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.2.0 (2015-10-02) +============================================ + +This update restructures the specification and begins to aggressively standardise +on using Swagger and JSON Schema to document HTTP endpoints and Events +respectively. It also introduces a number of new concepts to Matrix. + +Additions: + - New section: Feature Profiles. + - New section: Receipts. + - New section: Room history visibility. + - New event: ``m.receipt``. + - New event: ``m.room.canonical_alias`` + - New event: ``m.room.history_visibility`` + - New keys: ``/createRoom`` - allows room "presets" using ``preset`` and + ``initial_state`` keys. + - New endpoint: ``/tokenrefresh`` - Related to refreshing access tokens. + +Modifications: + - Convert most of the older HTTP APIs to Swagger documentation. + - Convert most of the older event formats to JSON Schema. + - Move selected client-server sections to be "Modules". + Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. From bde003fe8618195122dd300b03913dc1b28710c3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 16:14:24 +0100 Subject: [PATCH 213/223] Split the rooms out into a separate top level key. Divide the rooms into separate groups in preparation for adding tag support. Further subdivide the rooms into "joined/invited/archived" based the membership of the user in the room because that membership affects what events the user can view from the room. E.g only users that are joined to a room may see the ephemeral events for the room. --- api/client-server/v2_alpha/sync.yaml | 114 +++++++++++++++++---------- 1 file changed, 74 insertions(+), 40 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7daa33f52..7a003745e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/v2_alpha +basePath: /_matrix/client/api/v2_alpha consumes: - application/json produces: @@ -73,43 +73,71 @@ paths: ``/sync`` request. rooms: type: object - properties: - roomlist: - type: array - description: |- - A list of rooms that the client needs to update. - items: + description: |- + The updates to rooms, grouped according to the filter. By + default there is a single ``default`` group. + additionalProperties: + joined: + type: array + description: |- + A list of room ids that the user is a member of that + have updates. + items: + type: string. + invited: + type: array + description: |- + A list of room ids that the user has been invited to. + The entries in the room_map will have a ``invite`` key + containing the ``m.room.member`` event of the invite. + items: + type: string. + archived: + type: array + description: |- + A list of room ids that the user has left or been + banned from. The entries in the room_map will have a + ``state`` key and a ``timeline`` key. But will lack the + ``emphemeral`` key. + items: + type: string. + room_map: + description: |- + Map from room id to the updates for that room. The room ids + are referenced from the ``rooms`` key. + type: object + additionalProperties: + type: object + properties: + room_id: + type: string + description: |- + The ID of the room. + event_map: type: object - properties: - room_id: - type: string - description: |- - The ID of the room. - event_map: - type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" presence: description: |- The updates to the presence status of other users. @@ -129,8 +157,14 @@ paths: ] }, "rooms": { - "roomlist": [{ - "room_id": "!726s6s6q:example.com", + "default": { + "invited": [], + "archived": [], + "joined": ["!726s6s6q:example.com"] + } + }, + "room_map": { + "!726s6s6q:example.com": { "event_map": { "$66697273743031:example.com": { "sender": "@alice:example.com", @@ -177,6 +211,6 @@ paths: } ] } - }] + } } } From 047419f2ad9b9b72ed7de144d9c97bcc17e62030 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 16:21:48 +0100 Subject: [PATCH 214/223] Remove the room_id since it is redundent. Remove text about invite event handling till we've thought about it some more. --- api/client-server/v2_alpha/sync.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7a003745e..6b7a1b2ae 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -88,8 +88,6 @@ paths: type: array description: |- A list of room ids that the user has been invited to. - The entries in the room_map will have a ``invite`` key - containing the ``m.room.member`` event of the invite. items: type: string. archived: @@ -109,10 +107,6 @@ paths: additionalProperties: type: object properties: - room_id: - type: string - description: |- - The ID of the room. event_map: type: object description: |- From 417c5b53c4c6264f6b91a4bfbaaa4049da6934a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:24:33 +0100 Subject: [PATCH 215/223] Remove duplicate sentences from merge conflicts --- specification/modules/presence.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index cff4490be..79151c4fe 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -21,11 +21,8 @@ A presence list is a list of user IDs whose presence the user wants to follow. To be added to this list, the user being added must be invited by the list owner who must accept the invitation. -Each user has presence information associated with them. This encodes the -"availability" of that user, suitable for display on other clients. -This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. Their presence state is -represented by the ``presence`` key, which is an enum of one of the following: +User's presence state is represented by the ``presence`` key, which is an enum +of one of the following: - ``online`` : The default state when the user is connected to an event stream. @@ -35,7 +32,6 @@ represented by the ``presence`` key, which is an enum of one of the following: explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. - Events ------ From 28fd1aa205b28adadee838547aed62e9f2a0a115 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:34:06 +0100 Subject: [PATCH 216/223] Go into a bit more detail about feature profiles --- CHANGELOG.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66f3d0a80..6e3198ef9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,9 +9,14 @@ Specification changes in v0.2.0 (2015-10-02) ============================================ -This update restructures the specification and begins to aggressively standardise -on using Swagger and JSON Schema to document HTTP endpoints and Events -respectively. It also introduces a number of new concepts to Matrix. +This update fundamentally restructures the specification. The specification has +been split into more digestible "modules" which each describe a particular +function (e.g. typing). This was done in order make the specification easier to +maintain and help define which modules are mandatory for certain types +of clients. Types of clients along with the mandatory modules can be found in a +new "Feature Profiles" section. This update also begins to aggressively +standardise on using Swagger and JSON Schema to document HTTP endpoints and +Events respectively. It also introduces a number of new concepts to Matrix. Additions: - New section: Feature Profiles. From 97cdd8106cec02871d58a54f1abcafe7b33481a2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 14:21:16 +0100 Subject: [PATCH 217/223] s|client/api/v2_alpha|/client/v2_alpha| --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 6b7a1b2ae..c502d926f 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: From ed0f6d3ff38848250f830ad8ade02838d9c82f3e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 14:22:50 +0100 Subject: [PATCH 218/223] Typo --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c502d926f..437af251e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -96,7 +96,7 @@ paths: A list of room ids that the user has left or been banned from. The entries in the room_map will have a ``state`` key and a ``timeline`` key. But will lack the - ``emphemeral`` key. + ``ephemeral`` key. items: type: string. room_map: From a7b808c5cd8ec423342a9844241a31bdc339eff8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 15:26:33 +0100 Subject: [PATCH 219/223] Add a invite_state key for holding the state that is bundled with an invite. This is kept separate from the actual state for the room as it may be derived from an incomplete, unverified copy of the state that was bundled with an invite event received over federation. --- api/client-server/v2_alpha/sync.yaml | 38 +++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 437af251e..070788231 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -88,6 +88,9 @@ paths: type: array description: |- A list of room ids that the user has been invited to. + The entries in the room_map will have an + ``invite_state`` key. But will lack the ``ephemeral`` + key, the ``timeline`` key and the ``state`` key. items: type: string. archived: @@ -132,6 +135,21 @@ paths: in the timeline or state of the room. E.g. typing. allOf: - $ref: "definitions/event_batch.json" + invite_state: + description: |- + The state of a room that the user has been invited to. + These state events may only have the ``type``, + ``state_key`` and ``content`` keys present. + These events do not replace any state that the client + already has for the room, for example if the client has + archived the room. Instead the client should keep two + separate copies of the state: the one from the + ``invite_state`` and one from the archived ``state``. + If the client joins the room then the current state + will be given as a delta against the archived ``state`` + not the ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" presence: description: |- The updates to the presence status of other users. @@ -152,7 +170,7 @@ paths: }, "rooms": { "default": { - "invited": [], + "invited": ["!696r7674:example.com"], "archived": [], "joined": ["!726s6s6q:example.com"] } @@ -205,6 +223,24 @@ paths: } ] } + }, + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "@alice:example.com", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"membership": "invite"} + } + ] + } } } } From bbb5fa9398447f2196cf7c62ec457c7f717704ce Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 15:36:01 +0100 Subject: [PATCH 220/223] Fix the state_key in the example v2 response, include the "sender" key in the list of keys included with invite_state events --- api/client-server/v2_alpha/sync.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 070788231..49f42c6e1 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -138,8 +138,8 @@ paths: invite_state: description: |- The state of a room that the user has been invited to. - These state events may only have the ``type``, - ``state_key`` and ``content`` keys present. + These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys present. These events do not replace any state that the client already has for the room, for example if the client has archived the room. Instead the client should keep two @@ -230,13 +230,13 @@ paths: { "sender": "@alice:example.com", "type": "m.room.name", - "state_key": "@alice:example.com", + "state_key": "", "content": {"name": "My Room Name"} }, { "sender": "@alice:example.com", "type": "m.room.name", - "state_key": "", + "state_key": "@bob:example.com", "content": {"membership": "invite"} } ] From 0e5b00feaaa9c337fd097e13780fa7d06158ac0d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 16:10:25 +0100 Subject: [PATCH 221/223] Fix event type in v2 sync example --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 49f42c6e1..1b83d1d9d 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -235,7 +235,7 @@ paths: }, { "sender": "@alice:example.com", - "type": "m.room.name", + "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "invite"} } From 0ce533d153e9656818dfdefa565b1356afa4b391 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 13:42:03 +0100 Subject: [PATCH 222/223] Split the room_map into separate sections based on whether they are "joined/invited/archived". Rename the room_map to rooms and remove the grouping indirection. When we want groups then we can add them under a separate key, either at the top-level or as part of the events themselves. --- api/client-server/v2_alpha/sync.yaml | 295 ++++++++++++++------------- 1 file changed, 153 insertions(+), 142 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 1b83d1d9d..7d60d6392 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -74,82 +74,92 @@ paths: rooms: type: object description: |- - The updates to rooms, grouped according to the filter. By - default there is a single ``default`` group. - additionalProperties: - joined: - type: array - description: |- - A list of room ids that the user is a member of that - have updates. - items: - type: string. - invited: - type: array - description: |- - A list of room ids that the user has been invited to. - The entries in the room_map will have an - ``invite_state`` key. But will lack the ``ephemeral`` - key, the ``timeline`` key and the ``state`` key. - items: - type: string. - archived: - type: array - description: |- - A list of room ids that the user has left or been - banned from. The entries in the room_map will have a - ``state`` key and a ``timeline`` key. But will lack the - ``ephemeral`` key. - items: - type: string. - room_map: - description: |- - Map from room id to the updates for that room. The room ids - are referenced from the ``rooms`` key. - type: object - additionalProperties: - type: object - properties: - event_map: + Updates to rooms. + properties: + joined: + type: object + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't + recorded in the timeline or state of the room. + e.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + invited: + type: object + description: |- + The rooms that the user has been invited to. + additionalProperties: type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" - invite_state: - description: |- - The state of a room that the user has been invited to. - These state events may only have the `sender``, - ``type``, ``state_key`` and ``content`` keys present. - These events do not replace any state that the client - already has for the room, for example if the client has - archived the room. Instead the client should keep two - separate copies of the state: the one from the - ``invite_state`` and one from the archived ``state``. - If the client joins the room then the current state - will be given as a delta against the archived ``state`` - not the ``invite_state``. - allOf: - - $ref: "definitions/event_batch.json" + properties: + invite_state: + description: |- + The state of a room that the user has been invited + to. These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys + present. These events do not replace any state that + the client already has for the room, for example if + the client has archived the room. Instead the + client should keep two separate copies of the + state: the one from the ``invite_state`` and one + from the archived ``state``. If the client joins + the room then the current state will be given as a + delta against the archived ``state`` not the + ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" + archived: + type: object + description: |- + The rooms that the user has left or been banned from. The + entries in the room_map will lack an ``ephemeral`` key. + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room up to the point when + the user left. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room up to the point when the user left. + allOf: + - $ref: "definitions/timeline_batch.json" presence: description: |- The updates to the presence status of other users. @@ -169,78 +179,79 @@ paths: ] }, "rooms": { - "default": { - "invited": ["!696r7674:example.com"], - "archived": [], - "joined": ["!726s6s6q:example.com"] - } - }, - "room_map": { - "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 - } - }, - "state": { - "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" - ] - }, - "timeline": { - "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" - ], - "limited": true, - "prev_batch": "t34-23535_0_0" - }, - "ephemeral": { - "events": [ - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - } - ] - } - }, - "!696r7674:example.com": { - "invite_state": { - "events": [ - { + "joined": { + "!726s6s6q:example.com": { + "event_map": { + "$66697273743031:example.com": { "sender": "@alice:example.com", - "type": "m.room.name", - "state_key": "", - "content": {"name": "My Room Name"} + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 }, - { - "sender": "@alice:example.com", + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"} + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797 } - ] + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] + } } - } + }, + "invited": { + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "invite"} + } + ] + } + } + }, + "archived": {} } } From 39faccb00d7d22a19f08724d47833dff2e53bec3 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 09:38:46 -0500 Subject: [PATCH 223/223] CS API is not just v1 surely --- specification/1-client_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index c63742d17..59e6b68ee 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1,5 +1,5 @@ -Client-Server API v1 -==================== +Client-Server API +================= Overview --------